cdransf/ai.robots.txt
1
1
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-06-19 02:00:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Update from Dark Visitors
Some checks failed
/ run-tests (push) Waiting to run
Details
/ lint-json (push) Waiting to run
Details
/ ai-robots-txt (push) Has been cancelled
Details

Browse source
This commit is contained in:
dark-visitors 2025-05-22 00:58:45 +00:00
parent fedb658cc0
commit 7bf7f9164d
1 changed files with 19 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

19
robots.json
View file

@ -76,12 +76,12 @@
"frequency": "Only when prompted by a user.",
"description": "Used by plugins in ChatGPT to answer queries based on user input."
},
"ClaudeBot": {
"Claude-SearchBot": {
"operator": "[Anthropic](https://www.anthropic.com)",
"respect": "[Yes](https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler)",
"function": "Scrapes data to train Anthropic's AI products.",
"function": "Claude-SearchBot navigates the web to improve search result quality for users. It analyzes online content specifically to enhance the relevance and accuracy of search responses.",
"frequency": "No information provided.",
"description": "Scrapes data to train LLMs and AI products offered by Anthropic."
"description": "Claude-SearchBot navigates the web to improve search result quality for users. It analyzes online content specifically to enhance the relevance and accuracy of search responses."
},
"Claude-User": {
"operator": "[Anthropic](https://www.anthropic.com)",
@ -90,12 +90,19 @@
"frequency": "No information provided.",
"description": "Claude-User supports Claude AI users. When individuals ask questions to Claude, it may access websites using a Claude-User agent."
},
"Claude-SearchBot": {
"Claude-Web": {
"operator": "Anthropic",
"respect": "Unclear at this time.",
"function": "Undocumented AI Agents",
"frequency": "Unclear at this time.",
"description": "Claude-Web is an AI-related agent operated by Anthropic. It's currently unclear exactly what it's used for, since there's no official documentation. If you can provide more detail, please contact us. More info can be found at https://darkvisitors.com/agents/agents/claude-web"
},
"ClaudeBot": {
"operator": "[Anthropic](https://www.anthropic.com)",
"respect": "[Yes](https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler)",
"function": "Claude-SearchBot navigates the web to improve search result quality for users. It analyzes online content specifically to enhance the relevance and accuracy of search responses.",
"function": "Scrapes data to train Anthropic's AI products.",
"frequency": "No information provided.",
"description": "Claude-SearchBot navigates the web to improve search result quality for users. It analyzes online content specifically to enhance the relevance and accuracy of search responses."
"description": "Scrapes data to train LLMs and AI products offered by Anthropic."
},
"cohere-ai": {
"operator": "[Cohere](https://cohere.com)",