From ae8f74c10cec97ec758bf39345ff20717302c665 Mon Sep 17 00:00:00 2001
From: Kyle Buckingham <kyle@getsphere.ai>
Date: Tue, 1 Apr 2025 15:22:04 -0700
Subject: [PATCH 1/2] Update robots.json
---
robots.json | 23 +++++++++++++++--------
1 file changed, 15 insertions(+), 8 deletions(-)
diff --git a/robots.json b/robots.json
index e907c8b..f711d43 100644
--- a/robots.json
+++ b/robots.json
@@ -69,13 +69,6 @@
"frequency": "Only when prompted by a user.",
"description": "Used by plugins in ChatGPT to answer queries based on user input."
},
- "Claude-Web": {
- "operator": "[Anthropic](https://www.anthropic.com)",
- "respect": "Unclear at this time.",
- "function": "Scrapes data to train Anthropic's AI products.",
- "frequency": "No information provided.",
- "description": "Scrapes data to train LLMs and AI products offered by Anthropic."
- },
"ClaudeBot": {
"operator": "[Anthropic](https://www.anthropic.com)",
"respect": "[Yes](https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler)",
@@ -83,6 +76,20 @@
"frequency": "No information provided.",
"description": "Scrapes data to train LLMs and AI products offered by Anthropic."
},
+ "Claude-User": {
+ "operator": "[Anthropic](https://www.anthropic.com)",
+ "respect": "Unclear at this time.",
+ "function": "Claude-User supports Claude AI users. When individuals ask questions to Claude, it may access websites using a Claude-User agent.",
+ "frequency": "No information provided.",
+ "description": "Claude-User supports Claude AI users. When individuals ask questions to Claude, it may access websites using a Claude-User agent."
+ },
+ "Claude-SearchBot": {
+ "operator": "[Anthropic](https://www.anthropic.com)",
+ "respect": "Unclear at this time.",
+ "function": "Claude-SearchBot navigates the web to improve search result quality for users. It analyzes online content specifically to enhance the relevance and accuracy of search responses.",
+ "frequency": "No information provided.",
+ "description": "Claude-SearchBot navigates the web to improve search result quality for users. It analyzes online content specifically to enhance the relevance and accuracy of search responses."
+ },
"cohere-ai": {
"operator": "[Cohere](https://cohere.com)",
"respect": "Unclear at this time.",
@@ -328,4 +335,4 @@
"frequency": "No information.",
"description": "Retrieves data used for You.com web search engine and LLMs."
}
-}
\ No newline at end of file
+}
From 8dc36aa2e2bbc9b99bde043b593cdc6c9669f401 Mon Sep 17 00:00:00 2001
From: Kyle Buckingham <kyle@getsphere.ai>
Date: Tue, 1 Apr 2025 15:23:28 -0700
Subject: [PATCH 2/2] Update robots.txt
---
robots.txt | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/robots.txt b/robots.txt
index 8c79fc2..e19468d 100644
--- a/robots.txt
+++ b/robots.txt
@@ -8,8 +8,9 @@ User-agent: Brightbot 1.0
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGPT-User
-User-agent: Claude-Web
User-agent: ClaudeBot
+User-agent: Claude-User
+User-agent: Claude-SearchBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Crawlspace