Update from Dark Visitors

2025-08-13 20:22:57 +00:00 · 2025-08-01 01:12:53 +00:00 · 2025-07-31 01:05:21 +00:00 · 2025-07-30 01:05:21 +00:00 · 2025-07-29 01:11:26 +00:00 · 2025-07-28 19:19:15 +00:00
9 changed files with 288 additions and 28 deletions
--- a/.htaccess
+++ b/.htaccess
@ -1,3 +1,3 @@
 RewriteEngine On
-RewriteCond %{HTTP_USER_AGENT} (AI2Bot|Ai2Bot\-Dolma|aiHitBot|Amazonbot|Andibot|anthropic\-ai|Applebot|Applebot\-Extended|Brightbot\ 1\.0|Bytespider|CCBot|ChatGPT\-User|Claude\-SearchBot|Claude\-User|Claude\-Web|ClaudeBot|cohere\-ai|cohere\-training\-data\-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google\-CloudVertexBot|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|meta\-externalagent|Meta\-ExternalAgent|meta\-externalfetcher|Meta\-ExternalFetcher|MistralAI\-User/1\.0|NovaAct|OAI\-SearchBot|omgili|omgilibot|Operator|PanguBot|Perplexity\-User|PerplexityBot|PetalBot|PhindBot|QualifiedBot|Scrapy|SemrushBot\-OCOB|SemrushBot\-SWA|Sidetrade\ indexer\ bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|wpbot|YandexAdditional|YandexAdditionalBot|YouBot) [NC]
-RewriteRule !^/?robots\.txt$ - [F,L]
+RewriteCond %{HTTP_USER_AGENT} (AddSearchBot|AI2Bot|Ai2Bot\-Dolma|aiHitBot|Amazonbot|Andibot|anthropic\-ai|Applebot|Applebot\-Extended|Awario|bedrockbot|bigsur\.ai|Brightbot\ 1\.0|Bytespider|CCBot|ChatGPT\ Agent|ChatGPT\-User|Claude\-SearchBot|Claude\-User|Claude\-Web|ClaudeBot|CloudVertexBot|cohere\-ai|cohere\-training\-data\-crawler|Cotoyogi|Crawlspace|Datenbank\ Crawler|Devin|Diffbot|DuckAssistBot|Echobot\ Bot|EchoboxBot|FacebookBot|facebookexternalhit|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Gemini\-Deep\-Research|Google\-CloudVertexBot|Google\-Extended|GoogleAgent\-Mariner|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|LinerBot|meta\-externalagent|Meta\-ExternalAgent|meta\-externalfetcher|Meta\-ExternalFetcher|MistralAI\-User|MistralAI\-User/1\.0|MyCentralAIScraperBot|netEstate\ Imprint\ Crawler|NovaAct|OAI\-SearchBot|omgili|omgilibot|Operator|PanguBot|Panscient|panscient\.com|Perplexity\-User|PerplexityBot|PetalBot|PhindBot|Poseidon\ Research\ Crawler|QualifiedBot|QuillBot|quillbot\.com|SBIntuitionsBot|Scrapy|SemrushBot\-OCOB|SemrushBot\-SWA|Sidetrade\ indexer\ bot|Thinkbot|TikTokSpider|Timpibot|VelenPublicWebCrawler|WARDBot|Webzio\-Extended|wpbot|YaK|YandexAdditional|YandexAdditionalBot|YouBot) [NC]
+RewriteRule !^/?robots\.txt$ - [F]
--- a/2
+++ b/2
@ -1,3 +1,3 @@
@aibots {
-        header_regexp User-Agent "(AI2Bot|Ai2Bot\-Dolma|aiHitBot|Amazonbot|Andibot|anthropic\-ai|Applebot|Applebot\-Extended|Brightbot\ 1\.0|Bytespider|CCBot|ChatGPT\-User|Claude\-SearchBot|Claude\-User|Claude\-Web|ClaudeBot|cohere\-ai|cohere\-training\-data\-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google\-CloudVertexBot|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|meta\-externalagent|Meta\-ExternalAgent|meta\-externalfetcher|Meta\-ExternalFetcher|MistralAI\-User/1\.0|NovaAct|OAI\-SearchBot|omgili|omgilibot|Operator|PanguBot|Perplexity\-User|PerplexityBot|PetalBot|PhindBot|QualifiedBot|Scrapy|SemrushBot\-OCOB|SemrushBot\-SWA|Sidetrade\ indexer\ bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|wpbot|YandexAdditional|YandexAdditionalBot|YouBot)"
+        header_regexp User-Agent "(AddSearchBot|AI2Bot|Ai2Bot\-Dolma|aiHitBot|Amazonbot|Andibot|anthropic\-ai|Applebot|Applebot\-Extended|Awario|bedrockbot|bigsur\.ai|Brightbot\ 1\.0|Bytespider|CCBot|ChatGPT\ Agent|ChatGPT\-User|Claude\-SearchBot|Claude\-User|Claude\-Web|ClaudeBot|CloudVertexBot|cohere\-ai|cohere\-training\-data\-crawler|Cotoyogi|Crawlspace|Datenbank\ Crawler|Devin|Diffbot|DuckAssistBot|Echobot\ Bot|EchoboxBot|FacebookBot|facebookexternalhit|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Gemini\-Deep\-Research|Google\-CloudVertexBot|Google\-Extended|GoogleAgent\-Mariner|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|LinerBot|meta\-externalagent|Meta\-ExternalAgent|meta\-externalfetcher|Meta\-ExternalFetcher|MistralAI\-User|MistralAI\-User/1\.0|MyCentralAIScraperBot|netEstate\ Imprint\ Crawler|NovaAct|OAI\-SearchBot|omgili|omgilibot|Operator|PanguBot|Panscient|panscient\.com|Perplexity\-User|PerplexityBot|PetalBot|PhindBot|Poseidon\ Research\ Crawler|QualifiedBot|QuillBot|quillbot\.com|SBIntuitionsBot|Scrapy|SemrushBot\-OCOB|SemrushBot\-SWA|Sidetrade\ indexer\ bot|Thinkbot|TikTokSpider|Timpibot|VelenPublicWebCrawler|WARDBot|Webzio\-Extended|wpbot|YaK|YandexAdditional|YandexAdditionalBot|YouBot)"
 }
--- a/code/robots.py
+++ b/code/robots.py
@ -169,7 +169,7 @@ def json_to_htaccess(robot_json):
    # User agents that contain any of the blocked values.
    htaccess = "RewriteEngine On\n"
    htaccess += f"RewriteCond %{{HTTP_USER_AGENT}} {list_to_pcre(robot_json.keys())} [NC]\n"
-    htaccess += "RewriteRule !^/?robots\\.txt$ - [F,L]\n"
+    htaccess += "RewriteRule !^/?robots\\.txt$ - [F]\n"
    return htaccess

 def json_to_nginx(robot_json):
--- a/code/test_files/.htaccess
+++ b/code/test_files/.htaccess
@ -1,3 +1,3 @@
 RewriteEngine On
 RewriteCond %{HTTP_USER_AGENT} (AI2Bot|Ai2Bot\-Dolma|Amazonbot|anthropic\-ai|Applebot|Applebot\-Extended|Bytespider|CCBot|ChatGPT\-User|Claude\-Web|ClaudeBot|cohere\-ai|Diffbot|FacebookBot|facebookexternalhit|FriendlyCrawler|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|Meta\-ExternalAgent|Meta\-ExternalFetcher|OAI\-SearchBot|omgili|omgilibot|Perplexity\-User|PerplexityBot|PetalBot|Scrapy|Sidetrade\ indexer\ bot|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|YouBot|crawler\.with\.dots|star\*\*\*crawler|Is\ this\ a\ crawler\?|a\[mazing\]\{42\}\(robot\)|2\^32\$|curl\|sudo\ bash) [NC]
-RewriteRule !^/?robots\.txt$ - [F,L]
+RewriteRule !^/?robots\.txt$ - [F]
--- a/haproxy-block-ai-bots.txt
+++ b/haproxy-block-ai-bots.txt
@ -1,3 +1,4 @@
+AddSearchBot
 AI2Bot
 Ai2Bot-Dolma
 aiHitBot
@ -6,26 +7,38 @@ Andibot
 anthropic-ai
 Applebot
 Applebot-Extended
+Awario
+bedrockbot
+bigsur.ai
 Brightbot 1.0
 Bytespider
 CCBot
+ChatGPT Agent
 ChatGPT-User
 Claude-SearchBot
 Claude-User
 Claude-Web
 ClaudeBot
+CloudVertexBot
 cohere-ai
 cohere-training-data-crawler
 Cotoyogi
 Crawlspace
+Datenbank Crawler
+Devin
 Diffbot
 DuckAssistBot
+Echobot Bot
+EchoboxBot
 FacebookBot
+facebookexternalhit
 Factset_spyderbot
 FirecrawlAgent
 FriendlyCrawler
+Gemini-Deep-Research
 Google-CloudVertexBot
 Google-Extended
+GoogleAgent-Mariner
 GoogleOther
 GoogleOther-Image
 GoogleOther-Video
@ -36,31 +49,44 @@ ImagesiftBot
 img2dataset
 ISSCyberRiskCrawler
 Kangaroo Bot
+LinerBot
 meta-externalagent
 Meta-ExternalAgent
 meta-externalfetcher
 Meta-ExternalFetcher
+MistralAI-User
 MistralAI-User/1.0
+MyCentralAIScraperBot
+netEstate Imprint Crawler
 NovaAct
 OAI-SearchBot
 omgili
 omgilibot
 Operator
 PanguBot
+Panscient
+panscient.com
 Perplexity-User
 PerplexityBot
 PetalBot
 PhindBot
+Poseidon Research Crawler
 QualifiedBot
+QuillBot
+quillbot.com
+SBIntuitionsBot
 Scrapy
 SemrushBot-OCOB
 SemrushBot-SWA
 Sidetrade indexer bot
+Thinkbot
 TikTokSpider
 Timpibot
 VelenPublicWebCrawler
+WARDBot
 Webzio-Extended
 wpbot
+YaK
 YandexAdditional
 YandexAdditionalBot
 YouBot
--- a/nginx-block-ai-bots.conf
+++ b/nginx-block-ai-bots.conf
@ -1,3 +1,3 @@
-if ($http_user_agent ~* "(AI2Bot|Ai2Bot\-Dolma|aiHitBot|Amazonbot|Andibot|anthropic\-ai|Applebot|Applebot\-Extended|Brightbot\ 1\.0|Bytespider|CCBot|ChatGPT\-User|Claude\-SearchBot|Claude\-User|Claude\-Web|ClaudeBot|cohere\-ai|cohere\-training\-data\-crawler|Cotoyogi|Crawlspace|Diffbot|DuckAssistBot|FacebookBot|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Google\-CloudVertexBot|Google\-Extended|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|meta\-externalagent|Meta\-ExternalAgent|meta\-externalfetcher|Meta\-ExternalFetcher|MistralAI\-User/1\.0|NovaAct|OAI\-SearchBot|omgili|omgilibot|Operator|PanguBot|Perplexity\-User|PerplexityBot|PetalBot|PhindBot|QualifiedBot|Scrapy|SemrushBot\-OCOB|SemrushBot\-SWA|Sidetrade\ indexer\ bot|TikTokSpider|Timpibot|VelenPublicWebCrawler|Webzio\-Extended|wpbot|YandexAdditional|YandexAdditionalBot|YouBot)") {
+if ($http_user_agent ~* "(AddSearchBot|AI2Bot|Ai2Bot\-Dolma|aiHitBot|Amazonbot|Andibot|anthropic\-ai|Applebot|Applebot\-Extended|Awario|bedrockbot|bigsur\.ai|Brightbot\ 1\.0|Bytespider|CCBot|ChatGPT\ Agent|ChatGPT\-User|Claude\-SearchBot|Claude\-User|Claude\-Web|ClaudeBot|CloudVertexBot|cohere\-ai|cohere\-training\-data\-crawler|Cotoyogi|Crawlspace|Datenbank\ Crawler|Devin|Diffbot|DuckAssistBot|Echobot\ Bot|EchoboxBot|FacebookBot|facebookexternalhit|Factset_spyderbot|FirecrawlAgent|FriendlyCrawler|Gemini\-Deep\-Research|Google\-CloudVertexBot|Google\-Extended|GoogleAgent\-Mariner|GoogleOther|GoogleOther\-Image|GoogleOther\-Video|GPTBot|iaskspider/2\.0|ICC\-Crawler|ImagesiftBot|img2dataset|ISSCyberRiskCrawler|Kangaroo\ Bot|LinerBot|meta\-externalagent|Meta\-ExternalAgent|meta\-externalfetcher|Meta\-ExternalFetcher|MistralAI\-User|MistralAI\-User/1\.0|MyCentralAIScraperBot|netEstate\ Imprint\ Crawler|NovaAct|OAI\-SearchBot|omgili|omgilibot|Operator|PanguBot|Panscient|panscient\.com|Perplexity\-User|PerplexityBot|PetalBot|PhindBot|Poseidon\ Research\ Crawler|QualifiedBot|QuillBot|quillbot\.com|SBIntuitionsBot|Scrapy|SemrushBot\-OCOB|SemrushBot\-SWA|Sidetrade\ indexer\ bot|Thinkbot|TikTokSpider|Timpibot|VelenPublicWebCrawler|WARDBot|Webzio\-Extended|wpbot|YaK|YandexAdditional|YandexAdditionalBot|YouBot)") {
    return 403;
 }
--- a/robots.json
+++ b/robots.json
@ -1,4 +1,11 @@
 {
+    "AddSearchBot": {
+        "operator": "Unclear at this time.",
+        "respect": "Unclear at this time.",
+        "function": "AI Search Crawlers",
+        "frequency": "Unclear at this time.",
+        "description": "AddSearchBot is a web crawler that indexes website content for AddSearch's AI-powered site search solution, collecting data to provide fast and accurate search results. More info can be found at https://darkvisitors.com/agents/agents/addsearchbot"
+    },
    "AI2Bot": {
        "operator": "[Ai2](https://allenai.org/crawler)",
        "respect": "Yes",
@ -55,6 +62,27 @@
        "frequency": "Unclear at this time.",
        "description": "Apple has a secondary user agent, Applebot-Extended ... [that is] used to train Apple's foundation models powering generative AI features across Apple products, including Apple Intelligence, Services, and Developer Tools."
    },
+    "Awario": {
+        "operator": "Awario",
+        "respect": "Unclear at this time.",
+        "function": "AI Data Scrapers",
+        "frequency": "Unclear at this time.",
+        "description": "Awario is an AI data scraper operated by Awario. It's not currently known to be artificially intelligent or AI-related. If you think that's incorrect or can provide more detail about its purpose, please contact us. More info can be found at https://darkvisitors.com/agents/agents/awario"
+    },
+    "bedrockbot": {
+        "operator": "[Amazon](https://amazon.com)",
+        "respect": "[Yes](https://docs.aws.amazon.com/bedrock/latest/userguide/webcrawl-data-source-connector.html#configuration-webcrawl-connector)",
+        "function": "Data scraping for custom AI applications.",
+        "frequency": "Unclear at this time.",
+        "description": "Connects to and crawls URLs that have been selected for use in a user's AWS bedrock application."
+    },
+    "bigsur.ai": {
+        "operator": "Big Sur AI that fetches website content to enable AI-powered web agents, sales assistants, and content marketing solutions for businesses",
+        "respect": "Unclear at this time.",
+        "function": "AI Assistants",
+        "frequency": "Unclear at this time.",
+        "description": "bigsur.ai is a web crawler operated by Big Sur AI that fetches website content to enable AI-powered web agents, sales assistants, and content marketing solutions for businesses. More info can be found at https://darkvisitors.com/agents/agents/bigsur-ai"
+    },
    "Brightbot 1.0": {
        "operator": "Browsing.ai",
        "respect": "Unclear at this time.",
@ -76,6 +104,13 @@
        "frequency": "Monthly at present.",
        "description": "Web archive going back to 2008. [Cited in thousands of research papers per year](https://commoncrawl.org/research-papers)."
    },
+    "ChatGPT Agent": {
+        "operator": "Unclear at this time.",
+        "respect": "Unclear at this time.",
+        "function": "AI Agents",
+        "frequency": "Unclear at this time.",
+        "description": "ChatGPT Agent is an AI agent created by OpenAI that can use a web browser. It can intelligently navigate and interact with websites to complete multi-step tasks on behalf of a human user. More info can be found at https://darkvisitors.com/agents/agents/chatgpt-agent"
+    },
    "ChatGPT-User": {
        "operator": "[OpenAI](https://openai.com)",
        "respect": "Yes",
@ -111,6 +146,13 @@
        "frequency": "No information provided.",
        "description": "Scrapes data to train LLMs and AI products offered by Anthropic."
    },
+    "CloudVertexBot": {
+        "operator": "Unclear at this time.",
+        "respect": "Unclear at this time.",
+        "function": "AI Data Scrapers",
+        "frequency": "Unclear at this time.",
+        "description": "CloudVertexBot is a Google-operated crawler available to site owners to request targeted crawls of their own sites for AI training purposes on the Vertex AI platform. More info can be found at https://darkvisitors.com/agents/agents/cloudvertexbot"
+    },
    "cohere-ai": {
        "operator": "[Cohere](https://cohere.com)",
        "respect": "Unclear at this time.",
@ -139,6 +181,20 @@
        "frequency": "Unclear at this time.",
        "description": "Provides crawling services for any purpose, probably including AI model training."
    },
+    "Datenbank Crawler": {
+        "operator": "Datenbank",
+        "respect": "Unclear at this time.",
+        "function": "AI Data Scrapers",
+        "frequency": "Unclear at this time.",
+        "description": "Datenbank Crawler is an AI data scraper operated by Datenbank. It's not currently known to be artificially intelligent or AI-related. If you think that's incorrect or can provide more detail about its purpose, please contact us. More info can be found at https://darkvisitors.com/agents/agents/datenbank-crawler"
+    },
+    "Devin": {
+        "operator": "Devin AI",
+        "respect": "Unclear at this time.",
+        "function": "AI Assistants",
+        "frequency": "Unclear at this time.",
+        "description": "Devin is an AI assistant operated by Devin AI. It's not currently known to be artificially intelligent or AI-related. If you think that's incorrect or can provide more detail about its purpose, please contact us. More info can be found at https://darkvisitors.com/agents/agents/devin"
+    },
    "Diffbot": {
        "operator": "[Diffbot](https://www.diffbot.com/)",
        "respect": "At the discretion of Diffbot users.",
@ -153,6 +209,20 @@
        "frequency": "Unclear at this time.",
        "description": "DuckAssistBot is used by DuckDuckGo's DuckAssist feature to fetch content and generate realtime AI answers to user searches. More info can be found at https://darkvisitors.com/agents/agents/duckassistbot"
    },
+    "Echobot Bot": {
+        "operator": "Echobox",
+        "respect": "Unclear at this time.",
+        "function": "AI Data Scrapers",
+        "frequency": "Unclear at this time.",
+        "description": "Echobot Bot is an AI data scraper operated by Echobox. It's not currently known to be artificially intelligent or AI-related. If you think that's incorrect or can provide more detail about its purpose, please contact us. More info can be found at https://darkvisitors.com/agents/agents/echobot-bot"
+    },
+    "EchoboxBot": {
+        "operator": "[Echobox](https://echobox.com)",
+        "respect": "Unclear at this time.",
+        "function": "Data collection to support AI-powered products.",
+        "frequency": "Unclear at this time.",
+        "description": "Supports company's AI-powered social and email management products."
+    },
    "FacebookBot": {
        "operator": "Meta/Facebook",
        "respect": "[Yes](https://developers.facebook.com/docs/sharing/bot/)",
@ -160,6 +230,13 @@
        "frequency": "Up to 1 page per second",
        "description": "Officially used for training Meta \"speech recognition technology,\" unknown if used to train Meta AI specifically."
    },
+    "facebookexternalhit": {
+        "operator": "Meta/Facebook",
+        "respect": "[No](https://github.com/ai-robots-txt/ai.robots.txt/issues/40#issuecomment-2524591313)",
+        "function": "Ostensibly only for sharing, but likely used as an AI crawler as well",
+        "frequency": "Unclear at this time.",
+        "description": "Note that excluding FacebookExternalHit will block incorporating OpenGraph data when sharing in social media, including rich links in Apple's Messages app. [According to Meta](https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/), its purpose is \"to crawl the content of an app or website that was shared on one of Meta\u2019s family of apps\u2026\". However, see discussions [here](https://github.com/ai-robots-txt/ai.robots.txt/pull/21) and [here](https://github.com/ai-robots-txt/ai.robots.txt/issues/40#issuecomment-2524591313) for evidence to the contrary."
+    },
    "Factset_spyderbot": {
        "operator": "[Factset](https://www.factset.com/ai)",
        "respect": "Unclear at this time.",
@ -181,6 +258,13 @@
        "operator": "Unknown",
        "respect": "[Yes](https://imho.alex-kunz.com/2024/01/25/an-update-on-friendly-crawler)"
    },
+    "Gemini-Deep-Research": {
+        "operator": "Unclear at this time.",
+        "respect": "Unclear at this time.",
+        "function": "AI Assistants",
+        "frequency": "Unclear at this time.",
+        "description": "Gemini-Deep-Research is the agent responsible for collecting and scanning resources used in Google Gemini's Deep Research feature, which acts as a personal research assistant. More info can be found at https://darkvisitors.com/agents/agents/gemini-deep-research"
+    },
    "Google-CloudVertexBot": {
        "operator": "Google",
        "respect": "[Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)",
@ -195,12 +279,19 @@
        "frequency": "No information.",
        "description": "Used to train Gemini and Vertex AI generative APIs. Does not impact a site's inclusion or ranking in Google Search."
    },
+    "GoogleAgent-Mariner": {
+        "operator": "Unclear at this time.",
+        "respect": "Unclear at this time.",
+        "function": "AI Agents",
+        "frequency": "Unclear at this time.",
+        "description": "GoogleAgent-Mariner is an AI agent created by Google that can use a web browser. It can intelligently navigate and interact with websites to complete multi-step tasks on behalf of a human user. More info can be found at https://darkvisitors.com/agents/agents/googleagent-mariner"
+    },
    "GoogleOther": {
-        "description": "\"Used by various product teams for fetching publicly accessible content from sites. For example, it may be used for one-off crawls for internal research and development.\"",
-        "frequency": "No information.",
-        "function": "Scrapes data.",
        "operator": "Google",
-        "respect": "[Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)"
+        "respect": "[Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)",
+        "function": "Scrapes data.",
+        "frequency": "No information.",
+        "description": "\"Used by various product teams for fetching publicly accessible content from sites. For example, it may be used for one-off crawls for internal research and development.\""
    },
    "GoogleOther-Image": {
        "description": "\"Used by various product teams for fetching publicly accessible content from sites. For example, it may be used for one-off crawls for internal research and development.\"",
@ -231,11 +322,11 @@
        "respect": "No"
    },
    "ICC-Crawler": {
-        "description": "Use the collected data for artificial intelligence technologies; provide data to third parties, including commercial companies; those companies can use the data for their own business.",
-        "frequency": "No information.",
-        "function": "Scrapes data to train and support AI technologies.",
        "operator": "[NICT](https://nict.go.jp)",
-        "respect": "Yes"
+        "respect": "Yes",
+        "function": "Scrapes data to train and support AI technologies.",
+        "frequency": "No information.",
+        "description": "Use the collected data for artificial intelligence technologies; provide data to third parties, including commercial companies; those companies can use the data for their own business."
    },
    "ImagesiftBot": {
        "description": "Once images and text are downloaded from a webpage, ImageSift analyzes this data from the page and stores the information in an index. Our web intelligence products use this index to enable search and retrieval of similar images.",
@ -265,6 +356,13 @@
        "frequency": "Unclear at this time.",
        "description": "Kangaroo Bot is used by the company Kangaroo LLM to download data to train AI models tailored to Australian language and culture. More info can be found at https://darkvisitors.com/agents/agents/kangaroo-bot"
    },
+    "LinerBot": {
+        "operator": "Unclear at this time.",
+        "respect": "Unclear at this time.",
+        "function": "AI Assistants",
+        "frequency": "Unclear at this time.",
+        "description": "LinerBot is the web crawler used by Liner AI assistant to gather information from academic sources and websites to provide accurate answers with line-by-line source citations for research and scholarly work. More info can be found at https://darkvisitors.com/agents/agents/linerbot"
+    },
    "meta-externalagent": {
        "operator": "[Meta](https://developers.facebook.com/docs/sharing/webmasters/web-crawlers)",
        "respect": "Yes",
@ -293,6 +391,13 @@
        "frequency": "Unclear at this time.",
        "description": "Meta-ExternalFetcher is dispatched by Meta AI products in response to user prompts, when they need to fetch an individual links. More info can be found at https://darkvisitors.com/agents/agents/meta-externalfetcher"
    },
+    "MistralAI-User": {
+        "operator": "Mistral",
+        "respect": "Unclear at this time.",
+        "function": "AI Assistants",
+        "frequency": "Unclear at this time.",
+        "description": "MistralAI-User is an AI assistant operated by Mistral. It's not currently known to be artificially intelligent or AI-related. If you think that's incorrect or can provide more detail about its purpose, please contact us. More info can be found at https://darkvisitors.com/agents/agents/mistralai-user"
+    },
    "MistralAI-User/1.0": {
        "operator": "Mistral AI",
        "function": "Takes action based on user prompts.",
@ -300,6 +405,20 @@
        "description": "MistralAI-User is for user actions in LeChat. When users ask LeChat a question, it may visit a web page to help answer and include a link to the source in its response.",
        "respect": "Yes"
    },
+    "MyCentralAIScraperBot": {
+        "operator": "Unclear at this time.",
+        "respect": "Unclear at this time.",
+        "function": "AI data scraper",
+        "frequency": "Unclear at this time.",
+        "description": "Operator and data use is unclear at this time."
+    },
+    "netEstate Imprint Crawler": {
+        "operator": "netEstate",
+        "respect": "Unclear at this time.",
+        "function": "AI Data Scrapers",
+        "frequency": "Unclear at this time.",
+        "description": "netEstate Imprint Crawler is an AI data scraper operated by netEstate. If you think this is incorrect or can provide additional detail about its purpose, please contact us. More info can be found at https://darkvisitors.com/agents/agents/netestate-imprint-crawler"
+    },
    "NovaAct": {
        "operator": "Unclear at this time.",
        "respect": "Unclear at this time.",
@ -342,6 +461,20 @@
        "frequency": "Unclear at this time.",
        "description": "PanguBot is a web crawler operated by the Chinese company Huawei. It's used to download training data for its multimodal LLM (Large Language Model) called PanGu. More info can be found at https://darkvisitors.com/agents/agents/pangubot"
    },
+    "Panscient": {
+        "operator": "[Panscient](https://panscient.com)",
+        "respect": "[Yes](https://panscient.com/faq.htm)",
+        "function": "Data collection and analysis using machine learning and AI.",
+        "frequency": "The Panscient web crawler will request a page at most once every second from the same domain name or the same IP address.",
+        "description": "Compiles data on businesses and business professionals that is structured using AI and machine learning."
+    },
+    "panscient.com": {
+        "operator": "[Panscient](https://panscient.com)",
+        "respect": "[Yes](https://panscient.com/faq.htm)",
+        "function": "Data collection and analysis using machine learning and AI.",
+        "frequency": "The Panscient web crawler will request a page at most once every second from the same domain name or the same IP address.",
+        "description": "Compiles data on businesses and business professionals that is structured using AI and machine learning."
+    },
    "Perplexity-User": {
        "operator": "[Perplexity](https://www.perplexity.ai/)",
        "respect": "[No](https://docs.perplexity.ai/guides/bots)",
@ -357,11 +490,11 @@
        "description": "Crawls sites to surface as results in Perplexity."
    },
    "PetalBot": {
-        "description": "Operated by Huawei to provide search and AI assistant services.",
-        "frequency": "No explicit frequency provided.",
-        "function": "Used to provide recommendations in Hauwei assistant and AI search services.",
        "operator": "[Huawei](https://huawei.com/)",
-        "respect": "Yes"
+        "respect": "Yes",
+        "function": "Used to provide recommendations in Hauwei assistant and AI search services.",
+        "frequency": "No explicit frequency provided.",
+        "description": "Operated by Huawei to provide search and AI assistant services."
    },
    "PhindBot": {
        "description": "Company offers an AI agent that uses AI and generate extra web query on the fly",
@ -370,13 +503,41 @@
        "operator": "[phind](https://www.phind.com/)",
        "respect": "Unclear at this time."
    },
-    "QualifiedBot": {
-        "description": "Operated by Qualified as part of their suite of AI product offerings.",
+    "Poseidon Research Crawler": {
+        "operator": "[Poseidon Research](https://www.poseidonresearch.com)",
+        "description": "Lab focused on scaling the interpretability research necessary to make better AI systems possible.",
        "frequency": "No explicit frequency provided.",
-        "function": "Company offers AI agents and other related products; usage can be assumed to support said products.",
-        "operator": "[Qualified](https://www.qualified.com)",
+        "function": "AI research crawler",
        "respect": "Unclear at this time."
    },
+    "QualifiedBot": {
+        "operator": "[Qualified](https://www.qualified.com)",
+        "respect": "Unclear at this time.",
+        "function": "Company offers AI agents and other related products; usage can be assumed to support said products.",
+        "frequency": "No explicit frequency provided.",
+        "description": "Operated by Qualified as part of their suite of AI product offerings."
+    },
+    "QuillBot": {
+        "description": "Operated by QuillBot as part of their suite of AI product offerings.",
+        "frequency": "No explicit frequency provided.",
+        "function": "Company offers AI detection, writing tools and other services.",
+        "operator": "[Quillbot](https://quillbot.com)",
+        "respect": "Unclear at this time."
+    },
+    "quillbot.com": {
+        "description": "Operated by QuillBot as part of their suite of AI product offerings.",
+        "frequency": "No explicit frequency provided.",
+        "function": "Company offers AI detection, writing tools and other services.",
+        "operator": "[Quillbot](https://quillbot.com)",
+        "respect": "Unclear at this time."
+    },
+    "SBIntuitionsBot": {
+        "description": "AI development and information analysis",
+        "respect": "[Yes](https://www.sbintuitions.co.jp/en/bot/)",
+        "frequency": "No information.",
+        "function": "Uses data gathered in AI development and information analysis.",
+        "operator": "[SB Intuitions](https://www.sbintuitions.co.jp/en/)"
+    },
    "Scrapy": {
        "description": "\"AI and machine learning applications often need large amounts of quality data, and web data extraction is a fast, efficient way to build structured data sets.\"",
        "frequency": "No information.",
@ -389,14 +550,14 @@
        "respect": "[Yes](https://www.semrush.com/bot/)",
        "function": "Crawls your site for ContentShake AI tool.",
        "frequency": "Roughly once every 10 seconds.",
-        "description": "You enter one text (on-demand) and we will make suggestions on it (the tool uses AI but we are not actively crawling the web, you need to manually enter one text/URL)."
+        "description": "Data collected is used for the ContentShake AI tool reports."
    },
    "SemrushBot-SWA": {
        "operator": "[Semrush](https://www.semrush.com/)",
        "respect": "[Yes](https://www.semrush.com/bot/)",
-        "function": "Checks URLs on your site for SWA tool.",
+        "function": "Checks URLs on your site for SEO Writing Assistant.",
        "frequency": "Roughly once every 10 seconds.",
-        "description": "You enter one text (on-demand) and we will make suggestions on it (the tool uses AI but we are not actively crawling the web, you need to manually enter one text/URL)."
+        "description": "Data collected is used for the SEO Writing Assistant tool to check if URL is accessible."
    },
    "Sidetrade indexer bot": {
        "description": "AI product training.",
@ -405,6 +566,13 @@
        "operator": "[Sidetrade](https://www.sidetrade.com)",
        "respect": "Unclear at this time."
    },
+    "Thinkbot": {
+        "operator": "[Thinkbot](https://www.thinkbot.agency)",
+        "respect": "No",
+        "function": "Insights on AI integration and automation.",
+        "frequency": "Unclear at this time.",
+        "description": "Collects data for analysis on AI usage and automation."
+    },
    "TikTokSpider": {
        "operator": "ByteDance",
        "respect": "Unclear at this time.",
@ -426,6 +594,13 @@
        "operator": "[Velen Crawler](https://velen.io)",
        "respect": "[Yes](https://velen.io)"
    },
+    "WARDBot": {
+        "operator": "WEBSPARK",
+        "respect": "Unclear at this time.",
+        "function": "AI Data Scrapers",
+        "frequency": "Unclear at this time.",
+        "description": "WARDBot is an AI data scraper operated by WEBSPARK. It's not currently known to be artificially intelligent or AI-related. If you think that's incorrect or can provide more detail about its purpose, please contact us. More info can be found at https://darkvisitors.com/agents/agents/wardbot"
+    },
    "Webzio-Extended": {
        "operator": "Unclear at this time.",
        "respect": "Unclear at this time.",
@ -440,6 +615,13 @@
        "frequency": "Unclear at this time.",
        "description": "wpbot is a used to support the functionality of the AI Chatbot for WordPress plugin. It supports the use of customer models, data collection and customer support."
    },
+    "YaK": {
+        "operator": "[Meltwater](https://www.meltwater.com/en/suite/consumer-intelligence)",
+        "respect": "Unclear at this time.",
+        "function": "According to the [Meltwater Consumer Intelligence page](https://www.meltwater.com/en/suite/consumer-intelligence) 'By applying AI, data science, and market research expertise to a live feed of global data sources, we transform unstructured data into actionable insights allowing better decision-making'.",
+        "frequency": "Unclear at this time.",
+        "description": "Retrieves data used for Meltwater's AI enabled consumer intelligence suite"
+    },
    "YandexAdditional": {
        "operator": "[Yandex](https://yandex.ru)",
        "respect": "[Yes](https://yandex.ru/support/webmaster/en/search-appearance/fast.html?lang=en)",
--- a/robots.txt
+++ b/robots.txt
@ -1,3 +1,4 @@
+User-agent: AddSearchBot
 User-agent: AI2Bot
 User-agent: Ai2Bot-Dolma
 User-agent: aiHitBot
@ -6,26 +7,38 @@ User-agent: Andibot
 User-agent: anthropic-ai
 User-agent: Applebot
 User-agent: Applebot-Extended
+User-agent: Awario
+User-agent: bedrockbot
+User-agent: bigsur.ai
 User-agent: Brightbot 1.0
 User-agent: Bytespider
 User-agent: CCBot
+User-agent: ChatGPT Agent
 User-agent: ChatGPT-User
 User-agent: Claude-SearchBot
 User-agent: Claude-User
 User-agent: Claude-Web
 User-agent: ClaudeBot
+User-agent: CloudVertexBot
 User-agent: cohere-ai
 User-agent: cohere-training-data-crawler
 User-agent: Cotoyogi
 User-agent: Crawlspace
+User-agent: Datenbank Crawler
+User-agent: Devin
 User-agent: Diffbot
 User-agent: DuckAssistBot
+User-agent: Echobot Bot
+User-agent: EchoboxBot
 User-agent: FacebookBot
+User-agent: facebookexternalhit
 User-agent: Factset_spyderbot
 User-agent: FirecrawlAgent
 User-agent: FriendlyCrawler
+User-agent: Gemini-Deep-Research
 User-agent: Google-CloudVertexBot
 User-agent: Google-Extended
+User-agent: GoogleAgent-Mariner
 User-agent: GoogleOther
 User-agent: GoogleOther-Image
 User-agent: GoogleOther-Video
@ -36,31 +49,44 @@ User-agent: ImagesiftBot
 User-agent: img2dataset
 User-agent: ISSCyberRiskCrawler
 User-agent: Kangaroo Bot
+User-agent: LinerBot
 User-agent: meta-externalagent
 User-agent: Meta-ExternalAgent
 User-agent: meta-externalfetcher
 User-agent: Meta-ExternalFetcher
+User-agent: MistralAI-User
 User-agent: MistralAI-User/1.0
+User-agent: MyCentralAIScraperBot
+User-agent: netEstate Imprint Crawler
 User-agent: NovaAct
 User-agent: OAI-SearchBot
 User-agent: omgili
 User-agent: omgilibot
 User-agent: Operator
 User-agent: PanguBot
+User-agent: Panscient
+User-agent: panscient.com
 User-agent: Perplexity-User
 User-agent: PerplexityBot
 User-agent: PetalBot
 User-agent: PhindBot
+User-agent: Poseidon Research Crawler
 User-agent: QualifiedBot
+User-agent: QuillBot
+User-agent: quillbot.com
+User-agent: SBIntuitionsBot
 User-agent: Scrapy
 User-agent: SemrushBot-OCOB
 User-agent: SemrushBot-SWA
 User-agent: Sidetrade indexer bot
+User-agent: Thinkbot
 User-agent: TikTokSpider
 User-agent: Timpibot
 User-agent: VelenPublicWebCrawler
+User-agent: WARDBot
 User-agent: Webzio-Extended
 User-agent: wpbot
+User-agent: YaK
 User-agent: YandexAdditional
 User-agent: YandexAdditionalBot
 User-agent: YouBot
--- a/table-of-bot-metrics.md
+++ b/table-of-bot-metrics.md
@ -1,5 +1,6 @@
 | Name | Operator | Respects `robots.txt` | Data use | Visit regularity | Description |
 |------|----------|-----------------------|----------|------------------|-------------|
+| AddSearchBot | Unclear at this time. | Unclear at this time. | AI Search Crawlers | Unclear at this time. | AddSearchBot is a web crawler that indexes website content for AddSearch's AI-powered site search solution, collecting data to provide fast and accurate search results. More info can be found at https://darkvisitors.com/agents/agents/addsearchbot |
 | AI2Bot | [Ai2](https://allenai.org/crawler) | Yes | Content is used to train open language models. | No information provided. | Explores 'certain domains' to find web content. |
 | Ai2Bot\-Dolma | [Ai2](https://allenai.org/crawler) | Yes | Content is used to train open language models. | No information provided. | Explores 'certain domains' to find web content. |
 | aiHitBot | [aiHit](https://www.aihitdata.com/about) | Yes | A massive, artificial intelligence/machine learning, automated system. | No information provided. | Scrapes data for AI systems. |
@ -8,26 +9,38 @@
 | anthropic\-ai | [Anthropic](https://www.anthropic.com) | Unclear at this time. | Scrapes data to train Anthropic's AI products. | No information provided. | Scrapes data to train LLMs and AI products offered by Anthropic. |
 | Applebot | Unclear at this time. | Unclear at this time. | AI Search Crawlers | Unclear at this time. | Applebot is a web crawler used by Apple to index search results that allow the Siri AI Assistant to answer user questions. Siri's answers normally contain references to the website. More info can be found at https://darkvisitors.com/agents/agents/applebot |
 | Applebot\-Extended | [Apple](https://support.apple.com/en-us/119829#datausage) | Yes | Powers features in Siri, Spotlight, Safari, Apple Intelligence, and others. | Unclear at this time. | Apple has a secondary user agent, Applebot-Extended ... [that is] used to train Apple's foundation models powering generative AI features across Apple products, including Apple Intelligence, Services, and Developer Tools. |
+| Awario | Awario | Unclear at this time. | AI Data Scrapers | Unclear at this time. | Awario is an AI data scraper operated by Awario. It's not currently known to be artificially intelligent or AI-related. If you think that's incorrect or can provide more detail about its purpose, please contact us. More info can be found at https://darkvisitors.com/agents/agents/awario |
+| bedrockbot | [Amazon](https://amazon.com) | [Yes](https://docs.aws.amazon.com/bedrock/latest/userguide/webcrawl-data-source-connector.html#configuration-webcrawl-connector) | Data scraping for custom AI applications. | Unclear at this time. | Connects to and crawls URLs that have been selected for use in a user's AWS bedrock application. |
+| bigsur\.ai | Big Sur AI that fetches website content to enable AI-powered web agents, sales assistants, and content marketing solutions for businesses | Unclear at this time. | AI Assistants | Unclear at this time. | bigsur.ai is a web crawler operated by Big Sur AI that fetches website content to enable AI-powered web agents, sales assistants, and content marketing solutions for businesses. More info can be found at https://darkvisitors.com/agents/agents/bigsur-ai |
 | Brightbot 1\.0 | Browsing.ai | Unclear at this time. | LLM/AI training. | Unclear at this time. | Scrapes data to train LLMs and AI products focused on website customer support. |
 | Bytespider | ByteDance | No | LLM training. | Unclear at this time. | Downloads data to train LLMS, including ChatGPT competitors. |
 | CCBot | [Common Crawl Foundation](https://commoncrawl.org) | [Yes](https://commoncrawl.org/ccbot) | Provides open crawl dataset, used for many purposes, including Machine Learning/AI. | Monthly at present. | Web archive going back to 2008. [Cited in thousands of research papers per year](https://commoncrawl.org/research-papers). |
+| ChatGPT Agent | Unclear at this time. | Unclear at this time. | AI Agents | Unclear at this time. | ChatGPT Agent is an AI agent created by OpenAI that can use a web browser. It can intelligently navigate and interact with websites to complete multi-step tasks on behalf of a human user. More info can be found at https://darkvisitors.com/agents/agents/chatgpt-agent |
 | ChatGPT\-User | [OpenAI](https://openai.com) | Yes | Takes action based on user prompts. | Only when prompted by a user. | Used by plugins in ChatGPT to answer queries based on user input. |
 | Claude\-SearchBot | [Anthropic](https://www.anthropic.com) | [Yes](https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler) | Claude-SearchBot navigates the web to improve search result quality for users. It analyzes online content specifically to enhance the relevance and accuracy of search responses. | No information provided. | Claude-SearchBot navigates the web to improve search result quality for users. It analyzes online content specifically to enhance the relevance and accuracy of search responses. |
 | Claude\-User | [Anthropic](https://www.anthropic.com) | [Yes](https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler) | Claude-User supports Claude AI users. When individuals ask questions to Claude, it may access websites using a Claude-User agent. | No information provided. | Claude-User supports Claude AI users. When individuals ask questions to Claude, it may access websites using a Claude-User agent. |
 | Claude\-Web | Anthropic | Unclear at this time. | Undocumented AI Agents | Unclear at this time. | Claude-Web is an AI-related agent operated by Anthropic. It's currently unclear exactly what it's used for, since there's no official documentation. If you can provide more detail, please contact us. More info can be found at https://darkvisitors.com/agents/agents/claude-web |
 | ClaudeBot | [Anthropic](https://www.anthropic.com) | [Yes](https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler) | Scrapes data to train Anthropic's AI products. | No information provided. | Scrapes data to train LLMs and AI products offered by Anthropic. |
+| CloudVertexBot | Unclear at this time. | Unclear at this time. | AI Data Scrapers | Unclear at this time. | CloudVertexBot is a Google-operated crawler available to site owners to request targeted crawls of their own sites for AI training purposes on the Vertex AI platform. More info can be found at https://darkvisitors.com/agents/agents/cloudvertexbot |
 | cohere\-ai | [Cohere](https://cohere.com) | Unclear at this time. | Retrieves data to provide responses to user-initiated prompts. | Takes action based on user prompts. | Retrieves data based on user prompts. |
 | cohere\-training\-data\-crawler | Cohere to download training data for its LLMs (Large Language Models) that power its enterprise AI products | Unclear at this time. | AI Data Scrapers | Unclear at this time. | cohere-training-data-crawler is a web crawler operated by Cohere to download training data for its LLMs (Large Language Models) that power its enterprise AI products. More info can be found at https://darkvisitors.com/agents/agents/cohere-training-data-crawler |
 | Cotoyogi | [ROIS](https://ds.rois.ac.jp/en_center8/en_crawler/) | Yes | AI LLM Scraper. | No information provided. | Scrapes data for AI training in Japanese language. |
 | Crawlspace | [Crawlspace](https://crawlspace.dev) | [Yes](https://news.ycombinator.com/item?id=42756654) | Scrapes data | Unclear at this time. | Provides crawling services for any purpose, probably including AI model training. |
+| Datenbank Crawler | Datenbank | Unclear at this time. | AI Data Scrapers | Unclear at this time. | Datenbank Crawler is an AI data scraper operated by Datenbank. It's not currently known to be artificially intelligent or AI-related. If you think that's incorrect or can provide more detail about its purpose, please contact us. More info can be found at https://darkvisitors.com/agents/agents/datenbank-crawler |
+| Devin | Devin AI | Unclear at this time. | AI Assistants | Unclear at this time. | Devin is an AI assistant operated by Devin AI. It's not currently known to be artificially intelligent or AI-related. If you think that's incorrect or can provide more detail about its purpose, please contact us. More info can be found at https://darkvisitors.com/agents/agents/devin |
 | Diffbot | [Diffbot](https://www.diffbot.com/) | At the discretion of Diffbot users. | Aggregates structured web data for monitoring and AI model training. | Unclear at this time. | Diffbot is an application used to parse web pages into structured data; this data is used for monitoring or AI model training. |
 | DuckAssistBot | Unclear at this time. | Unclear at this time. | AI Assistants | Unclear at this time. | DuckAssistBot is used by DuckDuckGo's DuckAssist feature to fetch content and generate realtime AI answers to user searches. More info can be found at https://darkvisitors.com/agents/agents/duckassistbot |
+| Echobot Bot | Echobox | Unclear at this time. | AI Data Scrapers | Unclear at this time. | Echobot Bot is an AI data scraper operated by Echobox. It's not currently known to be artificially intelligent or AI-related. If you think that's incorrect or can provide more detail about its purpose, please contact us. More info can be found at https://darkvisitors.com/agents/agents/echobot-bot |
+| EchoboxBot | [Echobox](https://echobox.com) | Unclear at this time. | Data collection to support AI-powered products. | Unclear at this time. | Supports company's AI-powered social and email management products. |
 | FacebookBot | Meta/Facebook | [Yes](https://developers.facebook.com/docs/sharing/bot/) | Training language models | Up to 1 page per second | Officially used for training Meta "speech recognition technology," unknown if used to train Meta AI specifically. |
+| facebookexternalhit | Meta/Facebook | [No](https://github.com/ai-robots-txt/ai.robots.txt/issues/40#issuecomment-2524591313) | Ostensibly only for sharing, but likely used as an AI crawler as well | Unclear at this time. | Note that excluding FacebookExternalHit will block incorporating OpenGraph data when sharing in social media, including rich links in Apple's Messages app. [According to Meta](https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/), its purpose is "to crawl the content of an app or website that was shared on one of Meta’s family of apps…". However, see discussions [here](https://github.com/ai-robots-txt/ai.robots.txt/pull/21) and [here](https://github.com/ai-robots-txt/ai.robots.txt/issues/40#issuecomment-2524591313) for evidence to the contrary. |
 | Factset\_spyderbot | [Factset](https://www.factset.com/ai) | Unclear at this time. | AI model training. | No information provided. | Scrapes data for AI training. |
 | FirecrawlAgent | [Firecrawl](https://www.firecrawl.dev/) | Yes | AI scraper and LLM training | No information provided. | Scrapes data for AI systems and LLM training. |
 | FriendlyCrawler | Unknown | [Yes](https://imho.alex-kunz.com/2024/01/25/an-update-on-friendly-crawler) | We are using the data from the crawler to build datasets for machine learning experiments. | Unclear at this time. | Unclear who the operator is; but data is used for training/machine learning. |
+| Gemini\-Deep\-Research | Unclear at this time. | Unclear at this time. | AI Assistants | Unclear at this time. | Gemini-Deep-Research is the agent responsible for collecting and scanning resources used in Google Gemini's Deep Research feature, which acts as a personal research assistant. More info can be found at https://darkvisitors.com/agents/agents/gemini-deep-research |
 | Google\-CloudVertexBot | Google | [Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers) | Build and manage AI models for businesses employing Vertex AI | No information. | Google-CloudVertexBot crawls sites on the site owners' request when building Vertex AI Agents. |
 | Google\-Extended | Google | [Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers) | LLM training. | No information. | Used to train Gemini and Vertex AI generative APIs. Does not impact a site's inclusion or ranking in Google Search. |
+| GoogleAgent\-Mariner | Unclear at this time. | Unclear at this time. | AI Agents | Unclear at this time. | GoogleAgent-Mariner is an AI agent created by Google that can use a web browser. It can intelligently navigate and interact with websites to complete multi-step tasks on behalf of a human user. More info can be found at https://darkvisitors.com/agents/agents/googleagent-mariner |
 | GoogleOther | Google | [Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers) | Scrapes data. | No information. | "Used by various product teams for fetching publicly accessible content from sites. For example, it may be used for one-off crawls for internal research and development." |
 | GoogleOther\-Image | Google | [Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers) | Scrapes data. | No information. | "Used by various product teams for fetching publicly accessible content from sites. For example, it may be used for one-off crawls for internal research and development." |
 | GoogleOther\-Video | Google | [Yes](https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers) | Scrapes data. | No information. | "Used by various product teams for fetching publicly accessible content from sites. For example, it may be used for one-off crawls for internal research and development." |
@ -38,31 +51,44 @@
 | img2dataset | [img2dataset](https://github.com/rom1504/img2dataset) | Unclear at this time. | Scrapes images for use in LLMs. | At the discretion of img2dataset users. | Downloads large sets of images into datasets for LLM training or other purposes. |
 | ISSCyberRiskCrawler | [ISS-Corporate](https://iss-cyber.com) | No | Scrapes data to train machine learning models. | No information. | Used to train machine learning based models to quantify cyber risk. |
 | Kangaroo Bot | Unclear at this time. | Unclear at this time. | AI Data Scrapers | Unclear at this time. | Kangaroo Bot is used by the company Kangaroo LLM to download data to train AI models tailored to Australian language and culture. More info can be found at https://darkvisitors.com/agents/agents/kangaroo-bot |
+| LinerBot | Unclear at this time. | Unclear at this time. | AI Assistants | Unclear at this time. | LinerBot is the web crawler used by Liner AI assistant to gather information from academic sources and websites to provide accurate answers with line-by-line source citations for research and scholarly work. More info can be found at https://darkvisitors.com/agents/agents/linerbot |
 | meta\-externalagent | [Meta](https://developers.facebook.com/docs/sharing/webmasters/web-crawlers) | Yes | Used to train models and improve products. | No information. | "The Meta-ExternalAgent crawler crawls the web for use cases such as training AI models or improving products by indexing content directly." |
 | Meta\-ExternalAgent | Unclear at this time. | Unclear at this time. | AI Data Scrapers | Unclear at this time. | Meta-ExternalAgent is a web crawler used by Meta to download training data for its AI models and improve its products by indexing content directly. More info can be found at https://darkvisitors.com/agents/agents/meta-externalagent |
 | meta\-externalfetcher | Unclear at this time. | Unclear at this time. | AI Assistants | Unclear at this time. | Meta-ExternalFetcher is dispatched by Meta AI products in response to user prompts, when they need to fetch an individual links. More info can be found at https://darkvisitors.com/agents/agents/meta-externalfetcher |
 | Meta\-ExternalFetcher | Unclear at this time. | Unclear at this time. | AI Assistants | Unclear at this time. | Meta-ExternalFetcher is dispatched by Meta AI products in response to user prompts, when they need to fetch an individual links. More info can be found at https://darkvisitors.com/agents/agents/meta-externalfetcher |
+| MistralAI\-User | Mistral | Unclear at this time. | AI Assistants | Unclear at this time. | MistralAI-User is an AI assistant operated by Mistral. It's not currently known to be artificially intelligent or AI-related. If you think that's incorrect or can provide more detail about its purpose, please contact us. More info can be found at https://darkvisitors.com/agents/agents/mistralai-user |
 | MistralAI\-User/1\.0 | Mistral AI | Yes | Takes action based on user prompts. | Only when prompted by a user. | MistralAI-User is for user actions in LeChat. When users ask LeChat a question, it may visit a web page to help answer and include a link to the source in its response. |
+| MyCentralAIScraperBot | Unclear at this time. | Unclear at this time. | AI data scraper | Unclear at this time. | Operator and data use is unclear at this time. |
+| netEstate Imprint Crawler | netEstate | Unclear at this time. | AI Data Scrapers | Unclear at this time. | netEstate Imprint Crawler is an AI data scraper operated by netEstate. If you think this is incorrect or can provide additional detail about its purpose, please contact us. More info can be found at https://darkvisitors.com/agents/agents/netestate-imprint-crawler |
 | NovaAct | Unclear at this time. | Unclear at this time. | AI Agents | Unclear at this time. | Nova Act is an AI agent created by Amazon that can use a web browser. It can intelligently navigate and interact with websites to complete multi-step tasks on behalf of a human user. More info can be found at https://darkvisitors.com/agents/agents/novaact |
 | OAI\-SearchBot | [OpenAI](https://openai.com) | [Yes](https://platform.openai.com/docs/bots) | Search result generation. | No information. | Crawls sites to surface as results in SearchGPT. |
 | omgili | [Webz.io](https://webz.io/) | [Yes](https://webz.io/blog/web-data/what-is-the-omgili-bot-and-why-is-it-crawling-your-website/) | Data is sold. | No information. | Crawls sites for APIs used by Hootsuite, Sprinklr, NetBase, and other companies. Data also sold for research purposes or LLM training. |
 | omgilibot | [Webz.io](https://webz.io/) | [Yes](https://web.archive.org/web/20170704003301/http://omgili.com/Crawler.html) | Data is sold. | No information. | Legacy user agent initially used for Omgili search engine. Unknown if still used, `omgili` agent still used by Webz.io. |
 | Operator | Unclear at this time. | Unclear at this time. | AI Agents | Unclear at this time. | Operator is an AI agent created by OpenAI that can use a web browser. It can intelligently navigate and interact with websites to complete multi-step tasks on behalf of a human user. More info can be found at https://darkvisitors.com/agents/agents/operator |
 | PanguBot | the Chinese company Huawei | Unclear at this time. | AI Data Scrapers | Unclear at this time. | PanguBot is a web crawler operated by the Chinese company Huawei. It's used to download training data for its multimodal LLM (Large Language Model) called PanGu. More info can be found at https://darkvisitors.com/agents/agents/pangubot |
+| Panscient | [Panscient](https://panscient.com) | [Yes](https://panscient.com/faq.htm) | Data collection and analysis using machine learning and AI. | The Panscient web crawler will request a page at most once every second from the same domain name or the same IP address. | Compiles data on businesses and business professionals that is structured using AI and machine learning. |
+| panscient\.com | [Panscient](https://panscient.com) | [Yes](https://panscient.com/faq.htm) | Data collection and analysis using machine learning and AI. | The Panscient web crawler will request a page at most once every second from the same domain name or the same IP address. | Compiles data on businesses and business professionals that is structured using AI and machine learning. |
 | Perplexity\-User | [Perplexity](https://www.perplexity.ai/) | [No](https://docs.perplexity.ai/guides/bots) | Used to answer queries at the request of users. | Only when prompted by a user. | Visit web pages to help provide an accurate answer and include links to the page in Perplexity response. |
 | PerplexityBot | [Perplexity](https://www.perplexity.ai/) | [Yes](https://docs.perplexity.ai/guides/bots) | Search result generation. | No information. | Crawls sites to surface as results in Perplexity. |
 | PetalBot | [Huawei](https://huawei.com/) | Yes | Used to provide recommendations in Hauwei assistant and AI search services. | No explicit frequency provided. | Operated by Huawei to provide search and AI assistant services. |
 | PhindBot | [phind](https://www.phind.com/) | Unclear at this time. | AI-enhanced search engine. | No explicit frequency provided. | Company offers an AI agent that uses AI and generate extra web query on the fly |
+| Poseidon Research Crawler | [Poseidon Research](https://www.poseidonresearch.com) | Unclear at this time. | AI research crawler | No explicit frequency provided. | Lab focused on scaling the interpretability research necessary to make better AI systems possible. |
 | QualifiedBot | [Qualified](https://www.qualified.com) | Unclear at this time. | Company offers AI agents and other related products; usage can be assumed to support said products. | No explicit frequency provided. | Operated by Qualified as part of their suite of AI product offerings. |
+| QuillBot | [Quillbot](https://quillbot.com) | Unclear at this time. | Company offers AI detection, writing tools and other services. | No explicit frequency provided. | Operated by QuillBot as part of their suite of AI product offerings. |
+| quillbot\.com | [Quillbot](https://quillbot.com) | Unclear at this time. | Company offers AI detection, writing tools and other services. | No explicit frequency provided. | Operated by QuillBot as part of their suite of AI product offerings. |
+| SBIntuitionsBot | [SB Intuitions](https://www.sbintuitions.co.jp/en/) | [Yes](https://www.sbintuitions.co.jp/en/bot/) | Uses data gathered in AI development and information analysis. | No information. | AI development and information analysis |
 | Scrapy | [Zyte](https://www.zyte.com) | Unclear at this time. | Scrapes data for a variety of uses including training AI. | No information. | "AI and machine learning applications often need large amounts of quality data, and web data extraction is a fast, efficient way to build structured data sets." |
-| SemrushBot\-OCOB | [Semrush](https://www.semrush.com/) | [Yes](https://www.semrush.com/bot/) | Crawls your site for ContentShake AI tool. | Roughly once every 10 seconds. | You enter one text (on-demand) and we will make suggestions on it (the tool uses AI but we are not actively crawling the web, you need to manually enter one text/URL). |
-| SemrushBot\-SWA | [Semrush](https://www.semrush.com/) | [Yes](https://www.semrush.com/bot/) | Checks URLs on your site for SWA tool. | Roughly once every 10 seconds. | You enter one text (on-demand) and we will make suggestions on it (the tool uses AI but we are not actively crawling the web, you need to manually enter one text/URL). |
+| SemrushBot\-OCOB | [Semrush](https://www.semrush.com/) | [Yes](https://www.semrush.com/bot/) | Crawls your site for ContentShake AI tool. | Roughly once every 10 seconds. | Data collected is used for the ContentShake AI tool reports. |
+| SemrushBot\-SWA | [Semrush](https://www.semrush.com/) | [Yes](https://www.semrush.com/bot/) | Checks URLs on your site for SEO Writing Assistant. | Roughly once every 10 seconds. | Data collected is used for the SEO Writing Assistant tool to check if URL is accessible. |
 | Sidetrade indexer bot | [Sidetrade](https://www.sidetrade.com) | Unclear at this time. | Extracts data for a variety of uses including training AI. | No information. | AI product training. |
+| Thinkbot | [Thinkbot](https://www.thinkbot.agency) | No | Insights on AI integration and automation. | Unclear at this time. | Collects data for analysis on AI usage and automation. |
 | TikTokSpider | ByteDance | Unclear at this time. | LLM training. | Unclear at this time. | Downloads data to train LLMS, as per Bytespider. |
 | Timpibot | [Timpi](https://timpi.io) | Unclear at this time. | Scrapes data for use in training LLMs. | No information. | Makes data available for training AI models. |
 | VelenPublicWebCrawler | [Velen Crawler](https://velen.io) | [Yes](https://velen.io) | Scrapes data for business data sets and machine learning models. | No information. | "Our goal with this crawler is to build business datasets and machine learning models to better understand the web." |
+| WARDBot | WEBSPARK | Unclear at this time. | AI Data Scrapers | Unclear at this time. | WARDBot is an AI data scraper operated by WEBSPARK. It's not currently known to be artificially intelligent or AI-related. If you think that's incorrect or can provide more detail about its purpose, please contact us. More info can be found at https://darkvisitors.com/agents/agents/wardbot |
 | Webzio\-Extended | Unclear at this time. | Unclear at this time. | AI Data Scrapers | Unclear at this time. | Webzio-Extended is a web crawler used by Webz.io to maintain a repository of web crawl data that it sells to other companies, including those using it to train AI models. More info can be found at https://darkvisitors.com/agents/agents/webzio-extended |
 | wpbot | [QuantumCloud](https://www.quantumcloud.com) | Unclear at this time; opt out provided via [Google Form](https://forms.gle/ajBaxygz9jSR8p8G9) | Live chat support and lead generation. | Unclear at this time. | wpbot is a used to support the functionality of the AI Chatbot for WordPress plugin. It supports the use of customer models, data collection and customer support. |
+| YaK | [Meltwater](https://www.meltwater.com/en/suite/consumer-intelligence) | Unclear at this time. | According to the [Meltwater Consumer Intelligence page](https://www.meltwater.com/en/suite/consumer-intelligence) 'By applying AI, data science, and market research expertise to a live feed of global data sources, we transform unstructured data into actionable insights allowing better decision-making'. | Unclear at this time. | Retrieves data used for Meltwater's AI enabled consumer intelligence suite |
 | YandexAdditional | [Yandex](https://yandex.ru) | [Yes](https://yandex.ru/support/webmaster/en/search-appearance/fast.html?lang=en) | Scrapes/analyzes data for the YandexGPT LLM. | No information. | Retrieves data used for YandexGPT quick answers features. |
 | YandexAdditionalBot | [Yandex](https://yandex.ru) | [Yes](https://yandex.ru/support/webmaster/en/search-appearance/fast.html?lang=en) | Scrapes/analyzes data for the YandexGPT LLM. | No information. | Retrieves data used for YandexGPT quick answers features. |
 | YouBot | [You](https://about.you.com/youchat/) | [Yes](https://about.you.com/youbot/) | Scrapes data for search engine and LLMs. | No information. | Retrieves data used for You.com web search engine and LLMs. |
Author	SHA1	Message	Date
ai.robots.txt	ff9fc26404	Update from Dark Visitors	2025-08-01 01:12:53 +00:00
dark-visitors	146a229662	Update from Dark Visitors	2025-07-31 01:05:21 +00:00
ai.robots.txt	64f9d6ce9c	Update from Dark Visitors	2025-07-30 01:05:21 +00:00
dark-visitors	085dd1071e	Update from Dark Visitors	2025-07-29 01:11:26 +00:00
ai.robots.txt	9565c11d4c	Merge pull request #166 from ai-robots-txt/summaly-bot fix: remove summaly	2025-07-28 19:19:15 +00:00
Glyn Normington	8869442615	Merge pull request #166 from ai-robots-txt/summaly-bot fix: remove summaly	2025-07-28 20:19:06 +01:00
ai.robots.txt	27420d6fed	Merge pull request #164 from GlitzSmarter/main Adding YaK	2025-07-28 18:29:57 +00:00
Glyn Normington	44e58a5ece	Merge pull request #164 from GlitzSmarter/main Adding YaK	2025-07-28 19:29:48 +01:00
Glyn Normington	12c5368e04	Fix syntax	2025-07-28 19:29:19 +01:00
Glitz Smarter	c15065544a	Rewriting function in YaK Rewrote the function to make it clear the quote is according to the page on the Clearwater website.	2025-07-28 12:54:25 -04:00
Cory Dransfeldt	9171625db6	fix: remove summaly	2025-07-28 08:56:21 -07:00
Gregory	8b188d0612	Added YaK user-agent to list of robots Added Meltwater's AI to list of bots that use AI and scrape websites.	2025-07-26 23:50:11 -04:00
ai.robots.txt	47769c8429	Update from Dark Visitors	2025-07-17 01:04:20 +00:00
dark-visitors	9843c31303	Update from Dark Visitors	2025-07-16 01:03:43 +00:00
ai.robots.txt	567e94a6cf	Merge pull request #160 from sdavids/users/sdavids/feat/htaccess Simplify htaccess rewrite rule	2025-07-08 16:00:16 +00:00
Glyn Normington	30c4c037d9	Merge pull request #160 from sdavids/users/sdavids/feat/htaccess Simplify htaccess rewrite rule	2025-07-08 16:59:52 +01:00
Sebastian Davids	c7407e721d	Simplify htaccess rewrite rule https://httpd.apache.org/docs/2.4/rewrite/flags.html#flag_f fix #159 Signed-off-by: Sebastian Davids <sdavids@gmx.de>	2025-07-08 12:33:24 +02:00
dark-visitors	f6f504012f	Update from Dark Visitors	2025-07-08 01:01:25 +00:00
ai.robots.txt	63f1f56307	Merge pull request #158 from ai-robots-txt/thinkbot-summaly chore(robots.json): add SummalyBot + Thinkbot	2025-07-07 16:55:31 +00:00
Glyn Normington	91a5b3c995	Merge pull request #158 from ai-robots-txt/thinkbot-summaly chore(robots.json): add SummalyBot + Thinkbot	2025-07-07 17:55:22 +01:00
Cory Dransfeldt	c5dd4e98b6	chore(robots.json): add SummalyBot + Thinkbot	2025-07-07 09:49:26 -07:00
dark-visitors	70ff0c9fbd	Update from Dark Visitors	2025-07-03 01:00:49 +00:00
ai.robots.txt	20a242d390	Merge pull request #156 from nisbet-hubbard/patch-9 Update funtion and description for SemrushBot	2025-07-02 15:58:19 +00:00
Cory Dransfeldt	91ad302d68	Merge pull request #156 from nisbet-hubbard/patch-9 Update funtion and description for SemrushBot	2025-07-02 08:58:08 -07:00
dark-visitors	79bc22523f	Update from Dark Visitors	2025-07-02 01:01:05 +00:00
ai.robots.txt	2bfcee794d	Update from Dark Visitors	2025-06-29 01:07:47 +00:00
dark-visitors	7eb2099d3f	Update from Dark Visitors	2025-06-28 00:58:57 +00:00
nisbet-hubbard	accc48f327	Update funtion and description for SemrushBot	2025-06-21 08:48:58 +08:00
dark-visitors	4ed17b8e4a	Update from Dark Visitors	2025-06-17 01:00:21 +00:00
ai.robots.txt	5326c202b5	Merge pull request #154 from paulrudy/main re-add facebookexternalhit	2025-06-16 15:12:42 +00:00
Cory Dransfeldt	a31ae1e6d0	Merge pull request #154 from paulrudy/main re-add facebookexternalhit	2025-06-16 08:12:31 -07:00
paulrudy	7535893aec	re-add facebookexternalhit	2025-06-15 16:49:07 -07:00
ai.robots.txt	eb05f2f527	Merge pull request #153 from sergiospagnuolo/Poseidon Update robots.json with new crawler	2025-06-14 14:04:03 +00:00
Cory Dransfeldt	26a46c409d	Merge pull request #153 from sergiospagnuolo/Poseidon Update robots.json with new crawler	2025-06-14 07:03:52 -07:00
dark-visitors	2b68568ac2	Update from Dark Visitors	2025-06-14 00:58:11 +00:00
Sérgio Spagnuolo	b05f2fee00	Update robots.json with new crawler Update with Poseidon Research Crawler as found in nytimes.com/robots.txt	2025-06-13 17:15:13 -03:00
ai.robots.txt	e53d81c66d	Merge pull request #152 from ai-robots-txt/MyCentralAIScraperBot chore(robots.json): adds MyCentralAIScraperBot	2025-06-13 09:28:41 +00:00
Glyn Normington	20e327e74e	Merge pull request #152 from ai-robots-txt/MyCentralAIScraperBot chore(robots.json): adds MyCentralAIScraperBot	2025-06-13 10:28:32 +01:00
Glyn Normington	8f17718e76	Fix typo	2025-06-13 10:28:12 +01:00
Cory Dransfeldt	d760f9216f	chore(robots.json): adds MyCentralAIScraperBot	2025-06-12 13:08:29 -07:00
ai.robots.txt	842e2256e8	Merge pull request #150 from ai-robots-txt/semrush-bots chore(robots.json): adds additional SemrushBot user agents	2025-06-12 07:12:00 +00:00
Glyn Normington	229ea20426	Merge pull request #150 from ai-robots-txt/semrush-bots chore(robots.json): adds additional SemrushBot user agents	2025-06-12 08:11:51 +01:00
Cory Dransfeldt	14d68f05ba	chore(robots.json): adds additional SemrushBot user agents	2025-06-11 13:50:53 -07:00
dark-visitors	cf598b6b71	Update from Dark Visitors	2025-06-10 01:00:37 +00:00
ai.robots.txt	3759a6bf14	chore(robots.json): adds EchoboxBot (#148 )	2025-06-09 15:44:36 +00:00
Cory Dransfeldt	7867c3e26c	chore(robots.json): adds EchoboxBot (#148 )	2025-06-09 16:44:25 +01:00
dark-visitors	e21f6ae1b6	Update from Dark Visitors	2025-06-06 00:59:25 +00:00
ai.robots.txt	ac7ed17e71	Merge pull request #145 from ai-robots-txt/aws-bedrockbot chore(robots.json): adds bedrockbot	2025-06-05 16:51:17 +00:00
Glyn Normington	81747e6772	Merge pull request #145 from ai-robots-txt/aws-bedrockbot chore(robots.json): adds bedrockbot	2025-06-05 17:51:03 +01:00
Cory Dransfeldt	528d77bf07	chore(robots.json): adds bedrockbot	2025-06-05 09:14:23 -07:00
dark-visitors	77393df5aa	Update from Dark Visitors	2025-06-05 00:59:28 +00:00
ai.robots.txt	75ea75a95b	Merge pull request #143 from ai-robots-txt/panscient chore(robots.json): adds Panscient	2025-06-04 18:04:06 +00:00
Glyn Normington	2fca1ddcf1	Merge pull request #143 from ai-robots-txt/panscient chore(robots.json): adds Panscient	2025-06-04 19:03:53 +01:00
ai.robots.txt	9c28c63a0c	Merge pull request #142 from ai-robots-txt/quillbot chore(robots.json): adds Quillbot	2025-06-04 17:54:57 +00:00
Cory Dransfeldt	395c013eea	Merge pull request #142 from ai-robots-txt/quillbot chore(robots.json): adds Quillbot	2025-06-04 10:54:46 -07:00
Cory Dransfeldt	4568d69b0e	chore(robots.json): adds Panscient	2025-06-04 10:54:14 -07:00
Cory Dransfeldt	03831a7eb5	chore(robots.json): adds Quillbot	2025-06-04 10:46:58 -07:00
dark-visitors	2b5a59a303	Update from Dark Visitors	2025-06-04 01:00:07 +00:00
ai.robots.txt	3efabc603d	Merge pull request #141 from Ivan-Chupin/patch-1 Add SBIntuitionsBot	2025-06-03 23:28:48 +00:00
Cory Dransfeldt	b35f9a31d7	Merge pull request #141 from Ivan-Chupin/patch-1 Add SBIntuitionsBot	2025-06-03 16:28:36 -07:00
Ivan Chupin	8f75f4a2f5	Add SBIntuitionsBot	2025-06-04 03:48:42 +05:00