cdransf/ai.robots.txt
1
1
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-06-19 10:00:52 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

chore(robots.json): adds Panscient

Browse source
This commit is contained in:
Cory Dransfeldt 2025-06-04 10:54:14 -07:00
parent 2b5a59a303
commit 4568d69b0e
No known key found for this signature in database
1 changed files with 15 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

14
robots.json
View file

@ -342,6 +342,20 @@
"frequency": "Unclear at this time.", "frequency": "Unclear at this time.",
"description": "PanguBot is a web crawler operated by the Chinese company Huawei. It's used to download training data for its multimodal LLM (Large Language Model) called PanGu. More info can be found at https://darkvisitors.com/agents/agents/pangubot" "description": "PanguBot is a web crawler operated by the Chinese company Huawei. It's used to download training data for its multimodal LLM (Large Language Model) called PanGu. More info can be found at https://darkvisitors.com/agents/agents/pangubot"
}, },
"Panscient": {
"operator": "[Panscient](https://panscient.com)",
"respect": "[Yes](https://panscient.com/faq.htm)",
"function": "Data collection and analysis using machine learning and AI.",
"frequency": "The Panscient web crawler will request a page at most once every second from the same domain name or the same IP address.",
"description": "Compiles data on businesses and business professionals that is structured using AI and machine learning."
},
"panscient.com": {
"operator": "[Panscient](https://panscient.com)",
"respect": "[Yes](https://panscient.com/faq.htm)",
"function": "Data collection and analysis using machine learning and AI.",
"frequency": "The Panscient web crawler will request a page at most once every second from the same domain name or the same IP address.",
"description": "Compiles data on businesses and business professionals that is structured using AI and machine learning."
},
"Perplexity-User": { "Perplexity-User": {
"operator": "[Perplexity](https://www.perplexity.ai/)", "operator": "[Perplexity](https://www.perplexity.ai/)",
"respect": "[No](https://docs.perplexity.ai/guides/bots)", "respect": "[No](https://docs.perplexity.ai/guides/bots)",