cdransf/ai.robots.txt
1
1
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-06-19 02:00:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge pull request #143 from ai-robots-txt/panscient

Browse source 
chore(robots.json): adds Panscient
This commit is contained in:
Glyn Normington 2025-06-04 19:03:53 +01:00 • committed by GitHub
commit 2fca1ddcf1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 14 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

14
robots.json
View file

@ -342,6 +342,20 @@
"frequency": "Unclear at this time.",
"description": "PanguBot is a web crawler operated by the Chinese company Huawei. It's used to download training data for its multimodal LLM (Large Language Model) called PanGu. More info can be found at https://darkvisitors.com/agents/agents/pangubot"
},
"Panscient": {
"operator": "[Panscient](https://panscient.com)",
"respect": "[Yes](https://panscient.com/faq.htm)",
"function": "Data collection and analysis using machine learning and AI.",
"frequency": "The Panscient web crawler will request a page at most once every second from the same domain name or the same IP address.",
"description": "Compiles data on businesses and business professionals that is structured using AI and machine learning."
},
"panscient.com": {
"operator": "[Panscient](https://panscient.com)",
"respect": "[Yes](https://panscient.com/faq.htm)",
"function": "Data collection and analysis using machine learning and AI.",
"frequency": "The Panscient web crawler will request a page at most once every second from the same domain name or the same IP address.",
"description": "Compiles data on businesses and business professionals that is structured using AI and machine learning."
},
"Perplexity-User": {
"operator": "[Perplexity](https://www.perplexity.ai/)",
"respect": "[No](https://docs.perplexity.ai/guides/bots)",