cdransf/ai.robots.txt
1
1
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-06-19 10:00:52 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

chore(robots.json): adds additional SemrushBot user agents

Browse source
This commit is contained in:
Cory Dransfeldt 2025-06-11 13:50:53 -07:00
parent cf598b6b71
commit 14d68f05ba
No known key found for this signature in database
1 changed files with 28 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

28
robots.json
View file

@ -433,6 +433,27 @@
"operator": "[Zyte](https://www.zyte.com)", "operator": "[Zyte](https://www.zyte.com)",
"respect": "Unclear at this time." "respect": "Unclear at this time."
}, },
"SemrushBot": {
"operator": "[Semrush](https://www.semrush.com/)",
"respect": "[Yes](https://www.semrush.com/bot/)",
"function": "Crawls your site for ContentShake AI tool.",
"frequency": "Roughly once every 10 seconds.",
"description": "You enter one text (on-demand) and we will make suggestions on it (the tool uses AI but we are not actively crawling the web, you need to manually enter one text/URL)."
},
"SemrushBot-BA": {
"operator": "[Semrush](https://www.semrush.com/)",
"respect": "[Yes](https://www.semrush.com/bot/)",
"function": "Crawls your site for ContentShake AI tool.",
"frequency": "Roughly once every 10 seconds.",
"description": "You enter one text (on-demand) and we will make suggestions on it (the tool uses AI but we are not actively crawling the web, you need to manually enter one text/URL)."
},
"SemrushBot-CT": {
"operator": "[Semrush](https://www.semrush.com/)",
"respect": "[Yes](https://www.semrush.com/bot/)",
"function": "Crawls your site for ContentShake AI tool.",
"frequency": "Roughly once every 10 seconds.",
"description": "You enter one text (on-demand) and we will make suggestions on it (the tool uses AI but we are not actively crawling the web, you need to manually enter one text/URL)."
},
"SemrushBot-OCOB": { "SemrushBot-OCOB": {
"operator": "[Semrush](https://www.semrush.com/)", "operator": "[Semrush](https://www.semrush.com/)",
"respect": "[Yes](https://www.semrush.com/bot/)", "respect": "[Yes](https://www.semrush.com/bot/)",
@ -440,6 +461,13 @@
"frequency": "Roughly once every 10 seconds.", "frequency": "Roughly once every 10 seconds.",
"description": "You enter one text (on-demand) and we will make suggestions on it (the tool uses AI but we are not actively crawling the web, you need to manually enter one text/URL)." "description": "You enter one text (on-demand) and we will make suggestions on it (the tool uses AI but we are not actively crawling the web, you need to manually enter one text/URL)."
}, },
"SemrushBot-SI": {
"operator": "[Semrush](https://www.semrush.com/)",
"respect": "[Yes](https://www.semrush.com/bot/)",
"function": "Crawls your site for ContentShake AI tool.",
"frequency": "Roughly once every 10 seconds.",
"description": "You enter one text (on-demand) and we will make suggestions on it (the tool uses AI but we are not actively crawling the web, you need to manually enter one text/URL)."
},
"SemrushBot-SWA": { "SemrushBot-SWA": {
"operator": "[Semrush](https://www.semrush.com/)", "operator": "[Semrush](https://www.semrush.com/)",
"respect": "[Yes](https://www.semrush.com/bot/)", "respect": "[Yes](https://www.semrush.com/bot/)",