2025-06-19 18:10:52 +00:00
1 changed files with 0 additions and 1 deletions
--- a/code/robots.py
+++ b/code/robots.py
@ -30,7 +30,6 @@ def updated_robots_json(soup):
    """Update AI scraper information with data from darkvisitors."""
    existing_content = load_robots_json()
    to_include = [
-        "AI Agents",
        "AI Assistants",
        "AI Data Scrapers",
        "AI Search Crawlers",