diff --git a/code/robots.py b/code/robots.py index 8a06b55..86ea413 100755 --- a/code/robots.py +++ b/code/robots.py @@ -30,7 +30,6 @@ def updated_robots_json(soup): """Update AI scraper information with data from darkvisitors.""" existing_content = load_robots_json() to_include = [ - "AI Agents", "AI Assistants", "AI Data Scrapers", "AI Search Crawlers",