cdransf/ai.robots.txt
1
1
Fork 0
mirror of https://github.com/ai-robots-txt/ai.robots.txt.git synced 2025-06-19 18:10:52 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

feat: add ua-synonyms support with tests, scoped to json_to_* functions

Browse source
This commit is contained in:
brndnprog 2025-06-12 08:31:54 -07:00
parent f0606f0eac
commit 3148669363
2 changed files with 138 additions and 61 deletions
Download patch file Download diff file Expand all files Collapse all files

31
code/tests.py
View file

@ -1,7 +1,9 @@
#!/usr/bin/env python3
"""To run these tests just execute this script."""
import json
import re
import unittest
import re
from robots import (
json_to_txt,
@ -10,6 +12,7 @@ from robots import (
json_to_nginx,
json_to_haproxy,
json_to_caddy,
clean_robot_name
)
@ -26,8 +29,10 @@ class RobotsUnittestExtensions:
class TestRobotsTXTGeneration(unittest.TestCase, RobotsUnittestExtensions):
maxDiff = 8192
def setUp(self):
self.robots_dict = self.loadJson("test_files/robots.json")
def test_robots_txt_generation(self):
robots_txt = json_to_txt(self.robots_dict)
self.assertEqualsFile("test_files/robots.txt", robots_txt)
@ -35,8 +40,10 @@ class TestRobotsTXTGeneration(unittest.TestCase, RobotsUnittestExtensions):
class TestTableMetricsGeneration(unittest.TestCase, RobotsUnittestExtensions):
maxDiff = 32768
def setUp(self):
self.robots_dict = self.loadJson("test_files/robots.json")
def test_table_generation(self):
robots_table = json_to_table(self.robots_dict)
self.assertEqualsFile("test_files/table-of-bot-metrics.md", robots_table)
@ -44,8 +51,10 @@ class TestTableMetricsGeneration(unittest.TestCase, RobotsUnittestExtensions):
class TestHtaccessGeneration(unittest.TestCase, RobotsUnittestExtensions):
maxDiff = 8192
def setUp(self):
self.robots_dict = self.loadJson("test_files/robots.json")
def test_htaccess_generation(self):
robots_htaccess = json_to_htaccess(self.robots_dict)
self.assertEqualsFile("test_files/.htaccess", robots_htaccess)
@ -53,8 +62,10 @@ class TestHtaccessGeneration(unittest.TestCase, RobotsUnittestExtensions):
class TestNginxConfigGeneration(unittest.TestCase, RobotsUnittestExtensions):
maxDiff = 8192
def setUp(self):
self.robots_dict = self.loadJson("test_files/robots.json")
def test_nginx_generation(self):
robots_nginx = json_to_nginx(self.robots_dict)
self.assertEqualsFile("test_files/nginx-block-ai-bots.conf", robots_nginx)
@ -62,8 +73,10 @@ class TestNginxConfigGeneration(unittest.TestCase, RobotsUnittestExtensions):
class TestHaproxyConfigGeneration(unittest.TestCase, RobotsUnittestExtensions):
maxDiff = 8192
def setUp(self):
self.robots_dict = self.loadJson("test_files/robots.json")
def test_haproxy_generation(self):
robots_haproxy = json_to_haproxy(self.robots_dict)
self.assertEqualsFile("test_files/haproxy-block-ai-bots.txt", robots_haproxy)
@ -71,8 +84,10 @@ class TestHaproxyConfigGeneration(unittest.TestCase, RobotsUnittestExtensions):
class TestCaddyfileGeneration(unittest.TestCase, RobotsUnittestExtensions):
maxDiff = 8192
def setUp(self):
self.robots_dict = self.loadJson("test_files/robots.json")
def test_caddyfile_generation(self):
robots_caddyfile = json_to_caddy(self.robots_dict)
self.assertEqualsFile("test_files/Caddyfile", robots_caddyfile)
@ -80,20 +95,19 @@ class TestCaddyfileGeneration(unittest.TestCase, RobotsUnittestExtensions):
class TestRobotsNameCleaning(unittest.TestCase):
def test_clean_name(self):
from robots import clean_robot_name
self.assertEqual(clean_robot_name("Perplexity‑User"), "Perplexity-User")
class TestUASynonymsSupport(unittest.TestCase):
class TestUASynonymSupport(unittest.TestCase):
def setUp(self):
self.test_data = {
"MainBot": {
"ua-synonyms": ["mainbot/1.0", "Main-Bot"],
"operator": "TestCorp",
"respect": "No",
"function": "AI Bot",
"frequency": "Daily",
"description": "Used for testing ua-synonyms."
"function": "Test",
"frequency": "Often",
"description": "A test bot"
}
}
@ -104,11 +118,12 @@ class TestUASynonymsSupport(unittest.TestCase):
def test_htaccess_includes_synonyms(self):
output = json_to_htaccess(self.test_data)
for variant in ["MainBot", "mainbot/1.0", "Main-Bot"]:
self.assertIn(re.escape(variant), output)
pattern = r"(MainBot|mainbot/1\.0|Main\-Bot)"
self.assertRegex(output, pattern)
if __name__ == "__main__":
import os
os.chdir(os.path.dirname(__file__))
unittest.main(verbosity=2)