feat(bots.php, htaccess.liquid): share files with specific bots

composer.json

@@ -7,8 +7,7 @@
     "guzzlehttp/guzzle": "^7.9",
     "kaoken/markdown-it-php": "^14.1",
     "sokil/php-isocodes": "^4.2",
-    "sokil/php-isocodes-db-only": "^4.0",
+    "sokil/php-isocodes-db-only": "^4.0"
-    "voku/html-min": "^4.5"
   },
   "scripts": {
     "start": [

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "9998c1c4f0bcc7d5e58abc25de979750",
+    "content-hash": "6f62ebb63bb51c04310e829e19beeab5",
     "packages": [
         {
             "name": "guzzlehttp/guzzle",
@@ -686,71 +686,6 @@
             },
             "time": "2024-02-02T08:24:43+00:00"
         },
-        {
-            "name": "symfony/css-selector",
-            "version": "v7.3.0",
-            "source": {
-                "type": "git",
-                "url": "https://github.com/symfony/css-selector.git",
-                "reference": "601a5ce9aaad7bf10797e3663faefce9e26c24e2"
-            },
-            "dist": {
-                "type": "zip",
-                "url": "https://api.github.com/repos/symfony/css-selector/zipball/601a5ce9aaad7bf10797e3663faefce9e26c24e2",
-                "reference": "601a5ce9aaad7bf10797e3663faefce9e26c24e2",
-                "shasum": ""
-            },
-            "require": {
-                "php": ">=8.2"
-            },
-            "type": "library",
-            "autoload": {
-                "psr-4": {
-                    "Symfony\\Component\\CssSelector\\": ""
-                },
-                "exclude-from-classmap": [
-                    "/Tests/"
-                ]
-            },
-            "notification-url": "https://packagist.org/downloads/",
-            "license": [
-                "MIT"
-            ],
-            "authors": [
-                {
-                    "name": "Fabien Potencier",
-                    "email": "fabien@symfony.com"
-                },
-                {
-                    "name": "Jean-François Simon",
-                    "email": "jeanfrancois.simon@sensiolabs.com"
-                },
-                {
-                    "name": "Symfony Community",
-                    "homepage": "https://symfony.com/contributors"
-                }
-            ],
-            "description": "Converts CSS selectors to XPath expressions",
-            "homepage": "https://symfony.com",
-            "support": {
-                "source": "https://github.com/symfony/css-selector/tree/v7.3.0"
-            },
-            "funding": [
-                {
-                    "url": "https://symfony.com/sponsor",
-                    "type": "custom"
-                },
-                {
-                    "url": "https://github.com/fabpot",
-                    "type": "github"
-                },
-                {
-                    "url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
-                    "type": "tidelift"
-                }
-            ],
-            "time": "2024-09-25T14:21:43+00:00"
-        },
         {
             "name": "symfony/deprecation-contracts",
             "version": "v3.6.0",
@@ -817,154 +752,6 @@
                 }
             ],
             "time": "2024-09-25T14:21:43+00:00"
-        },
-        {
-            "name": "voku/html-min",
-            "version": "4.5.1",
-            "source": {
-                "type": "git",
-                "url": "https://github.com/voku/HtmlMin.git",
-                "reference": "872dac444652d30791ca5d0c23fbe1e6b9cab5bc"
-            },
-            "dist": {
-                "type": "zip",
-                "url": "https://api.github.com/repos/voku/HtmlMin/zipball/872dac444652d30791ca5d0c23fbe1e6b9cab5bc",
-                "reference": "872dac444652d30791ca5d0c23fbe1e6b9cab5bc",
-                "shasum": ""
-            },
-            "require": {
-                "ext-dom": "*",
-                "php": ">=7.0.0",
-                "voku/simple_html_dom": "~4.8.5"
-            },
-            "require-dev": {
-                "phpunit/phpunit": "~6.0 || ~7.0 || ~9.0"
-            },
-            "type": "library",
-            "autoload": {
-                "psr-4": {
-                    "voku\\": "src/voku/"
-                }
-            },
-            "notification-url": "https://packagist.org/downloads/",
-            "license": [
-                "MIT"
-            ],
-            "authors": [
-                {
-                    "name": "Lars Moelleken",
-                    "homepage": "https://www.moelleken.org/"
-                }
-            ],
-            "description": "HTML Compressor and Minifier",
-            "homepage": "https://github.com/voku/HtmlMin",
-            "keywords": [
-                "compress",
-                "compression",
-                "compressor",
-                "html",
-                "minifier"
-            ],
-            "support": {
-                "issues": "https://github.com/voku/HtmlMin/issues",
-                "source": "https://github.com/voku/HtmlMin/tree/4.5.1"
-            },
-            "funding": [
-                {
-                    "url": "https://www.paypal.me/moelleken",
-                    "type": "custom"
-                },
-                {
-                    "url": "https://github.com/voku",
-                    "type": "github"
-                },
-                {
-                    "url": "https://www.patreon.com/voku",
-                    "type": "patreon"
-                }
-            ],
-            "time": "2024-05-25T08:01:45+00:00"
-        },
-        {
-            "name": "voku/simple_html_dom",
-            "version": "4.8.10",
-            "source": {
-                "type": "git",
-                "url": "https://github.com/voku/simple_html_dom.git",
-                "reference": "716822ed52ed3a1881542be07a786270de390e99"
-            },
-            "dist": {
-                "type": "zip",
-                "url": "https://api.github.com/repos/voku/simple_html_dom/zipball/716822ed52ed3a1881542be07a786270de390e99",
-                "reference": "716822ed52ed3a1881542be07a786270de390e99",
-                "shasum": ""
-            },
-            "require": {
-                "ext-dom": "*",
-                "ext-libxml": "*",
-                "ext-simplexml": "*",
-                "php": ">=7.0.0",
-                "symfony/css-selector": "~3.0 || ~4.0 || ~5.0 || ~6.0 || ~7.0"
-            },
-            "require-dev": {
-                "phpunit/phpunit": "~6.0 || ~7.0 || ~9.0"
-            },
-            "suggest": {
-                "voku/portable-utf8": "If you need e.g. UTF-8 fixed output."
-            },
-            "type": "library",
-            "autoload": {
-                "psr-4": {
-                    "voku\\helper\\": "src/voku/helper/"
-                }
-            },
-            "notification-url": "https://packagist.org/downloads/",
-            "license": [
-                "MIT"
-            ],
-            "authors": [
-                {
-                    "name": "dimabdc",
-                    "email": "support@titor.ru",
-                    "homepage": "https://github.com/dimabdc",
-                    "role": "Developer"
-                },
-                {
-                    "name": "Lars Moelleken",
-                    "homepage": "https://www.moelleken.org/",
-                    "role": "Fork-Maintainer"
-                }
-            ],
-            "description": "Simple HTML DOM package.",
-            "homepage": "https://github.com/voku/simple_html_dom",
-            "keywords": [
-                "HTML Parser",
-                "dom",
-                "php dom"
-            ],
-            "support": {
-                "issues": "https://github.com/voku/simple_html_dom/issues",
-                "source": "https://github.com/voku/simple_html_dom/tree/4.8.10"
-            },
-            "funding": [
-                {
-                    "url": "https://www.paypal.me/moelleken",
-                    "type": "custom"
-                },
-                {
-                    "url": "https://github.com/voku",
-                    "type": "github"
-                },
-                {
-                    "url": "https://www.patreon.com/voku",
-                    "type": "patreon"
-                },
-                {
-                    "url": "https://tidelift.com/funding/github/packagist/voku/simple_html_dom",
-                    "type": "tidelift"
-                }
-            ],
-            "time": "2024-07-03T16:05:14+00:00"
         }
     ],
     "packages-dev": [],

config/filters/general.js

@@ -38,4 +38,5 @@ export default {
     return `${string}s${trailing ? `${trailing}` : ''}`;
   },
   jsonEscape: (string) => JSON.stringify(string),
+  regexEscape: (string) => string.replace(/[.*+?^${}()[]\\]/g, '\\$&')
 };

config/plugins/html-config.js

@@ -2,7 +2,7 @@ import htmlmin from 'html-minifier-terser';
 
 export const htmlConfig = (eleventyConfig) => {
   eleventyConfig.addTransform('html-minify', (content, path) => {
-    if (path && (path.endsWith('.html') || path.endsWith('.php'))) {
+    if (path && (path.endsWith('.html') || path.endsWith('.php') && !path.includes("api"))) {
       return htmlmin.minify(content, {
         collapseBooleanAttributes: true,
         collapseWhitespace: true,

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "coryd.dev",
-  "version": "8.1.8",
+  "version": "8.2.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "coryd.dev",
-      "version": "8.1.8",
+      "version": "8.2.0",
       "license": "MIT",
       "dependencies": {
         "minisearch": "^7.1.2",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "coryd.dev",
-  "version": "8.1.8",
+  "version": "8.2.0",
   "description": "The source for my personal site. Built using 11ty (and other tools).",
   "type": "module",
   "engines": {

src/includes/fetchers/artist.php.liquid

@@ -4,7 +4,6 @@
   require __DIR__ . "/../../server/utils/init.php";
 
   use App\Classes\ArtistFetcher;
-  use voku\helper\HtmlMin;
 
   $requestUri = $_SERVER["REQUEST_URI"];
   $url = trim(parse_url($requestUri, PHP_URL_PATH), "/");
@@ -21,7 +20,6 @@
   $page = $artist;
 
   extract(setupPageMetadata($page, $requestUri));
-  ob_start();
   header("Cache-Control: public, max-age=3600");
   header("Expires: " . gmdate("D, d M Y H:i:s", time() + 3600) . " GMT");
 

src/includes/fetchers/book.php.liquid

@@ -4,7 +4,6 @@
   require __DIR__ . "/../../server/utils/init.php";
 
   use App\Classes\BookFetcher;
-  use voku\helper\HtmlMin;
 
   $requestUri = $_SERVER["REQUEST_URI"];
   $url = trim(parse_url($requestUri, PHP_URL_PATH), "/");
@@ -22,7 +21,6 @@
   $globals = $page["globals"];
 
   extract(setupPageMetadata($page, $requestUri));
-  ob_start();
   header("Cache-Control: public, max-age=3600");
   header("Expires: " . gmdate("D, d M Y H:i:s", time() + 3600) . " GMT");
 

src/includes/fetchers/genre.php.liquid

@@ -4,7 +4,6 @@
   require __DIR__ . "/../../server/utils/init.php";
 
   use App\Classes\GenreFetcher;
-  use voku\helper\HtmlMin;
 
   $requestUri = $_SERVER["REQUEST_URI"];
   $url = trim(parse_url($requestUri, PHP_URL_PATH), "/");
@@ -21,8 +20,6 @@
   $globals = $page["globals"];
 
   extract(setupPageMetadata($page, $requestUri));
-
-  ob_start();
   header("Cache-Control: public, max-age=3600");
   header("Expires: " . gmdate("D, d M Y H:i:s", time() + 3600) . " GMT");
 

src/includes/fetchers/movie.php.liquid

@@ -4,7 +4,6 @@
   require __DIR__ . "/../../server/utils/init.php";
 
   use App\Classes\MovieFetcher;
-  use voku\helper\HtmlMin;
 
   $requestUri = $_SERVER["REQUEST_URI"];
   $url = trim(parse_url($requestUri, PHP_URL_PATH), "/");
@@ -22,7 +21,6 @@
   $globals = $page["globals"];
 
   extract(setupPageMetadata($page, $requestUri));
-  ob_start();
   header("Cache-Control: public, max-age=3600");
   header("Expires: " . gmdate("D, d M Y H:i:s", time() + 3600) . " GMT");
 

src/includes/fetchers/show.php.liquid

@@ -4,7 +4,6 @@
   require __DIR__ . "/../../server/utils/init.php";
 
   use App\Classes\ShowFetcher;
-  use voku\helper\HtmlMin;
 
   $requestUri = $_SERVER["REQUEST_URI"];
   $url = trim(parse_url($requestUri, PHP_URL_PATH), "/");
@@ -22,7 +21,6 @@
   $globals = $page["globals"];
 
   extract(setupPageMetadata($page, $requestUri));
-  ob_start();
   header("Cache-Control: public, max-age=3600");
   header("Expires: " . gmdate("D, d M Y H:i:s", time() + 3600) . " GMT");
 

src/includes/fetchers/tags.php.liquid

@@ -5,7 +5,6 @@
 
   use App\Classes\TagFetcher;
   use App\Classes\GlobalsFetcher;
-  use voku\helper\HtmlMin;
 
   $requestUri = $_SERVER["REQUEST_URI"];
   $url = trim(parse_url($requestUri, PHP_URL_PATH), "/");
@@ -63,7 +62,6 @@
   ];
 
   extract(setupPageMetadata($page, $requestUri));
-  ob_start();
   header("Cache-Control: public, max-age=3600");
   header("Expires: " . gmdate("D, d M Y H:i:s", time() + 3600) . " GMT");
 

src/meta/bots.php.liquid

@@ -0,0 +1,48 @@
+---
+permalink: "/api/bots.php"
+layout: null
+eleventyExcludeFromCollections: true
+excludeFromSitemap: true
+---
+<?php
+
+  require __DIR__ . '/../server/utils/init.php';
+
+  $blockedAgents = [
+    {%- assign allAgents = "" -%}
+    {%- for robot in robots -%}
+      {%- for userAgent in robot.user_agents -%}
+        {%- if userAgent != "*" and userAgent != "NaN" -%}
+          '{{ userAgent | escape }}'{% unless forloop.last %},{% endunless %}
+        {%- endif -%}
+      {%- endfor -%}
+    {%- endfor -%}
+  ];
+
+  $ua = $_SERVER['HTTP_USER_AGENT'] ?? '';
+  $shouldShareFile = false;
+
+  foreach ($blockedAgents as $bot) {
+    if (stripos($ua, $bot) !== false) {
+      $shouldShareFile = true;
+      break;
+    }
+  }
+
+  if ($shouldShareFile) {
+    header('Content-Type: application/octet-stream');
+    header('Content-Disposition: attachment; filename="10GB_stream.dat"');
+
+    $chunkSize = 1024 * 1024; // 1MB
+    $totalChunks = 10240;
+
+    for ($i = 0; $i < $totalChunks; $i++) {
+      echo random_bytes($chunkSize);
+      flush();
+      usleep(50000);
+    }
+
+    exit;
+  }
+
+  redirectTo404();

src/meta/htaccess.liquid

@@ -83,23 +83,22 @@ Redirect {{ redirect.status_code | default: "301" }} {{ redirect.source_url }} {
 </FilesMatch>
 {% endfor %}
 
-{%- assign userAgents = "" -%}
+{%- assign escapedAgents = "" -%}
-{% for robot in robots -%}
+{%- for robot in robots -%}
-  {%- for userAgent in robot.user_agents -%}
+  {%- for agent in robot.user_agents -%}
-    {%- if userAgent != "*" and userAgent != "NaN" -%}
+    {%- if agent != "*" and agent != "NaN" -%}
-      {%- assign userAgents = userAgents | append: userAgent %}
+      {%- assign escaped = agent | regexEscape -%}
-      {%- unless forloop.last -%}
+      {%- assign escapedAgents = escapedAgents | append: escaped | append: "|" -%}
-        {%- assign userAgents = userAgents | append: "|" -%}
-      {%- endunless -%}
     {%- endif -%}
   {%- endfor -%}
-{%- endfor %}
+{%- endfor -%}
-{% if userAgents != "" or referers != "" -%}
+{%- assign userAgentsRegex = escapedAgents | slice: 0, escapedAgents.size | rstrip: "|" -%}
-RewriteCond %{HTTP_USER_AGENT} "{{ userAgents }}" [NC]
+{% if userAgentsRegex != "" %}
+RewriteEngine On
+RewriteCond %{HTTP_USER_AGENT} "{{ userAgentsRegex }}" [NC]
 RewriteCond %{REQUEST_URI} !^/robots\.txt$ [NC]
-RewriteRule .* /403/index.html [L,R=403]
+RewriteRule .* /api/bots.php [L]
-{%- endif %}
+{% endif %}
-
 <IfModule mod_deflate.c>
   AddOutputFilterByType DEFLATE text/html text/plain text/xml text/css text/javascript application/javascript application/json
 </IfModule>

src/pages/dynamic/artist.php.liquid

@@ -111,13 +111,3 @@ schema: artist
     <?php renderMediaGrid($artist["albums"]); ?>
   <?php endif; ?>
 </article>
-<?php
-  $html = ob_get_clean();
-  $htmlMin = new HtmlMin();
-  $htmlMin->doOptimizeAttributes(true);
-  $htmlMin->doRemoveComments(true);
-  $htmlMin->doSumUpWhitespace(true);
-  $htmlMin->doRemoveWhitespaceAroundTags(true);
-  $htmlMin->doOptimizeViaHtmlDomParser(false);
-  echo $htmlMin->minify($html);
-?>

src/pages/dynamic/book.php.liquid

@@ -67,13 +67,3 @@ schema: book
     <?= parseMarkdown($book["description"]) ?>
   <?php endif; ?>
 </article>
-<?php
-  $html = ob_get_clean();
-  $htmlMin = new HtmlMin();
-  $htmlMin->doOptimizeAttributes(true);
-  $htmlMin->doRemoveComments(true);
-  $htmlMin->doSumUpWhitespace(true);
-  $htmlMin->doRemoveWhitespaceAroundTags(true);
-  $htmlMin->doOptimizeViaHtmlDomParser(false);
-  echo $htmlMin->minify($html);
-?>

src/pages/dynamic/genre.php.liquid

@@ -45,13 +45,3 @@ schema: genre
     <button data-toggle-button>Show more</button>
   <?php endif; ?>
 </article>
-<?php
-  $html = ob_get_clean();
-  $htmlMin = new HtmlMin();
-  $htmlMin->doOptimizeAttributes(true);
-  $htmlMin->doRemoveComments(true);
-  $htmlMin->doSumUpWhitespace(true);
-  $htmlMin->doRemoveWhitespaceAroundTags(true);
-  $htmlMin->doOptimizeViaHtmlDomParser(false);
-  echo $htmlMin->minify($html);
-?>

src/pages/dynamic/movie.php.liquid

@@ -63,13 +63,3 @@ schema: movie
     <?= parseMarkdown($movie["description"]) ?>
   <?php endif; ?>
 </article>
-<?php
-  $html = ob_get_clean();
-  $htmlMin = new HtmlMin();
-  $htmlMin->doOptimizeAttributes(true);
-  $htmlMin->doRemoveComments(true);
-  $htmlMin->doSumUpWhitespace(true);
-  $htmlMin->doRemoveWhitespaceAroundTags(true);
-  $htmlMin->doOptimizeViaHtmlDomParser(false);
-  echo $htmlMin->minify($html);
-?>

src/pages/dynamic/show.php.liquid

@@ -60,13 +60,3 @@ schema: show
     <?= parseMarkdown($show["description"]) ?>
   <?php endif; ?>
 </article>
-<?php
-  $html = ob_get_clean();
-  $htmlMin = new HtmlMin();
-  $htmlMin->doOptimizeAttributes(true);
-  $htmlMin->doRemoveComments(true);
-  $htmlMin->doSumUpWhitespace(true);
-  $htmlMin->doRemoveWhitespaceAroundTags(true);
-  $htmlMin->doOptimizeViaHtmlDomParser(false);
-  echo $htmlMin->minify($html);
-?>

src/pages/dynamic/tags.php.liquid

@@ -38,13 +38,3 @@ schema: tags
 </article>
 <?php endforeach; ?>
 <?php renderPaginator($pagination, $totalPages); ?>
-<?php
-  $html = ob_get_clean();
-  $htmlMin = new HtmlMin();
-  $htmlMin->doOptimizeAttributes(true);
-  $htmlMin->doRemoveComments(true);
-  $htmlMin->doSumUpWhitespace(true);
-  $htmlMin->doRemoveWhitespaceAroundTags(true);
-  $htmlMin->doOptimizeViaHtmlDomParser(false);
-  echo $htmlMin->minify($html);
-?>