feat(bots.php, htaccess.liquid): share files with specific bots

fix(*.php.liquid): remove unused html minification dependency
2025-05-30 14:54:21 -07:00 · 2025-05-30 14:38:40 -07:00
20 changed files with 68 additions and 307 deletions
--- a/composer.json
+++ b/composer.json
@ -7,8 +7,7 @@
    "guzzlehttp/guzzle": "^7.9",
    "kaoken/markdown-it-php": "^14.1",
    "sokil/php-isocodes": "^4.2",
-    "sokil/php-isocodes-db-only": "^4.0",
-    "voku/html-min": "^4.5"
+    "sokil/php-isocodes-db-only": "^4.0"
  },
  "scripts": {
    "start": [
--- a/composer.lock
+++ b/composer.lock
@ -4,7 +4,7 @@
        "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
        "This file is @generated automatically"
    ],
-    "content-hash": "9998c1c4f0bcc7d5e58abc25de979750",
+    "content-hash": "6f62ebb63bb51c04310e829e19beeab5",
    "packages": [
        {
            "name": "guzzlehttp/guzzle",
@ -686,71 +686,6 @@
            },
            "time": "2024-02-02T08:24:43+00:00"
        },
-        {
-            "name": "symfony/css-selector",
-            "version": "v7.3.0",
-            "source": {
-                "type": "git",
-                "url": "https://github.com/symfony/css-selector.git",
-                "reference": "601a5ce9aaad7bf10797e3663faefce9e26c24e2"
-            },
-            "dist": {
-                "type": "zip",
-                "url": "https://api.github.com/repos/symfony/css-selector/zipball/601a5ce9aaad7bf10797e3663faefce9e26c24e2",
-                "reference": "601a5ce9aaad7bf10797e3663faefce9e26c24e2",
-                "shasum": ""
-            },
-            "require": {
-                "php": ">=8.2"
-            },
-            "type": "library",
-            "autoload": {
-                "psr-4": {
-                    "Symfony\\Component\\CssSelector\\": ""
-                },
-                "exclude-from-classmap": [
-                    "/Tests/"
-                ]
-            },
-            "notification-url": "https://packagist.org/downloads/",
-            "license": [
-                "MIT"
-            ],
-            "authors": [
-                {
-                    "name": "Fabien Potencier",
-                    "email": "fabien@symfony.com"
-                },
-                {
-                    "name": "Jean-François Simon",
-                    "email": "jeanfrancois.simon@sensiolabs.com"
-                },
-                {
-                    "name": "Symfony Community",
-                    "homepage": "https://symfony.com/contributors"
-                }
-            ],
-            "description": "Converts CSS selectors to XPath expressions",
-            "homepage": "https://symfony.com",
-            "support": {
-                "source": "https://github.com/symfony/css-selector/tree/v7.3.0"
-            },
-            "funding": [
-                {
-                    "url": "https://symfony.com/sponsor",
-                    "type": "custom"
-                },
-                {
-                    "url": "https://github.com/fabpot",
-                    "type": "github"
-                },
-                {
-                    "url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
-                    "type": "tidelift"
-                }
-            ],
-            "time": "2024-09-25T14:21:43+00:00"
-        },
        {
            "name": "symfony/deprecation-contracts",
            "version": "v3.6.0",
@ -817,154 +752,6 @@
                }
            ],
            "time": "2024-09-25T14:21:43+00:00"
-        },
-        {
-            "name": "voku/html-min",
-            "version": "4.5.1",
-            "source": {
-                "type": "git",
-                "url": "https://github.com/voku/HtmlMin.git",
-                "reference": "872dac444652d30791ca5d0c23fbe1e6b9cab5bc"
-            },
-            "dist": {
-                "type": "zip",
-                "url": "https://api.github.com/repos/voku/HtmlMin/zipball/872dac444652d30791ca5d0c23fbe1e6b9cab5bc",
-                "reference": "872dac444652d30791ca5d0c23fbe1e6b9cab5bc",
-                "shasum": ""
-            },
-            "require": {
-                "ext-dom": "*",
-                "php": ">=7.0.0",
-                "voku/simple_html_dom": "~4.8.5"
-            },
-            "require-dev": {
-                "phpunit/phpunit": "~6.0 || ~7.0 || ~9.0"
-            },
-            "type": "library",
-            "autoload": {
-                "psr-4": {
-                    "voku\\": "src/voku/"
-                }
-            },
-            "notification-url": "https://packagist.org/downloads/",
-            "license": [
-                "MIT"
-            ],
-            "authors": [
-                {
-                    "name": "Lars Moelleken",
-                    "homepage": "https://www.moelleken.org/"
-                }
-            ],
-            "description": "HTML Compressor and Minifier",
-            "homepage": "https://github.com/voku/HtmlMin",
-            "keywords": [
-                "compress",
-                "compression",
-                "compressor",
-                "html",
-                "minifier"
-            ],
-            "support": {
-                "issues": "https://github.com/voku/HtmlMin/issues",
-                "source": "https://github.com/voku/HtmlMin/tree/4.5.1"
-            },
-            "funding": [
-                {
-                    "url": "https://www.paypal.me/moelleken",
-                    "type": "custom"
-                },
-                {
-                    "url": "https://github.com/voku",
-                    "type": "github"
-                },
-                {
-                    "url": "https://www.patreon.com/voku",
-                    "type": "patreon"
-                }
-            ],
-            "time": "2024-05-25T08:01:45+00:00"
-        },
-        {
-            "name": "voku/simple_html_dom",
-            "version": "4.8.10",
-            "source": {
-                "type": "git",
-                "url": "https://github.com/voku/simple_html_dom.git",
-                "reference": "716822ed52ed3a1881542be07a786270de390e99"
-            },
-            "dist": {
-                "type": "zip",
-                "url": "https://api.github.com/repos/voku/simple_html_dom/zipball/716822ed52ed3a1881542be07a786270de390e99",
-                "reference": "716822ed52ed3a1881542be07a786270de390e99",
-                "shasum": ""
-            },
-            "require": {
-                "ext-dom": "*",
-                "ext-libxml": "*",
-                "ext-simplexml": "*",
-                "php": ">=7.0.0",
-                "symfony/css-selector": "~3.0 || ~4.0 || ~5.0 || ~6.0 || ~7.0"
-            },
-            "require-dev": {
-                "phpunit/phpunit": "~6.0 || ~7.0 || ~9.0"
-            },
-            "suggest": {
-                "voku/portable-utf8": "If you need e.g. UTF-8 fixed output."
-            },
-            "type": "library",
-            "autoload": {
-                "psr-4": {
-                    "voku\\helper\\": "src/voku/helper/"
-                }
-            },
-            "notification-url": "https://packagist.org/downloads/",
-            "license": [
-                "MIT"
-            ],
-            "authors": [
-                {
-                    "name": "dimabdc",
-                    "email": "support@titor.ru",
-                    "homepage": "https://github.com/dimabdc",
-                    "role": "Developer"
-                },
-                {
-                    "name": "Lars Moelleken",
-                    "homepage": "https://www.moelleken.org/",
-                    "role": "Fork-Maintainer"
-                }
-            ],
-            "description": "Simple HTML DOM package.",
-            "homepage": "https://github.com/voku/simple_html_dom",
-            "keywords": [
-                "HTML Parser",
-                "dom",
-                "php dom"
-            ],
-            "support": {
-                "issues": "https://github.com/voku/simple_html_dom/issues",
-                "source": "https://github.com/voku/simple_html_dom/tree/4.8.10"
-            },
-            "funding": [
-                {
-                    "url": "https://www.paypal.me/moelleken",
-                    "type": "custom"
-                },
-                {
-                    "url": "https://github.com/voku",
-                    "type": "github"
-                },
-                {
-                    "url": "https://www.patreon.com/voku",
-                    "type": "patreon"
-                },
-                {
-                    "url": "https://tidelift.com/funding/github/packagist/voku/simple_html_dom",
-                    "type": "tidelift"
-                }
-            ],
-            "time": "2024-07-03T16:05:14+00:00"
        }
    ],
    "packages-dev": [],
--- a/config/filters/general.js
+++ b/config/filters/general.js
@ -38,4 +38,5 @@ export default {
    return `${string}s${trailing ? `${trailing}` : ''}`;
  },
  jsonEscape: (string) => JSON.stringify(string),
+  regexEscape: (string) => string.replace(/[.*+?^${}()[]\\]/g, '\\$&')
 };
--- a/config/plugins/html-config.js
+++ b/config/plugins/html-config.js
@ -2,7 +2,7 @@ import htmlmin from 'html-minifier-terser';

 export const htmlConfig = (eleventyConfig) => {
  eleventyConfig.addTransform('html-minify', (content, path) => {
-    if (path && (path.endsWith('.html') || path.endsWith('.php'))) {
+    if (path && (path.endsWith('.html') || path.endsWith('.php') && !path.includes("api"))) {
      return htmlmin.minify(content, {
        collapseBooleanAttributes: true,
        collapseWhitespace: true,
--- a/package-lock.json
+++ b/package-lock.json
@ -1,12 +1,12 @@
 {
  "name": "coryd.dev",
-  "version": "8.1.8",
+  "version": "8.2.0",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "coryd.dev",
-      "version": "8.1.8",
+      "version": "8.2.0",
      "license": "MIT",
      "dependencies": {
        "minisearch": "^7.1.2",
--- a/package.json
+++ b/package.json
@ -1,6 +1,6 @@
 {
  "name": "coryd.dev",
-  "version": "8.1.8",
+  "version": "8.2.0",
  "description": "The source for my personal site. Built using 11ty (and other tools).",
  "type": "module",
  "engines": {
--- a/src/includes/fetchers/artist.php.liquid
+++ b/src/includes/fetchers/artist.php.liquid
@ -4,7 +4,6 @@
  require __DIR__ . "/../../server/utils/init.php";

  use App\Classes\ArtistFetcher;
-  use voku\helper\HtmlMin;

  $requestUri = $_SERVER["REQUEST_URI"];
  $url = trim(parse_url($requestUri, PHP_URL_PATH), "/");
@ -21,7 +20,6 @@
  $page = $artist;

  extract(setupPageMetadata($page, $requestUri));
-  ob_start();
  header("Cache-Control: public, max-age=3600");
  header("Expires: " . gmdate("D, d M Y H:i:s", time() + 3600) . " GMT");

--- a/src/includes/fetchers/book.php.liquid
+++ b/src/includes/fetchers/book.php.liquid
@ -4,7 +4,6 @@
  require __DIR__ . "/../../server/utils/init.php";

  use App\Classes\BookFetcher;
-  use voku\helper\HtmlMin;

  $requestUri = $_SERVER["REQUEST_URI"];
  $url = trim(parse_url($requestUri, PHP_URL_PATH), "/");
@ -22,7 +21,6 @@
  $globals = $page["globals"];

  extract(setupPageMetadata($page, $requestUri));
-  ob_start();
  header("Cache-Control: public, max-age=3600");
  header("Expires: " . gmdate("D, d M Y H:i:s", time() + 3600) . " GMT");

--- a/src/includes/fetchers/genre.php.liquid
+++ b/src/includes/fetchers/genre.php.liquid
@ -4,7 +4,6 @@
  require __DIR__ . "/../../server/utils/init.php";

  use App\Classes\GenreFetcher;
-  use voku\helper\HtmlMin;

  $requestUri = $_SERVER["REQUEST_URI"];
  $url = trim(parse_url($requestUri, PHP_URL_PATH), "/");
@ -21,8 +20,6 @@
  $globals = $page["globals"];

  extract(setupPageMetadata($page, $requestUri));
-
-  ob_start();
  header("Cache-Control: public, max-age=3600");
  header("Expires: " . gmdate("D, d M Y H:i:s", time() + 3600) . " GMT");

--- a/src/includes/fetchers/movie.php.liquid
+++ b/src/includes/fetchers/movie.php.liquid
@ -4,7 +4,6 @@
  require __DIR__ . "/../../server/utils/init.php";

  use App\Classes\MovieFetcher;
-  use voku\helper\HtmlMin;

  $requestUri = $_SERVER["REQUEST_URI"];
  $url = trim(parse_url($requestUri, PHP_URL_PATH), "/");
@ -22,7 +21,6 @@
  $globals = $page["globals"];

  extract(setupPageMetadata($page, $requestUri));
-  ob_start();
  header("Cache-Control: public, max-age=3600");
  header("Expires: " . gmdate("D, d M Y H:i:s", time() + 3600) . " GMT");

--- a/src/includes/fetchers/show.php.liquid
+++ b/src/includes/fetchers/show.php.liquid
@ -4,7 +4,6 @@
  require __DIR__ . "/../../server/utils/init.php";

  use App\Classes\ShowFetcher;
-  use voku\helper\HtmlMin;

  $requestUri = $_SERVER["REQUEST_URI"];
  $url = trim(parse_url($requestUri, PHP_URL_PATH), "/");
@ -22,7 +21,6 @@
  $globals = $page["globals"];

  extract(setupPageMetadata($page, $requestUri));
-  ob_start();
  header("Cache-Control: public, max-age=3600");
  header("Expires: " . gmdate("D, d M Y H:i:s", time() + 3600) . " GMT");

--- a/src/includes/fetchers/tags.php.liquid
+++ b/src/includes/fetchers/tags.php.liquid
@ -5,7 +5,6 @@

  use App\Classes\TagFetcher;
  use App\Classes\GlobalsFetcher;
-  use voku\helper\HtmlMin;

  $requestUri = $_SERVER["REQUEST_URI"];
  $url = trim(parse_url($requestUri, PHP_URL_PATH), "/");
@ -63,7 +62,6 @@
  ];

  extract(setupPageMetadata($page, $requestUri));
-  ob_start();
  header("Cache-Control: public, max-age=3600");
  header("Expires: " . gmdate("D, d M Y H:i:s", time() + 3600) . " GMT");

--- a/src/meta/bots.php.liquid
+++ b/src/meta/bots.php.liquid
@ -0,0 +1,48 @@
+---
+permalink: "/api/bots.php"
+layout: null
+eleventyExcludeFromCollections: true
+excludeFromSitemap: true
+---
+<?php
+
+  require __DIR__ . '/../server/utils/init.php';
+
+  $blockedAgents = [
+    {%- assign allAgents = "" -%}
+    {%- for robot in robots -%}
+      {%- for userAgent in robot.user_agents -%}
+        {%- if userAgent != "*" and userAgent != "NaN" -%}
+          '{{ userAgent | escape }}'{% unless forloop.last %},{% endunless %}
+        {%- endif -%}
+      {%- endfor -%}
+    {%- endfor -%}
+  ];
+
+  $ua = $_SERVER['HTTP_USER_AGENT'] ?? '';
+  $shouldShareFile = false;
+
+  foreach ($blockedAgents as $bot) {
+    if (stripos($ua, $bot) !== false) {
+      $shouldShareFile = true;
+      break;
+    }
+  }
+
+  if ($shouldShareFile) {
+    header('Content-Type: application/octet-stream');
+    header('Content-Disposition: attachment; filename="10GB_stream.dat"');
+
+    $chunkSize = 1024 * 1024; // 1MB
+    $totalChunks = 10240;
+
+    for ($i = 0; $i < $totalChunks; $i++) {
+      echo random_bytes($chunkSize);
+      flush();
+      usleep(50000);
+    }
+
+    exit;
+  }
+
+  redirectTo404();
--- a/src/meta/htaccess.liquid
+++ b/src/meta/htaccess.liquid
@ -83,23 +83,22 @@ Redirect {{ redirect.status_code | default: "301" }} {{ redirect.source_url }} {
 </FilesMatch>
 {% endfor %}

-{%- assign userAgents = "" -%}
-{% for robot in robots -%}
-  {%- for userAgent in robot.user_agents -%}
-    {%- if userAgent != "*" and userAgent != "NaN" -%}
-      {%- assign userAgents = userAgents | append: userAgent %}
-      {%- unless forloop.last -%}
-        {%- assign userAgents = userAgents | append: "|" -%}
-      {%- endunless -%}
+{%- assign escapedAgents = "" -%}
+{%- for robot in robots -%}
+  {%- for agent in robot.user_agents -%}
+    {%- if agent != "*" and agent != "NaN" -%}
+      {%- assign escaped = agent | regexEscape -%}
+      {%- assign escapedAgents = escapedAgents | append: escaped | append: "|" -%}
    {%- endif -%}
  {%- endfor -%}
-{%- endfor %}
-{% if userAgents != "" or referers != "" -%}
-RewriteCond %{HTTP_USER_AGENT} "{{ userAgents }}" [NC]
+{%- endfor -%}
+{%- assign userAgentsRegex = escapedAgents | slice: 0, escapedAgents.size | rstrip: "|" -%}
+{% if userAgentsRegex != "" %}
+RewriteEngine On
+RewriteCond %{HTTP_USER_AGENT} "{{ userAgentsRegex }}" [NC]
 RewriteCond %{REQUEST_URI} !^/robots\.txt$ [NC]
-RewriteRule .* /403/index.html [L,R=403]
-{%- endif %}
-
+RewriteRule .* /api/bots.php [L]
+{% endif %}
 <IfModule mod_deflate.c>
  AddOutputFilterByType DEFLATE text/html text/plain text/xml text/css text/javascript application/javascript application/json
 </IfModule>
--- a/src/pages/dynamic/artist.php.liquid
+++ b/src/pages/dynamic/artist.php.liquid
@ -111,13 +111,3 @@ schema: artist
    <?php renderMediaGrid($artist["albums"]); ?>
  <?php endif; ?>
 </article>
-<?php
-  $html = ob_get_clean();
-  $htmlMin = new HtmlMin();
-  $htmlMin->doOptimizeAttributes(true);
-  $htmlMin->doRemoveComments(true);
-  $htmlMin->doSumUpWhitespace(true);
-  $htmlMin->doRemoveWhitespaceAroundTags(true);
-  $htmlMin->doOptimizeViaHtmlDomParser(false);
-  echo $htmlMin->minify($html);
-?>
--- a/src/pages/dynamic/book.php.liquid
+++ b/src/pages/dynamic/book.php.liquid
@ -67,13 +67,3 @@ schema: book
    <?= parseMarkdown($book["description"]) ?>
  <?php endif; ?>
 </article>
-<?php
-  $html = ob_get_clean();
-  $htmlMin = new HtmlMin();
-  $htmlMin->doOptimizeAttributes(true);
-  $htmlMin->doRemoveComments(true);
-  $htmlMin->doSumUpWhitespace(true);
-  $htmlMin->doRemoveWhitespaceAroundTags(true);
-  $htmlMin->doOptimizeViaHtmlDomParser(false);
-  echo $htmlMin->minify($html);
-?>
--- a/src/pages/dynamic/genre.php.liquid
+++ b/src/pages/dynamic/genre.php.liquid
@ -45,13 +45,3 @@ schema: genre
    <button data-toggle-button>Show more</button>
  <?php endif; ?>
 </article>
-<?php
-  $html = ob_get_clean();
-  $htmlMin = new HtmlMin();
-  $htmlMin->doOptimizeAttributes(true);
-  $htmlMin->doRemoveComments(true);
-  $htmlMin->doSumUpWhitespace(true);
-  $htmlMin->doRemoveWhitespaceAroundTags(true);
-  $htmlMin->doOptimizeViaHtmlDomParser(false);
-  echo $htmlMin->minify($html);
-?>
--- a/src/pages/dynamic/movie.php.liquid
+++ b/src/pages/dynamic/movie.php.liquid
@ -63,13 +63,3 @@ schema: movie
    <?= parseMarkdown($movie["description"]) ?>
  <?php endif; ?>
 </article>
-<?php
-  $html = ob_get_clean();
-  $htmlMin = new HtmlMin();
-  $htmlMin->doOptimizeAttributes(true);
-  $htmlMin->doRemoveComments(true);
-  $htmlMin->doSumUpWhitespace(true);
-  $htmlMin->doRemoveWhitespaceAroundTags(true);
-  $htmlMin->doOptimizeViaHtmlDomParser(false);
-  echo $htmlMin->minify($html);
-?>
--- a/src/pages/dynamic/show.php.liquid
+++ b/src/pages/dynamic/show.php.liquid
@ -60,13 +60,3 @@ schema: show
    <?= parseMarkdown($show["description"]) ?>
  <?php endif; ?>
 </article>
-<?php
-  $html = ob_get_clean();
-  $htmlMin = new HtmlMin();
-  $htmlMin->doOptimizeAttributes(true);
-  $htmlMin->doRemoveComments(true);
-  $htmlMin->doSumUpWhitespace(true);
-  $htmlMin->doRemoveWhitespaceAroundTags(true);
-  $htmlMin->doOptimizeViaHtmlDomParser(false);
-  echo $htmlMin->minify($html);
-?>
--- a/src/pages/dynamic/tags.php.liquid
+++ b/src/pages/dynamic/tags.php.liquid
@ -38,13 +38,3 @@ schema: tags
 </article>
 <?php endforeach; ?>
 <?php renderPaginator($pagination, $totalPages); ?>
-<?php
-  $html = ob_get_clean();
-  $htmlMin = new HtmlMin();
-  $htmlMin->doOptimizeAttributes(true);
-  $htmlMin->doRemoveComments(true);
-  $htmlMin->doSumUpWhitespace(true);
-  $htmlMin->doRemoveWhitespaceAroundTags(true);
-  $htmlMin->doOptimizeViaHtmlDomParser(false);
-  echo $htmlMin->minify($html);
-?>
Author	SHA1	Message	Date
Cory Dransfeldt	92e123cd57	feat(bots.php, htaccess.liquid): share files with specific bots	2025-05-30 14:54:21 -07:00
Cory Dransfeldt	79464d6769	fix(*.php.liquid): remove unused html minification dependency	2025-05-30 14:38:40 -07:00