feat(bots.php, htaccess.liquid): share files with specific bots

config/filters/general.js

@@ -38,4 +38,5 @@ export default {
     return `${string}s${trailing ? `${trailing}` : ''}`;
   },
   jsonEscape: (string) => JSON.stringify(string),
+  regexEscape: (string) => string.replace(/[.*+?^${}()[]\\]/g, '\\$&')
 };

config/plugins/html-config.js

@@ -2,7 +2,7 @@ import htmlmin from 'html-minifier-terser';
 
 export const htmlConfig = (eleventyConfig) => {
   eleventyConfig.addTransform('html-minify', (content, path) => {
-    if (path && (path.endsWith('.html') || path.endsWith('.php'))) {
+    if (path && (path.endsWith('.html') || path.endsWith('.php') && !path.includes("api"))) {
       return htmlmin.minify(content, {
         collapseBooleanAttributes: true,
         collapseWhitespace: true,

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "coryd.dev",
-  "version": "8.1.9",
+  "version": "8.2.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "coryd.dev",
-      "version": "8.1.9",
+      "version": "8.2.0",
       "license": "MIT",
       "dependencies": {
         "minisearch": "^7.1.2",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "coryd.dev",
-  "version": "8.1.9",
+  "version": "8.2.0",
   "description": "The source for my personal site. Built using 11ty (and other tools).",
   "type": "module",
   "engines": {

src/meta/bots.php.liquid

@@ -0,0 +1,48 @@
+---
+permalink: "/api/bots.php"
+layout: null
+eleventyExcludeFromCollections: true
+excludeFromSitemap: true
+---
+<?php
+
+  require __DIR__ . '/../server/utils/init.php';
+
+  $blockedAgents = [
+    {%- assign allAgents = "" -%}
+    {%- for robot in robots -%}
+      {%- for userAgent in robot.user_agents -%}
+        {%- if userAgent != "*" and userAgent != "NaN" -%}
+          '{{ userAgent | escape }}'{% unless forloop.last %},{% endunless %}
+        {%- endif -%}
+      {%- endfor -%}
+    {%- endfor -%}
+  ];
+
+  $ua = $_SERVER['HTTP_USER_AGENT'] ?? '';
+  $shouldShareFile = false;
+
+  foreach ($blockedAgents as $bot) {
+    if (stripos($ua, $bot) !== false) {
+      $shouldShareFile = true;
+      break;
+    }
+  }
+
+  if ($shouldShareFile) {
+    header('Content-Type: application/octet-stream');
+    header('Content-Disposition: attachment; filename="10GB_stream.dat"');
+
+    $chunkSize = 1024 * 1024; // 1MB
+    $totalChunks = 10240;
+
+    for ($i = 0; $i < $totalChunks; $i++) {
+      echo random_bytes($chunkSize);
+      flush();
+      usleep(50000);
+    }
+
+    exit;
+  }
+
+  redirectTo404();

src/meta/htaccess.liquid

@@ -83,23 +83,23 @@ Redirect {{ redirect.status_code | default: "301" }} {{ redirect.source_url }} {
 </FilesMatch>
 {% endfor %}
 
-{%- assign userAgents = "" -%}
+{%- assign escapedAgents = "" -%}
-{% for robot in robots -%}
+{%- for robot in robots -%}
-  {%- for userAgent in robot.user_agents -%}
+  {%- for agent in robot.user_agents -%}
-    {%- if userAgent != "*" and userAgent != "NaN" -%}
+    {%- if agent != "*" and agent != "NaN" -%}
-      {%- assign userAgents = userAgents | append: userAgent %}
+      {%- assign escaped = agent | regexEscape -%}
-      {%- unless forloop.last -%}
+      {%- assign escapedAgents = escapedAgents | append: escaped | append: "|" -%}
-        {%- assign userAgents = userAgents | append: "|" -%}
-      {%- endunless -%}
     {%- endif -%}
   {%- endfor -%}
-{%- endfor %}
+{%- endfor -%}
-{% if userAgents != "" or referers != "" -%}
+{%- assign userAgentsRegex = escapedAgents | slice: 0, escapedAgents.size | rstrip: "|" -%}
-RewriteCond %{HTTP_USER_AGENT} "{{ userAgents }}" [NC]
+{% if userAgentsRegex != "" %}
+RewriteEngine On
+RewriteCond %{HTTP_USER_AGENT} "{{ userAgentsRegex }}" [NC]
 RewriteCond %{REQUEST_URI} !^/robots\.txt$ [NC]
-RewriteRule .* /403/index.html [L,R=403]
+RewriteCond %{REQUEST_URI} !^/api/bots\.php$ [NC]
-{%- endif %}
+RewriteRule .* /api/bots.php [L]
-
+{% endif %}
 <IfModule mod_deflate.c>
   AddOutputFilterByType DEFLATE text/html text/plain text/xml text/css text/javascript application/javascript application/json
 </IfModule>