Add option to allow all URLs to be crawlable via robots.txt

shlinkio · Apr 21, 2024 · b9b6ec2 · b9b6ec2
1 parent a89b53a
commit b9b6ec2
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 3 deletions.
diff --git a/config/autoload/robots.global.php b/config/autoload/robots.global.php
@@ -0,0 +1,13 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\Core;
+
+return [
+
+    'robots' => [
+        'allow-all' => (bool) Config\EnvVars::ROBOTS_ALLOW_ALL->loadFromEnv(false),
+    ],
+
+];
diff --git a/module/Core/config/dependencies.config.php b/module/Core/config/dependencies.config.php
@@ -189,7 +189,7 @@
             'Logger_Shlink',
             Options\QrCodeOptions::class,
         ],
-        Action\RobotsAction::class => [Crawling\CrawlingHelper::class],
+        Action\RobotsAction::class => [Crawling\CrawlingHelper::class, 'config.robots.allow-all'],
 
         ShortUrl\Resolver\PersistenceShortUrlRelationResolver::class => [
             'em',

diff --git a/module/Core/src/Action/RobotsAction.php b/module/Core/src/Action/RobotsAction.php
@@ -15,9 +15,9 @@
 
 use const PHP_EOL;
 
-class RobotsAction implements RequestHandlerInterface, StatusCodeInterface
+readonly class RobotsAction implements RequestHandlerInterface, StatusCodeInterface
 {
-    public function __construct(private readonly CrawlingHelperInterface $crawlingHelper)
+    public function __construct(private CrawlingHelperInterface $crawlingHelper, private bool $allowAll = false)
     {
     }
 
@@ -37,6 +37,11 @@ private function buildRobots(): iterable
 
         ROBOTS;
 
+        if ($this->allowAll) {
+            yield 'Disallow: /rest';
+            return;
+        }
+
         $shortCodes = $this->crawlingHelper->listCrawlableShortCodes();
         foreach ($shortCodes as $shortCode) {
             yield sprintf('Allow: /%s%s', $shortCode, PHP_EOL);

diff --git a/module/Core/src/Config/EnvVars.php b/module/Core/src/Config/EnvVars.php
@@ -72,6 +72,7 @@ enum EnvVars: string
     case TIMEZONE = 'TIMEZONE';
     case MULTI_SEGMENT_SLUGS_ENABLED = 'MULTI_SEGMENT_SLUGS_ENABLED';
     case MEMORY_LIMIT = 'MEMORY_LIMIT';
+    case ROBOTS_ALLOW_ALL = 'ROBOTS_ALLOW_ALL';
 
     public function loadFromEnv(mixed $default = null): mixed
     {