added robots.txt for API endpoints (#3165)

* added robots.txt for API endpoints * added path for robots.txt for API endpoints * Merge imports --------- Co-authored-by: Dhruv Bhanushali <[email protected]>
WordPress · Oct 10, 2023 · 1487f91 · 1487f91
1 parent b28665a
commit 1487f91
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 1 deletion.
diff --git a/api/api/templates/robots.txt b/api/api/templates/robots.txt
@@ -0,0 +1,8 @@
+# Block API endpoints
+User-agent: *
+Disallow: /v1/images/
+Disallow: /v1/audio/
+Disallow: /v1/auth/
+
+User-agent: GPTBot
+Disallow: /
diff --git a/api/conf/urls/__init__.py b/api/conf/urls/__init__.py
@@ -8,7 +8,7 @@
 from django.conf.urls.static import static
 from django.contrib import admin
 from django.urls import include, path
-from django.views.generic import RedirectView
+from django.views.generic import RedirectView, TemplateView
 from rest_framework.routers import SimpleRouter
 
 from api.views.audio_views import AudioViewSet
@@ -35,6 +35,13 @@
     path("admin/", admin.site.urls),
     path("healthcheck/", HealthCheck.as_view(), name="health"),
     path("v1/", include(versioned_paths)),
+    path(
+        "robots.txt/",
+        TemplateView.as_view(
+            template_name="robots.txt",
+            content_type="text/plain",
+        ),
+    ),
 ]
 
 if settings.ENVIRONMENT == "local":