From 1487f918ca191edf2f4e02ef038076f1f2dcedca Mon Sep 17 00:00:00 2001
From: Madhur Prajapati <96643023+MysteryManav@users.noreply.github.com>
Date: Tue, 10 Oct 2023 16:53:11 +0530
Subject: [PATCH] added robots.txt for API endpoints (#3165)

* added robots.txt for API endpoints

* added path for robots.txt for API endpoints

* Merge imports

---------

Co-authored-by: Dhruv Bhanushali <dhruv_b@live.com>
---
 api/api/templates/robots.txt | 8 ++++++++
 api/conf/urls/__init__.py    | 9 ++++++++-
 2 files changed, 16 insertions(+), 1 deletion(-)
 create mode 100644 api/api/templates/robots.txt

diff --git a/api/api/templates/robots.txt b/api/api/templates/robots.txt
new file mode 100644
index 00000000000..811e0134451
--- /dev/null
+++ b/api/api/templates/robots.txt
@@ -0,0 +1,8 @@
+# Block API endpoints
+User-agent: *
+Disallow: /v1/images/
+Disallow: /v1/audio/
+Disallow: /v1/auth/
+
+User-agent: GPTBot
+Disallow: /
diff --git a/api/conf/urls/__init__.py b/api/conf/urls/__init__.py
index 432bc5aa8f9..83b3a1ae7f6 100644
--- a/api/conf/urls/__init__.py
+++ b/api/conf/urls/__init__.py
@@ -8,7 +8,7 @@
 from django.conf.urls.static import static
 from django.contrib import admin
 from django.urls import include, path
-from django.views.generic import RedirectView
+from django.views.generic import RedirectView, TemplateView
 from rest_framework.routers import SimpleRouter
 
 from api.views.audio_views import AudioViewSet
@@ -35,6 +35,13 @@
     path("admin/", admin.site.urls),
     path("healthcheck/", HealthCheck.as_view(), name="health"),
     path("v1/", include(versioned_paths)),
+    path(
+        "robots.txt/",
+        TemplateView.as_view(
+            template_name="robots.txt",
+            content_type="text/plain",
+        ),
+    ),
 ]
 
 if settings.ENVIRONMENT == "local":