diff --git a/api/api/templates/robots.txt b/api/api/templates/robots.txt new file mode 100644 index 00000000000..811e0134451 --- /dev/null +++ b/api/api/templates/robots.txt @@ -0,0 +1,8 @@ +# Block API endpoints +User-agent: * +Disallow: /v1/images/ +Disallow: /v1/audio/ +Disallow: /v1/auth/ + +User-agent: GPTBot +Disallow: / diff --git a/api/conf/urls/__init__.py b/api/conf/urls/__init__.py index 432bc5aa8f9..83b3a1ae7f6 100644 --- a/api/conf/urls/__init__.py +++ b/api/conf/urls/__init__.py @@ -8,7 +8,7 @@ from django.conf.urls.static import static from django.contrib import admin from django.urls import include, path -from django.views.generic import RedirectView +from django.views.generic import RedirectView, TemplateView from rest_framework.routers import SimpleRouter from api.views.audio_views import AudioViewSet @@ -35,6 +35,13 @@ path("admin/", admin.site.urls), path("healthcheck/", HealthCheck.as_view(), name="health"), path("v1/", include(versioned_paths)), + path( + "robots.txt/", + TemplateView.as_view( + template_name="robots.txt", + content_type="text/plain", + ), + ), ] if settings.ENVIRONMENT == "local":