Update robots to allow crawling on non-prod

sgcarstrends · Dec 2, 2024 · fdde46e · fdde46e
1 parent c268c7e
commit fdde46e
Showing 1 changed file with 6 additions and 3 deletions.
diff --git a/app/robots.ts b/app/robots.ts
@@ -4,13 +4,16 @@ import type { MetadataRoute } from "next";
 
 const robots = (): MetadataRoute.Robots => {
   // Set the default rule
-  let rules: MetadataRoute.Robots["rules"] = { userAgent: "*" };
+  let rules: MetadataRoute.Robots["rules"];
 
   // Allow or disallow indexing based on app environment
   if (APP_ENV === AppEnv.PROD) {
-    rules = { ...rules, allow: "/" };
+    rules = { userAgent: "*", allow: "/" };
   } else {
-    rules = { ...rules, disallow: "/" };
+    rules = [
+      { userAgent: "*", disallow: "/" },
+      { userAgent: "AhrefsSiteAudit", allow: "/" },
+    ];
   }
 
   return {