From fdde46e2c351e83f094da43f54ee6a8502654dc7 Mon Sep 17 00:00:00 2001
From: Ru Chern Chong <ruchern.chong@sgcarstrends.com>
Date: Mon, 2 Dec 2024 15:30:31 +0800
Subject: [PATCH] Update robots to allow crawling on non-prod

---
 app/robots.ts | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/app/robots.ts b/app/robots.ts
index 735c522..55d5cc5 100644
--- a/app/robots.ts
+++ b/app/robots.ts
@@ -4,13 +4,16 @@ import type { MetadataRoute } from "next";
 
 const robots = (): MetadataRoute.Robots => {
   // Set the default rule
-  let rules: MetadataRoute.Robots["rules"] = { userAgent: "*" };
+  let rules: MetadataRoute.Robots["rules"];
 
   // Allow or disallow indexing based on app environment
   if (APP_ENV === AppEnv.PROD) {
-    rules = { ...rules, allow: "/" };
+    rules = { userAgent: "*", allow: "/" };
   } else {
-    rules = { ...rules, disallow: "/" };
+    rules = [
+      { userAgent: "*", disallow: "/" },
+      { userAgent: "AhrefsSiteAudit", allow: "/" },
+    ];
   }
 
   return {