[Issue #2841] Make the robots.txt dynamic so that it can mutate per environment

frontend/src/app/robots.ts

@@ -0,0 +1,42 @@
+// initial rules were absorbed from static robots.txt file
+
+import type { MetadataRoute } from "next";
+import { environment } from "src/constants/environments";
+
+export const dynamic = "force-dynamic";
+
+export default function robots(): MetadataRoute.Robots {
+  return {
+    rules: [
+      environment.ENVIRONMENT === "dev" // switching this temporarily to ensure that the variable is being set at AWS runtime as expected, will make it "prod" after confirming in Dev
+        ? {
+            userAgent: "*",
+            allow: "/",
+            disallow: [
+              // don't disallow search for now as without a sitemap it's Google's only way of finding stuff
+              // search is a high cost, low information subset of the opportunity page data, which is also available via the Sitemap (soon)
+              // "/search",
+              // Prevent crawling of Next.js build files.
+              "/_next/",
+              "/_next*",
+              "/img/",
+              "/*.json$",
+              "/*_buildManifest.js$",
+              "/*_middlewareManifest.js$",
+              "/*_ssgManifest.js$",
+              "/*.js$",
+              // Prevent crawling of Next.js api routes.
+              "/api/",
+              // Prevent crawling of static assets in the public folder.
+              "/public/",
+            ],
+          }
+        : {
+            userAgent: "*",
+            disallow: "/",
+          },
+    ],
+    // our sitemap isn't ready yet
+    // sitemap: "https://acme.com/sitemap.xml",
+  };
+}

frontend/src/constants/environments.ts

@@ -1,5 +1,4 @@
 const {
-  NODE_ENV,
   NEXT_PUBLIC_BASE_PATH,
   USE_SEARCH_MOCK_DATA = "",
   SENDY_API_URL,
@@ -11,14 +10,13 @@ const {
   FEATURE_SEARCH_OFF = "false",
   FEATURE_OPPORTUNITY_OFF = "false",
   NEXT_BUILD = "false",
+  ENVIRONMENT = "dev",
 } = process.env;
 
 // home for all interpreted server side environment variables
 export const environment: { [key: string]: string } = {
   LEGACY_HOST:
-    NODE_ENV === "production"
-      ? "https://grants.gov"
-      : "https://test.grants.gov",
+    ENVIRONMENT === "prod" ? "https://grants.gov" : "https://test.grants.gov",
   NEXT_PUBLIC_BASE_PATH: NEXT_PUBLIC_BASE_PATH ?? "",
   USE_SEARCH_MOCK_DATA,
   SENDY_API_URL: SENDY_API_URL || "",
@@ -31,4 +29,5 @@ export const environment: { [key: string]: string } = {
   FEATURE_OPPORTUNITY_OFF,
   FEATURE_SEARCH_OFF,
   NEXT_BUILD,
+  ENVIRONMENT,
 };

infra/frontend/app-config/env-config/environment-variables.tf

@@ -7,6 +7,8 @@ locals {
     NEW_RELIC_ENABLED = "true"
     # see https://github.com/newrelic/node-newrelic?tab=readme-ov-file#setup
     NODE_OPTIONS = "-r newrelic"
+    # expose the current AWS Env to the FE Next Node Server at Runtime
+    ENVIRONMENT = var.environment
   }
 
   # Configuration for secrets