[Issue #2841] Make the robots.txt dynamic so that it can mutate per e…

…nvironment (#3379)

## Summary
Fixes #2841

### Time to review: __5 mins__

## Changes proposed
We need to make the robots.txt dynamic per environment so that we can
ban crawling of the lower environments but still support nuanced rules
for upper environments.

## Context for reviewers
As a follow up change we will also make the Sitemap dynamic as well
since we don't want to point to Sitemaps outside of Production. Our
Sitemap isn't currently ready for being utilized so not including it for
now.

## Additional information
Tested this locally and confirmed that we do have a dynamic robots.txt
that reacts to changes to the environment variable at runtime, not build
time. Not sure if this will all hook together as expected when it tries
to run in AWS, so having this test for "dev" temporarily until we prove
it is all likely to work in Prod.

frontend/src/app/robots.ts

@@ -0,0 +1,42 @@
+// initial rules were absorbed from static robots.txt file
+
+import type { MetadataRoute } from "next";
+import { environment } from "src/constants/environments";
+
+export const dynamic = "force-dynamic";
+
+export default function robots(): MetadataRoute.Robots {
+  return {
+    rules: [
+      environment.ENVIRONMENT === "dev" // switching this temporarily to ensure that the variable is being set at AWS runtime as expected, will make it "prod" after confirming in Dev
+        ? {
+            userAgent: "*",
+            allow: "/",
+            disallow: [
+              // don't disallow search for now as without a sitemap it's Google's only way of finding stuff
+              // search is a high cost, low information subset of the opportunity page data, which is also available via the Sitemap (soon)
+              // "/search",
+              // Prevent crawling of Next.js build files.
+              "/_next/",
+              "/_next*",
+              "/img/",
+              "/*.json$",
+              "/*_buildManifest.js$",
+              "/*_middlewareManifest.js$",
+              "/*_ssgManifest.js$",
+              "/*.js$",
+              // Prevent crawling of Next.js api routes.
+              "/api/",
+              // Prevent crawling of static assets in the public folder.
+              "/public/",
+            ],
+          }
+        : {
+            userAgent: "*",
+            disallow: "/",
+          },
+    ],
+    // our sitemap isn't ready yet
+    // sitemap: "https://acme.com/sitemap.xml",
+  };
+}

frontend/src/constants/environments.ts

@@ -1,5 +1,4 @@
 const {
-  NODE_ENV,
   NEXT_PUBLIC_BASE_PATH,
   USE_SEARCH_MOCK_DATA = "",
   SENDY_API_URL,
@@ -11,14 +10,13 @@ const {
   FEATURE_SEARCH_OFF = "false",
   FEATURE_OPPORTUNITY_OFF = "false",
   NEXT_BUILD = "false",
+  ENVIRONMENT = "dev",
 } = process.env;
 
 // home for all interpreted server side environment variables
 export const environment: { [key: string]: string } = {
   LEGACY_HOST:
-    NODE_ENV === "production"
-      ? "https://grants.gov"
-      : "https://test.grants.gov",
+    ENVIRONMENT === "prod" ? "https://grants.gov" : "https://test.grants.gov",
   NEXT_PUBLIC_BASE_PATH: NEXT_PUBLIC_BASE_PATH ?? "",
   USE_SEARCH_MOCK_DATA,
   SENDY_API_URL: SENDY_API_URL || "",
@@ -31,4 +29,5 @@ export const environment: { [key: string]: string } = {
   FEATURE_OPPORTUNITY_OFF,
   FEATURE_SEARCH_OFF,
   NEXT_BUILD,
+  ENVIRONMENT,
 };

infra/frontend/app-config/env-config/environment-variables.tf

@@ -7,6 +7,8 @@ locals {
     NEW_RELIC_ENABLED = "true"
     # see https://github.com/newrelic/node-newrelic?tab=readme-ov-file#setup
     NODE_OPTIONS = "-r newrelic"
+    # expose the current AWS Env to the FE Next Node Server at Runtime
+    ENVIRONMENT = var.environment
   }
 
   # Configuration for secrets