From ff177618deb70b2fbb8ddebc2d528d38af08efbf Mon Sep 17 00:00:00 2001 From: Matt Dragon Date: Thu, 2 Jan 2025 11:38:03 -0500 Subject: [PATCH 1/5] Make the robots.txt dynamic so that it can react to which environment it's executing in --- frontend/public/robots.txt | 16 -------- frontend/src/app/robots.ts | 40 +++++++++++++++++++ .../env-config/environment-variables.tf | 2 + 3 files changed, 42 insertions(+), 16 deletions(-) delete mode 100644 frontend/public/robots.txt create mode 100644 frontend/src/app/robots.ts diff --git a/frontend/public/robots.txt b/frontend/public/robots.txt deleted file mode 100644 index 1562495ce..000000000 --- a/frontend/public/robots.txt +++ /dev/null @@ -1,16 +0,0 @@ -User-agent: * -Allow: / -# Prevent crawling of Next.js build files. -Disallow: /_next/ -Disallow: /_next* -Disallow: /img/ -Disallow: /*.json$ -Disallow: /*_buildManifest.js$ -Disallow: /*_middlewareManifest.js$ -Disallow: /*_ssgManifest.js$ -Disallow: /*.js$ -# Prevent crawling of Next.js api routes. -Disallow: /api/ -# Prevent crawling of static assets in the public folder. -Disallow: /public/ - diff --git a/frontend/src/app/robots.ts b/frontend/src/app/robots.ts new file mode 100644 index 000000000..c1f1653a3 --- /dev/null +++ b/frontend/src/app/robots.ts @@ -0,0 +1,40 @@ +// initial rules were absorbed from static robots.txt file + +import type { MetadataRoute } from "next"; + +export const dynamic = "force-dynamic"; + +export default function robots(): MetadataRoute.Robots { + return { + rules: [ + process.env.AWS_ENVIRONMENT === "prod" + ? { + userAgent: "*", + allow: "/", + disallow: [ + // search is a high cost, low information subset of the opportunity page data + "/search", + // Prevent crawling of Next.js build files. + "/_next/", + "/_next*", + "/img/", + "/*.json$", + "/*_buildManifest.js$", + "/*_middlewareManifest.js$", + "/*_ssgManifest.js$", + "/*.js$", + // Prevent crawling of Next.js api routes. + "/api/", + // Prevent crawling of static assets in the public folder. + "/public/", + ], + } + : { + userAgent: "*", + disallow: "/", + }, + ], + // our sitemap isn't ready yet + // sitemap: "https://acme.com/sitemap.xml", + }; +} diff --git a/infra/frontend/app-config/env-config/environment-variables.tf b/infra/frontend/app-config/env-config/environment-variables.tf index db0bfa482..be8076fa5 100644 --- a/infra/frontend/app-config/env-config/environment-variables.tf +++ b/infra/frontend/app-config/env-config/environment-variables.tf @@ -7,6 +7,8 @@ locals { NEW_RELIC_ENABLED = "true" # see https://github.com/newrelic/node-newrelic?tab=readme-ov-file#setup NODE_OPTIONS = "-r newrelic" + # expose the current AWS Env to the FE Next Node Server at Runtime + AWS_ENVIRONMENT = var.environment_name } # Configuration for secrets From 917576d0bb1fcd27fbfc3ce6e4d3247c29aacac0 Mon Sep 17 00:00:00 2001 From: Matt Dragon Date: Thu, 2 Jan 2025 11:43:51 -0500 Subject: [PATCH 2/5] Fixes and change for lower env testing --- frontend/src/app/robots.ts | 2 +- infra/frontend/app-config/env-config/environment-variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/robots.ts b/frontend/src/app/robots.ts index c1f1653a3..42b747237 100644 --- a/frontend/src/app/robots.ts +++ b/frontend/src/app/robots.ts @@ -7,7 +7,7 @@ export const dynamic = "force-dynamic"; export default function robots(): MetadataRoute.Robots { return { rules: [ - process.env.AWS_ENVIRONMENT === "prod" + process.env.AWS_ENVIRONMENT === "dev" // switching this temporarily to ensure that the variable is being set at AWS runtime as expected, will make it "prod" after confirming in Dev ? { userAgent: "*", allow: "/", diff --git a/infra/frontend/app-config/env-config/environment-variables.tf b/infra/frontend/app-config/env-config/environment-variables.tf index be8076fa5..cb7c7a645 100644 --- a/infra/frontend/app-config/env-config/environment-variables.tf +++ b/infra/frontend/app-config/env-config/environment-variables.tf @@ -8,7 +8,7 @@ locals { # see https://github.com/newrelic/node-newrelic?tab=readme-ov-file#setup NODE_OPTIONS = "-r newrelic" # expose the current AWS Env to the FE Next Node Server at Runtime - AWS_ENVIRONMENT = var.environment_name + AWS_ENVIRONMENT = var.environment } # Configuration for secrets From be08fc143f8cbd7d50d94e7f1859aa5f9a7be96c Mon Sep 17 00:00:00 2001 From: Matt Dragon Date: Thu, 2 Jan 2025 16:48:35 -0500 Subject: [PATCH 3/5] Leave search crawlable for now --- frontend/src/app/robots.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/robots.ts b/frontend/src/app/robots.ts index 42b747237..944ebb75b 100644 --- a/frontend/src/app/robots.ts +++ b/frontend/src/app/robots.ts @@ -12,8 +12,9 @@ export default function robots(): MetadataRoute.Robots { userAgent: "*", allow: "/", disallow: [ - // search is a high cost, low information subset of the opportunity page data - "/search", + // don't disallow search for now as without a sitemap it's Google's only way of finding stuff + // search is a high cost, low information subset of the opportunity page data, which is also available via the Sitemap (soon) + // "/search", // Prevent crawling of Next.js build files. "/_next/", "/_next*", From 2ac3ba6456a6a89d2eac9ee98f3445832652cb9f Mon Sep 17 00:00:00 2001 From: Matt Dragon Date: Fri, 3 Jan 2025 10:25:49 -0500 Subject: [PATCH 4/5] Pull Env Var through environments --- frontend/src/app/robots.ts | 3 ++- frontend/src/constants/environments.ts | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/robots.ts b/frontend/src/app/robots.ts index 944ebb75b..03eb111c1 100644 --- a/frontend/src/app/robots.ts +++ b/frontend/src/app/robots.ts @@ -1,13 +1,14 @@ // initial rules were absorbed from static robots.txt file import type { MetadataRoute } from "next"; +import { environment } from "src/constants/environments"; export const dynamic = "force-dynamic"; export default function robots(): MetadataRoute.Robots { return { rules: [ - process.env.AWS_ENVIRONMENT === "dev" // switching this temporarily to ensure that the variable is being set at AWS runtime as expected, will make it "prod" after confirming in Dev + environment.AWS_ENVIRONMENT === "dev" // switching this temporarily to ensure that the variable is being set at AWS runtime as expected, will make it "prod" after confirming in Dev ? { userAgent: "*", allow: "/", diff --git a/frontend/src/constants/environments.ts b/frontend/src/constants/environments.ts index f86aefb98..6fb04a420 100644 --- a/frontend/src/constants/environments.ts +++ b/frontend/src/constants/environments.ts @@ -11,6 +11,7 @@ const { FEATURE_SEARCH_OFF = "false", FEATURE_OPPORTUNITY_OFF = "false", NEXT_BUILD = "false", + AWS_ENVIRONMENT = "dev", } = process.env; // home for all interpreted server side environment variables @@ -31,4 +32,5 @@ export const environment: { [key: string]: string } = { FEATURE_OPPORTUNITY_OFF, FEATURE_SEARCH_OFF, NEXT_BUILD, + AWS_ENVIRONMENT, }; From 3a279df944fd93a1c91fc00bb86c0db81634526f Mon Sep 17 00:00:00 2001 From: Matt Dragon Date: Fri, 3 Jan 2025 13:41:52 -0500 Subject: [PATCH 5/5] Clean up per PR --- frontend/src/app/robots.ts | 2 +- frontend/src/constants/environments.ts | 9 +++------ .../app-config/env-config/environment-variables.tf | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/frontend/src/app/robots.ts b/frontend/src/app/robots.ts index 03eb111c1..6d8157d6f 100644 --- a/frontend/src/app/robots.ts +++ b/frontend/src/app/robots.ts @@ -8,7 +8,7 @@ export const dynamic = "force-dynamic"; export default function robots(): MetadataRoute.Robots { return { rules: [ - environment.AWS_ENVIRONMENT === "dev" // switching this temporarily to ensure that the variable is being set at AWS runtime as expected, will make it "prod" after confirming in Dev + environment.ENVIRONMENT === "dev" // switching this temporarily to ensure that the variable is being set at AWS runtime as expected, will make it "prod" after confirming in Dev ? { userAgent: "*", allow: "/", diff --git a/frontend/src/constants/environments.ts b/frontend/src/constants/environments.ts index 6fb04a420..4bd63598b 100644 --- a/frontend/src/constants/environments.ts +++ b/frontend/src/constants/environments.ts @@ -1,5 +1,4 @@ const { - NODE_ENV, NEXT_PUBLIC_BASE_PATH, USE_SEARCH_MOCK_DATA = "", SENDY_API_URL, @@ -11,15 +10,13 @@ const { FEATURE_SEARCH_OFF = "false", FEATURE_OPPORTUNITY_OFF = "false", NEXT_BUILD = "false", - AWS_ENVIRONMENT = "dev", + ENVIRONMENT = "dev", } = process.env; // home for all interpreted server side environment variables export const environment: { [key: string]: string } = { LEGACY_HOST: - NODE_ENV === "production" - ? "https://grants.gov" - : "https://test.grants.gov", + ENVIRONMENT === "prod" ? "https://grants.gov" : "https://test.grants.gov", NEXT_PUBLIC_BASE_PATH: NEXT_PUBLIC_BASE_PATH ?? "", USE_SEARCH_MOCK_DATA, SENDY_API_URL: SENDY_API_URL || "", @@ -32,5 +29,5 @@ export const environment: { [key: string]: string } = { FEATURE_OPPORTUNITY_OFF, FEATURE_SEARCH_OFF, NEXT_BUILD, - AWS_ENVIRONMENT, + ENVIRONMENT, }; diff --git a/infra/frontend/app-config/env-config/environment-variables.tf b/infra/frontend/app-config/env-config/environment-variables.tf index cb7c7a645..bb08013ea 100644 --- a/infra/frontend/app-config/env-config/environment-variables.tf +++ b/infra/frontend/app-config/env-config/environment-variables.tf @@ -8,7 +8,7 @@ locals { # see https://github.com/newrelic/node-newrelic?tab=readme-ov-file#setup NODE_OPTIONS = "-r newrelic" # expose the current AWS Env to the FE Next Node Server at Runtime - AWS_ENVIRONMENT = var.environment + ENVIRONMENT = var.environment } # Configuration for secrets