diff --git a/frontend/public/robots.txt b/frontend/public/robots.txt deleted file mode 100644 index 1562495ce..000000000 --- a/frontend/public/robots.txt +++ /dev/null @@ -1,16 +0,0 @@ -User-agent: * -Allow: / -# Prevent crawling of Next.js build files. -Disallow: /_next/ -Disallow: /_next* -Disallow: /img/ -Disallow: /*.json$ -Disallow: /*_buildManifest.js$ -Disallow: /*_middlewareManifest.js$ -Disallow: /*_ssgManifest.js$ -Disallow: /*.js$ -# Prevent crawling of Next.js api routes. -Disallow: /api/ -# Prevent crawling of static assets in the public folder. -Disallow: /public/ - diff --git a/frontend/src/app/robots.ts b/frontend/src/app/robots.ts new file mode 100644 index 000000000..6d8157d6f --- /dev/null +++ b/frontend/src/app/robots.ts @@ -0,0 +1,42 @@ +// initial rules were absorbed from static robots.txt file + +import type { MetadataRoute } from "next"; +import { environment } from "src/constants/environments"; + +export const dynamic = "force-dynamic"; + +export default function robots(): MetadataRoute.Robots { + return { + rules: [ + environment.ENVIRONMENT === "dev" // switching this temporarily to ensure that the variable is being set at AWS runtime as expected, will make it "prod" after confirming in Dev + ? { + userAgent: "*", + allow: "/", + disallow: [ + // don't disallow search for now as without a sitemap it's Google's only way of finding stuff + // search is a high cost, low information subset of the opportunity page data, which is also available via the Sitemap (soon) + // "/search", + // Prevent crawling of Next.js build files. + "/_next/", + "/_next*", + "/img/", + "/*.json$", + "/*_buildManifest.js$", + "/*_middlewareManifest.js$", + "/*_ssgManifest.js$", + "/*.js$", + // Prevent crawling of Next.js api routes. + "/api/", + // Prevent crawling of static assets in the public folder. + "/public/", + ], + } + : { + userAgent: "*", + disallow: "/", + }, + ], + // our sitemap isn't ready yet + // sitemap: "https://acme.com/sitemap.xml", + }; +} diff --git a/frontend/src/constants/environments.ts b/frontend/src/constants/environments.ts index f86aefb98..4bd63598b 100644 --- a/frontend/src/constants/environments.ts +++ b/frontend/src/constants/environments.ts @@ -1,5 +1,4 @@ const { - NODE_ENV, NEXT_PUBLIC_BASE_PATH, USE_SEARCH_MOCK_DATA = "", SENDY_API_URL, @@ -11,14 +10,13 @@ const { FEATURE_SEARCH_OFF = "false", FEATURE_OPPORTUNITY_OFF = "false", NEXT_BUILD = "false", + ENVIRONMENT = "dev", } = process.env; // home for all interpreted server side environment variables export const environment: { [key: string]: string } = { LEGACY_HOST: - NODE_ENV === "production" - ? "https://grants.gov" - : "https://test.grants.gov", + ENVIRONMENT === "prod" ? "https://grants.gov" : "https://test.grants.gov", NEXT_PUBLIC_BASE_PATH: NEXT_PUBLIC_BASE_PATH ?? "", USE_SEARCH_MOCK_DATA, SENDY_API_URL: SENDY_API_URL || "", @@ -31,4 +29,5 @@ export const environment: { [key: string]: string } = { FEATURE_OPPORTUNITY_OFF, FEATURE_SEARCH_OFF, NEXT_BUILD, + ENVIRONMENT, }; diff --git a/infra/frontend/app-config/env-config/environment-variables.tf b/infra/frontend/app-config/env-config/environment-variables.tf index db0bfa482..bb08013ea 100644 --- a/infra/frontend/app-config/env-config/environment-variables.tf +++ b/infra/frontend/app-config/env-config/environment-variables.tf @@ -7,6 +7,8 @@ locals { NEW_RELIC_ENABLED = "true" # see https://github.com/newrelic/node-newrelic?tab=readme-ov-file#setup NODE_OPTIONS = "-r newrelic" + # expose the current AWS Env to the FE Next Node Server at Runtime + ENVIRONMENT = var.environment } # Configuration for secrets