build: clean up configuration and build

mirrors improvements in comses/comses.net#696

* use one shared .env for all non-secret config (replaces server/.env
  and client/.../config.ts)
* continue to use files for secrets, but with docker compose secrets
* move base_url mapping to shared/settings.ts from `configure` script
* replace server/deploy/ with top level deploy dir
* clean up anything unused or unecessary from Makefile

.dockerignore

@@ -1,6 +1,7 @@
 client/node_modules
 server/node_modules
-server/deploy/Dockerfile*
+server/Dockerfile.dev
+server/Dockerfile.prod
 server/output
 docker
 keys

Makefile

@@ -1,54 +1,39 @@
-include config.mk
-
-DB_USER=marsmadness
-TEST_DB_NAME=pom_testing
 DB_DATA_PATH=docker/data
 DATA_DUMP_PATH=docker/dump
 LOG_DATA_PATH=docker/logs
+
 DB_PASSWORD_PATH=keys/pom_db_password
-REDIS_SETTINGS_PATH=keys/settings.json
-SERVER_ENV_TEMPLATE=server/.env.template
-SERVER_ENV=server/.env
 PGPASS_PATH=keys/.pgpass
 SECRET_KEY_PATH=keys/secret_key
-SENTRY_DSN_PATH=keys/sentry_dsn
-SENTRY_DSN=$(shell cat $(SENTRY_DSN_PATH))
-MAIL_API_KEY_PATH=keys/mail_api_key
-SECRETS=$(MAIL_API_KEY_PATH) $(DB_PASSWORD_PATH) $(PGPASS_PATH) $(SENTRY_DSN_PATH) $(SECRET_KEY_PATH)
-SHARED_CONFIG_PATH=shared/src/assets/config.ts
-BUILD_ID=$(shell git describe --tags --abbrev=1)
-GA_TAG_PATH=keys/ga_tag
-GA_TAG=$(shell cat $(GA_TAG_PATH))
+GENERATED_SECRETS=$(DB_PASSWORD_PATH) $(PGPASS_PATH) $(SECRET_KEY_PATH)
+EXT_SECRETS=mail_api_key google_client_secret facebook_client_secret
 
-.PHONY: build
-build: docker-compose.yml
-	docker compose pull db redis
-	docker compose build --pull
+ENVREPLACE := deploy/scripts/envreplace
+DEPLOY_CONF_DIR=deploy/conf
+ENV_TEMPLATE=${DEPLOY_CONF_DIR}/.env.template
+DYNAMIC_SETTINGS_TEMPLATE=${DEPLOY_CONF_DIR}/settings.template.json
+DYNAMIC_SETTINGS_PATH=keys/settings.json
+
+include config.mk
+include .env
 
-.PHONY: browser
-browser:
-	firefox --new-tab --url 'ext+container:name=Bob&url=http://localhost:8081/#/game' \
-		--new-tab --url 'ext+container:name=Amanda&url=http://localhost:8081/#/game' \
-		--new-tab --url 'ext+container:name=Frank&url=http://localhost:8081/#/game' \
-		--new-tab --url 'ext+container:name=Sydney&url=http://localhost:8081/#/game' \
-		--new-tab --url 'ext+container:name=Adison&url=http://localhost:8081/#/game' \
-		--new-tab --url 'ext+container:name=Bob2&url=http://localhost:8081/#/game' \
-		--new-tab --url 'ext+container:name=Amanda2&url=http://localhost:8081/#/game' \
-		--new-tab --url 'ext+container:name=Frank2&url=http://localhost:8081/#/game' \
-		--new-tab --url 'ext+container:name=Sydney2&url=http://localhost:8081/#/game' \
-		--new-tab --url 'ext+container:name=Adison2&url=http://localhost:8081/#/game'
-
-.PHONY: browser-staging
-browser-staging:
-	firefox --new-tab --url 'ext+container:name=Bob&url=http://alpha.portofmars.asu.edu' \
-		--new-tab --url 'ext+container:name=Amanda&url=http://alpha.portofmars.asu.edu' \
-		--new-tab --url 'ext+container:name=Frank&url=http://alpha.portofmars.asu.edu' \
-		--new-tab --url 'ext+container:name=Sydney&url=http://alpha.portofmars.asu.edu' \
-		--new-tab --url 'ext+container:name=Adison&url=http://alpha.portofmars.asu.edu'
+.EXPORT_ALL_VARIABLES:
+
+$(LOG_DATA_PATH):
+	mkdir -p $(LOG_DATA_PATH)
+
+$(DB_DATA_PATH):
+	mkdir -p "$(DB_DATA_PATH)"
+
+$(DATA_DUMP_PATH):
+	mkdir -p $(DATA_DUMP_PATH)
 
 keys:
 	mkdir -p keys
 
+$(DYNAMIC_SETTINGS_PATH): $(DYNAMIC_SETTINGS_TEMPLATE) | keys
+	cp $(DYNAMIC_SETTINGS_TEMPLATE) $(DYNAMIC_SETTINGS_PATH)
+
 $(DB_PASSWORD_PATH): | keys
 	DB_PASSWORD=$$(openssl rand -base64 48); \
 	TODAY=$$(date +%Y-%m-%d-%H:%M:%S); \
@@ -58,72 +43,56 @@ $(DB_PASSWORD_PATH): | keys
 	fi; \
 	echo "$${DB_PASSWORD}" > $(DB_PASSWORD_PATH)
 
-$(LOG_DATA_PATH):
-	mkdir -p $(LOG_DATA_PATH)
-
-$(DATA_DUMP_PATH):
-	mkdir -p $(DATA_DUMP_PATH)
-
-$(REDIS_SETTINGS_PATH): server/deploy/settings.template.json | keys
-	cp server/deploy/settings.template.json $(REDIS_SETTINGS_PATH)
-
-$(SERVER_ENV): $(SERVER_ENV_TEMPLATE) $(SECRETS)
-	if [ ! -f $(SERVER_ENV) ]; then \
-		cp $(SERVER_ENV_TEMPLATE) $(SERVER_ENV); \
-		DB_PASSWORD=$$(cat $(DB_PASSWORD_PATH)); \
-		SECRET_KEY=$$(cat $(SECRET_KEY_PATH)); \
-		sed \
-			-e "s|BASE_URL=.*|BASE_URL=${POM_BASE_URL}|" \
-			-e "s|DB_PASSWORD=.*|DB_PASSWORD=$${DB_PASSWORD}|" \
-			-e "s|SECRET_KEY=.*|SECRET_KEY=$${SECRET_KEY}|" \
-			$(SERVER_ENV_TEMPLATE) > $(SERVER_ENV); \
-	else \
-		echo "$(SERVER_ENV) already exists. skipping"; \
-	fi
-
-$(PGPASS_PATH): $(DB_PASSWORD_PATH) server/deploy/pgpass.template | keys
-	DB_PASSWORD=$$(cat $(DB_PASSWORD_PATH)); \
-	sed "s|DB_PASSWORD|$$DB_PASSWORD|g" server/deploy/pgpass.template > $(PGPASS_PATH)
+$(PGPASS_PATH): $(DB_PASSWORD_PATH) | keys
+	echo "${DB_HOST}:5432:*:${DB_USER}:$$(cat $(DB_PASSWORD_PATH))" > $(PGPASS_PATH)
 	chmod 0600 $(PGPASS_PATH)
 
-$(MAIL_API_KEY_PATH): | keys
-	touch "$(MAIL_API_KEY_PATH)"
+$(SECRET_KEY_PATH): | keys
+	SECRET_KEY=$$(openssl rand -base64 48); \
+	echo $${SECRET_KEY} > $(SECRET_KEY_PATH)
 
-$(SENTRY_DSN_PATH): | keys
-	touch "$(SENTRY_DSN_PATH)"
+.PHONY: secrets
+secrets: keys $(GENERATED_SECRETS)
+	for secret_path in $(EXT_SECRETS); do \
+		touch keys/$$secret_path; \
+	done
+
+.env: $(ENV_TEMPLATE)
+	if [ ! -f .env ];  then \
+		cp $(ENV_TEMPLATE) .env; \
+	fi
 
-$(GA_TAG_PATH): | keys
-	touch "$(GA_TAG_PATH)"
+.PHONY: release-version
+release-version: .env
+	$(ENVREPLACE) RELEASE_VERSION $$(git describe --tags --abbrev=1) .env
 
-$(DB_DATA_PATH):
-	mkdir -p "$(DB_DATA_PATH)"
-
-.PHONY: secrets
-secrets: $(SECRETS)
+docker-compose.yml: base.yml $(DEPLOY_ENVIRONMENT).yml config.mk $(DB_DATA_PATH) $(DATA_DUMP_PATH) $(LOG_DATA_PATH) $(DYNAMIC_SETTINGS_PATH) secrets $(PGPASS_PATH) release-version
+	case "$(DEPLOY_ENVIRONMENT)" in \
+	  dev|staging|prod) docker compose -f base.yml -f "$(DEPLOY_ENVIRONMENT).yml" config > docker-compose.yml;; \
+	  *) echo "invalid environment. must be either dev, staging or prod" 1>&2; exit 1;; \
+	esac
 
-$(SECRET_KEY_PATH): | keys
-	SECRET_KEY=$$(openssl rand -base64 48); \
-	echo $${SECRET_KEY} > $(SECRET_KEY_PATH)
+.PHONY: build
+build: docker-compose.yml
+	docker compose pull db redis
+	docker compose build --pull
 
-.PHONY: settings
-settings: $(SENTRY_DSN_PATH) $(SECRET_KEY_PATH) $(GA_TAG_PATH) | keys
-	echo 'export const BUILD_ID = "${BUILD_ID}";' > $(SHARED_CONFIG_PATH)
-	echo 'export const SENTRY_DSN = "${SENTRY_DSN}";' >> $(SHARED_CONFIG_PATH)
-	echo 'export const GA_TAG = "${GA_TAG}";' >> $(SHARED_CONFIG_PATH)
+.PHONY: deploy
+deploy: build
+	docker compose up -d 
 
+.PHONY: buildprod
+buildprod: docker-compose.yml
+	docker compose run --rm client npm run build
+	docker compose run --rm server npm run build
 
+.PHONY: initialize
 initialize: build
 	docker compose run --rm server npm run initdb
 
-docker-compose.yml: base.yml $(ENVIR).yml config.mk $(DB_DATA_PATH) $(DATA_DUMP_PATH) $(LOG_DATA_PATH) $(REDIS_SETTINGS_PATH) $(PGPASS_PATH) $(SERVER_ENV) settings
-	case "$(ENVIR)" in \
-	  dev|staging|prod) docker compose -f base.yml -f "$(ENVIR).yml" config > docker-compose.yml;; \
-	  *) echo "invalid environment. must be either dev, staging or prod" 1>&2; exit 1;; \
-	esac
-
 .PHONY: test-setup
 test-setup: docker-compose.yml
-	docker compose run --rm server bash -c "dropdb --if-exists -h db -U ${DB_USER} ${TEST_DB_NAME} && createdb -h db -U ${DB_USER} ${TEST_DB_NAME} && npm run test-setup"
+	docker compose run --rm server bash -c "dropdb --if-exists -h ${DB_HOST} -U ${DB_USER} ${TEST_DB_NAME} && createdb -h db -U ${DB_USER} ${TEST_DB_NAME} && npm run test-setup"
 
 .PHONY: test
 test: test-setup
@@ -132,16 +101,7 @@ test: test-setup
 
 .PHONY: test-server
 test-server: test-setup
-	docker compose run --rm server npm run test $(tests)
-
-.PHONY: deploy
-deploy: build
-	docker compose up -d 
-
-.PHONY: buildprod
-buildprod: docker-compose.yml
-	docker compose run --rm client npm run build
-	docker compose run --rm server npm run build
+	docker compose run --rm server npm run test
 
 .PHONY: docker-clean
 docker-clean:
@@ -151,4 +111,5 @@ docker-clean:
 
 .PHONY: clean
 clean:
-	rm -f server/.env # any other generated resources? SHARED_CONFIG_PATH?
+	@echo "Backing up generated files to /tmp directory"
+	mv .env config.mk docker-compose.yml $(shell mktemp -d)

base.yml

@@ -4,6 +4,14 @@ services:
       context: .
     restart: always
     image: port-of-mars/server:dev
+    secrets:
+      - pom_db_password
+      - mail_api_key
+      - secret_key
+      - google_client_secret
+      - facebook_client_secret
+    env_file:
+      - .env
     depends_on:
       redis:
         condition: service_started
@@ -13,9 +21,8 @@ services:
       - ./docker/dump:/dump
       - ./docker/logs:/var/log/port-of-mars
       - ./keys/.pgpass:/root/.pgpass
-      - ./keys:/run/secrets
+      - ./keys/settings.json:/run/secrets/settings.json
       - ./scripts:/scripts
-      - ./server/.env:/code/server/.env
       - ./.prettierrc:/code/.prettierrc
   redis:
     image: redis:7
@@ -27,12 +34,25 @@ services:
       timeout: 5s
       retries: 5
     image: postgres:12
+    secrets:
+      - pom_db_password
     restart: always
     environment:
-      POSTGRES_USER: marsmadness
+      POSTGRES_DB: ${DB_NAME}
+      POSTGRES_USER: ${DB_USER}
       POSTGRES_PASSWORD_FILE: /run/secrets/pom_db_password
-      POSTGRES_DB: port_of_mars
       PGDATA: /var/lib/postgresql/data/pgdata
     volumes:
-      - ./keys/pom_db_password:/run/secrets/pom_db_password:ro
       - ./docker/data:/var/lib/postgresql/data/pgdata
+
+secrets:
+  pom_db_password:
+    file: ./keys/pom_db_password
+  mail_api_key:
+    file: ./keys/mail_api_key
+  secret_key:
+    file: ./keys/secret_key
+  google_client_secret:
+    file: ./keys/google_client_secret
+  facebook_client_secret:
+    file: ./keys/facebook_client_secret

client/src/components/global/Footer.vue

@@ -93,7 +93,7 @@ [email protected]
       © 2020-{{ currentYear }}
       <a href="https://www.azregents.edu/">Arizona Board of Regents</a> |
 
-      <a :href="constants.GITHUB_URL">{{ constants.BUILD_ID }}</a>
+      <a :href="constants.GITHUB_URL">{{ constants.RELEASE_VERSION }}</a>
     </div>
   </footer>
   <!-- </b-container> -->

client/src/main.ts

@@ -7,7 +7,7 @@ import Vuex from "vuex";
 import * as Sentry from "@sentry/browser";
 import { Vue as VueIntegration } from "@sentry/integrations";
 import { Integrations } from "@sentry/tracing";
-import { isStagingOrProduction, Constants } from "@port-of-mars/shared/settings";
+import { isStagingOrProduction, Constants, SERVER_URL_WS } from "@port-of-mars/shared/settings";
 import { Ajax } from "@port-of-mars/client/plugins/ajax";
 import { TypedStore } from "@port-of-mars/client/plugins/tstore";
 import { getAssetUrl, SfxManager } from "@port-of-mars/client/util";
@@ -40,7 +40,7 @@ if (isStagingOrProduction()) {
   );
 }
 
-const $client = new Colyseus.Client(process.env.SERVER_URL_WS || undefined);
+const $client = new Colyseus.Client(SERVER_URL_WS || undefined);
 const $sfx = new SfxManager();
 
 Vue.prototype.$getAssetUrl = getAssetUrl;

client/src/util.ts

@@ -1,10 +1,11 @@
 import { SetSfx } from "@port-of-mars/shared/game/responses";
 import { Sfx } from "@port-of-mars/shared/game/responses";
+import { SERVER_URL_HTTP } from "@port-of-mars/shared/settings";
 import { Howl } from "howler";
 
 export function url(path: string) {
   // workaround to connect to localhost:2567 server endpoints
-  return `${process.env.SERVER_URL_HTTP}${path}`;
+  return `${SERVER_URL_HTTP}${path}`;
 }
 
 export function getAssetUrl(path: string) {

client/vite.config.ts

@@ -1,14 +1,8 @@
 import vue from "@vitejs/plugin-vue2";
 import { defineConfig } from "vite";
 import path from "path";
-import fs from "fs";
 import autoprefixer from "autoprefixer";
 
-let SENTRY_DSN = "";
-if (fs.existsSync("/run/secrets/sentry_dsn")) {
-  SENTRY_DSN = fs.readFileSync("/run/secrets/sentry_dsn", "utf8").trim();
-}
-
 export default defineConfig({
   plugins: [vue()],
   server: {
@@ -36,12 +30,10 @@ export default defineConfig({
     },
   },
   define: {
-    "process.env.SERVER_URL_WS": JSON.stringify(
-      process.env.NODE_ENV === "development" ? "ws://localhost:2567" : ""
-    ),
-    "process.env.SERVER_URL_HTTP": JSON.stringify(
-      process.env.NODE_ENV === "development" ? "http://localhost:2567" : ""
-    ),
-    "process.env.SENTRY_DSN": JSON.stringify(SENTRY_DSN),
+    // make process.env work on the client
+    ...Object.keys(process.env).reduce((acc, key) => {
+      acc[`process.env.${key}`] = JSON.stringify(process.env[key]);
+      return acc;
+    }, {}),
   },
 });

configure

@@ -4,18 +4,8 @@ set -o nounset
 set -o pipefail
 set -o errexit
 
-declare -A url_map
+DEPLOY_ENVIRONMENT=${1:-dev} # dev|staging|prod
 
-url_map['dev']='http://localhost:8081'
-url_map['staging']='https://staging.portofmars.asu.edu'
-url_map['prod']='https://portofmars.asu.edu'
+echo "configuring for **${DEPLOY_ENVIRONMENT}** environment"
 
-ENVIR=${1:-dev} # dev|staging|prod
-POM_BASE_URL=${url_map[$ENVIR]}
-
-echo "configuring for **${ENVIR}** environment with base url $POM_BASE_URL"
-
-envsubst > config.mk <<EOF
-ENVIR=$ENVIR
-POM_BASE_URL=$POM_BASE_URL
-EOF
+echo "DEPLOY_ENVIRONMENT=${DEPLOY_ENVIRONMENT}" > config.mk

deploy/conf/.env.template

@@ -0,0 +1,12 @@
+RELEASE_VERSION=
+
+DB_HOST=db
+DB_NAME=port_of_mars
+DB_USER=marsmadness
+TEST_DB_NAME=pom_testing
+
+SENTRY_DSN=
+GA_TAG=
+
+GOOGLE_CLIENT_ID=changeme
+FACEBOOK_CLIENT_ID=changeme