PR-455 - merge

.github/workflows/basic-tests.yml

@@ -1,6 +1,6 @@
-name: 'Basic (Unit) Tests'
+# Install python and node. Run lint and unit tests.
 
-# **What it does**: Setups up python dependencies and runs tests.
+name: 'Basic (Unit) Tests'
 # **Why we have it**: Automatically run tests to ensure code doesn't introduce regressions.
 # **Who does it impact**: Python small-scale "unit" tests.
 

.github/workflows/external-pr-open.yml

@@ -1,7 +1,6 @@
 # Ambuda workflow to build and publish docker image
 
-name: Build and test incoming
-
+name: Basic image build and test
 env:
   AMBUDA_VERSION: v0.1
   AMBUDA_HOST_IP: 127.0.0.1
@@ -19,7 +18,6 @@ jobs:
   build_and_test:
     name: Build external pr image
     runs-on: ubuntu-22.04
-    environment: staging
     permissions:
       packages: write
       contents: read
@@ -38,7 +36,7 @@ jobs:
 
       - name: Build and Test docker image
         id: build-to-test
-        uses: docker/build-push-action@v3.2.0
+        uses: docker/build-push-action@v4.0.0
         with:
           context: .
           file: build/containers/Dockerfile.final

.github/workflows/rel-pr-create.yml

@@ -1,14 +1,13 @@
-# Ambuda workflow to build and publish docker image
-
-name: Create Release PR on seeing new code 
+# Check for changes in "main". Push changes to "releases".
 
+name: Push changes from main to releases
 env:
   GH_TOKEN: ${{ github.token }}
 
   # head/branch where current changes reside befor merge
-  PR_SOURCE_BRANCH: development
+  PR_SOURCE_BRANCH: main
   # base - branch intended to change once the proposed changes are meged.
-  PR_TARGET_BRANCH: releases
+  PR_TARGET_BRANCH: release
 
 on:
   workflow_dispatch:
@@ -28,16 +27,16 @@ jobs:
       - uses: actions/checkout@v3
         with:
           ref: ${{ env.PR_TARGET_BRANCH }}  
-      - name: Find the recent PR merge on development
+      - name: Find the recent PR merge on ${{ env.PR_SOURCE_BRANCH }}
         id: find_pr
         run: |
-          LAST_RUNTIME=$(date +'%Y-%m-%dT%H:%M:%S' --date '-30000 min')
+          LAST_RUNTIME=$(date +'%Y-%m-%dT%H:%M:%S' --date '-1800 min')
           gh repo set-default ambuda-org/ambuda
           echo "PR_NUMBER=$(gh pr list --state merged --base ${{ env.PR_SOURCE_BRANCH }} --search "merged:>$LAST_RUNTIME" -L 1 --json number| jq '.[].number')" >> $GITHUB_OUTPUT
 
   create_pr:
     runs-on: ubuntu-22.04
-    name: Create PR on releases branch
+    name: Create PR on release branch
     environment: staging
     permissions:
       packages: write
@@ -61,4 +60,4 @@ jobs:
           base: ${{ env.PR_TARGET_BRANCH }}  
           branch: ${{ env.PR_TARGET_BRANCH }}-${{ env.PR_NUMBER }}
           title: PR-${{ env.PR_NUMBER }} - merge
-          body:  development/PR-${{ env.PR_NUMBER }} merge is open
+          body:  ${{ env.PR_SOURCE_BRANCH }}/PR-${{ env.PR_NUMBER }} merge is open

.github/workflows/rel-pr-merged.yml

@@ -1,7 +1,6 @@
-# Ambuda workflow to build and publish docker image
-
-name: Teardown staging
+# Cleanup staging. Usually occurs after a pr is merged or closed.
 
+name: Teardown staging deployment
 env:
   AMBUDA_VERSION: v0.1
 
@@ -13,7 +12,7 @@ on:
   #     - 'v*'
   pull_request:
     branches:
-      - 'releases'
+      - 'release'
     types:
       - closed
 

.github/workflows/rel-pr-open.yml

@@ -1,17 +1,17 @@
-# Ambuda workflow to build and publish docker image
-
-name: Build publish and staging
+# Build release image. Push image to ghcr.io. Deploy on staging environment.
 
+name: Release image build & publish
 env:
   AMBUDA_VERSION: v0.1
   AMBUDA_HOST_IP: 127.0.0.1
   AMBUDA_HOST_PORT: 5090
   REGISTRY: ghcr.io
 
 on:
+  workflow_dispatch:
   pull_request:
     branches:
-      - 'releases'
+      - 'release'
     types: [opened, reopened, synchronize]
 
 jobs:
@@ -60,7 +60,7 @@ jobs:
 
       - name: Build and push Docker images
         id: publish
-        uses: docker/build-push-action@v3.2.0
+        uses: docker/build-push-action@v4.0.0
         with:
           context: .
           file: build/containers/Dockerfile.final

.gitignore

@@ -1,6 +1,8 @@
 # Large data files, including texts, parse data, and (in local dev)
 # image uploads.
 data/
+deploy/data_database
+deploy/data_files
 
 # Autogenerated files (compiled CSS, testing, documentation, ...)
 .coverage

Makefile

@@ -1,3 +1,34 @@
+# Environment. Valid values are: local, staging, and prod
+AMBUDA_DEPLOYMENT_ENV=local
+AMBUDA_HOST_IP=0.0.0.0
+AMBUDA_HOST_PORT=5000
+
+# Control the verbosity of messages using a flag
+ifdef mode
+	ifeq ("$(origin mode)", "command line")
+		BUILD_MODE = $(mode)
+	endif
+else
+	BUILD_MODE = default
+endif
+
+ifdef ($(BUILD_MODE),dev)
+	IO_REDIRECT = 
+	DOCKER_VERBOSITY = 
+	DOCKER_LOG_LEVEL = 
+	DOCKER_DETACH = 
+else ifeq ($(BUILD_MODE),quiet)
+	IO_REDIRECT = &> /dev/null
+	DOCKER_VERBOSITY = -qq
+	DOCKER_LOG_LEVEL = --log-level ERROR
+	DOCKER_DETACH = --detach
+else ifeq ($(BUILD_MODE),default)
+	IO_REDIRECT = 
+	DOCKER_VERBOSITY = 
+	DOCKER_LOG_LEVEsL = 
+	DOCKER_DETACH = --detach
+endif
+
 # Needed because we have folders called "docs" and "test" that confuse `make`.
 .PHONY: docs test py-venv-check clean
 
@@ -11,11 +42,6 @@ AMBUDA_NAME=ambuda
 AMBUDA_IMAGE=${AMBUDA_NAME}:${AMBUDA_VERSION}-${GITBRANCH}-${GITCOMMIT}
 AMBUDA_IMAGE_LATEST="$(AMBUDA_NAME)-rel:latest"
 
-# Environment. Valid values are: local, staging, and prod
-AMBUDA_DEPLOYMENT_ENV=local
-AMBUDA_HOST_IP=0.0.0.0
-AMBUDA_HOST_PORT=5090
-
 py-venv-check: 
 ifeq ("$(VIRTUAL_ENV)","")
 	@echo "Error! Python venv not activated. Activate venv to proceed. Run: "
@@ -96,16 +122,25 @@ db-seed-all: py-venv-check
 	python -m ambuda.seed.dictionaries.shabdasagara
 	python -m ambuda.seed.dictionaries.vacaspatyam
 
-
-# Common development commands
+# Local run commands
 # ===============================================
+.PHONY: devserver celery
+devserver: 
+	make mode=dev docker-start
+
+# Run a local Celery instance for background tasks.
+celery: 
+	celery -A ambuda.tasks worker --loglevel=INFO
 
+# Docker commands
+# ===============================================
+.PHONY: docker-setup-db docker-build docker-start docker-stop docker-logs
 # Start DB using Docker.
 docker-setup-db: docker-build 
 ifneq ("$(wildcard $(DB_FILE))","")
 	@echo "Ambuda using your existing database!"
 else
-	@docker --log-level ERROR compose -p ambuda-${AMBUDA_DEPLOYMENT_ENV} -f deploy/${AMBUDA_DEPLOYMENT_ENV}/docker-compose-dbsetup.yml up &> /dev/null
+	@docker ${DOCKER_LOG_LEVEL} compose -p ambuda-${AMBUDA_DEPLOYMENT_ENV} -f deploy/${AMBUDA_DEPLOYMENT_ENV}/docker-compose-dbsetup.yml up ${IO_REDIRECT}
 	@echo "Ambuda Database : ✔ "
 endif
 
@@ -114,31 +149,31 @@ endif
 docker-build: 
 	@echo "> Ambuda build is in progress. Expect it to take 2-5 minutes."
 	@printf "%0.s-" {1..21} && echo
-	@docker build -q -t ${AMBUDA_IMAGE} -t ${AMBUDA_IMAGE_LATEST} -f build/containers/Dockerfile.final ${PWD} > /dev/null
+	@docker build ${DOCKER_VEBOSITY} -t ${AMBUDA_IMAGE} -t ${AMBUDA_IMAGE_LATEST} -f build/containers/Dockerfile.final ${PWD} ${IO_REDIRECT}
 	@echo "Ambuda Image    : ✔ (${AMBUDA_IMAGE}, ${AMBUDA_IMAGE_LATEST})"
 
 # Start Docker services.
 docker-start: docker-build docker-setup-db
-	@docker --log-level ERROR compose -p ambuda-${AMBUDA_DEPLOYMENT_ENV} -f deploy/${AMBUDA_DEPLOYMENT_ENV}/docker-compose.yml up --detach &> /dev/null
+	@docker ${DOCKER_LOG_LEVEL} compose -p ambuda-${AMBUDA_DEPLOYMENT_ENV} -f deploy/${AMBUDA_DEPLOYMENT_ENV}/docker-compose.yml up ${DOCKER_DETACH} ${IO_REDIRECT}
 	@echo "Ambuda WebApp   : ✔ "
 	@echo "Ambuda URL      : http://${AMBUDA_HOST_IP}:${AMBUDA_HOST_PORT}"
 	@printf "%0.s-" {1..21} && echo
 	@echo 'To stop, run "make docker-stop".'
 
 # Stop docker services
 docker-stop: 
-	@docker --log-level ERROR compose -p ambuda-${AMBUDA_DEPLOYMENT_ENV} -f deploy/${AMBUDA_DEPLOYMENT_ENV}/docker-compose.yml stop
-	@docker --log-level ERROR compose -p ambuda-${AMBUDA_DEPLOYMENT_ENV} -f deploy/${AMBUDA_DEPLOYMENT_ENV}/docker-compose.yml rm
+	@docker ${DOCKER_LOG_LEVEL} compose -p ambuda-${AMBUDA_DEPLOYMENT_ENV} -f deploy/${AMBUDA_DEPLOYMENT_ENV}/docker-compose.yml stop
+	@docker ${DOCKER_LOG_LEVEL} compose -p ambuda-${AMBUDA_DEPLOYMENT_ENV} -f deploy/${AMBUDA_DEPLOYMENT_ENV}/docker-compose.yml rm
 	@echo "Ambuda URL stopped"
 
 # Show docker logs
 docker-logs: 
 	@docker compose -p ambuda-${AMBUDA_DEPLOYMENT_ENV} -f deploy/${AMBUDA_DEPLOYMENT_ENV}/docker-compose.yml logs
 
-# Run a local Celery instance for background tasks.
-celery: 
-	celery -A ambuda.tasks worker --loglevel=INFO
 
+
+# Lint commands
+# ===============================================
 # Check imports in Python code
 lint-isort:
 	@echo "Running Python isort to organize module imports"
@@ -163,6 +198,8 @@ lint-check: js-lint py-lint
 	black . --diff
 	@echo 'Lint completed'
 
+# Test, coverage and documentation commands
+# ===============================================
 # Run all Python unit tests.
 test: py-venv-check
 	pytest .

ambuda/seed/lookup/__init__.py

@@ -11,6 +11,7 @@ def run():
         create_bot_user.run()
     except Exception as ex:
         raise Exception(
-            "Error: Failed to create page statuses, " 
+            "Error: Failed to create page statuses, "
             "create roles, and creat bot user."
-            f"Error: {ex}")
+            f"Error: {ex}"
+        )

ambuda/seed/texts/gretil.py

@@ -116,11 +116,10 @@ def run():
         for spec in ALLOW:
             add_document(engine, spec)
     except Exception as ex:
-        raise Exception(
-            "Error: Failed to get latest from GRETIL. "
-            f"Error: {ex}")
+        raise Exception("Error: Failed to get latest from GRETIL. " f"Error: {ex}")
 
     log("Done.")
 
+
 if __name__ == "__main__":
     run()

ambuda/templates/about/people.html

@@ -57,7 +57,7 @@ <h1>{{ _('People') }}</h1>
 Ashwin has worked on a variety of projects around open source software
 development, digital humanities, cloud infrastructure and architecture, and
 cybersecurity. His interests in Sanskrit include Vedic texts and Vedantic
-commentaries. Ashwin holds a B.S. in Computer Science in Stanford University
+commentaries. Ashwin holds a B.S. in Computer Science from Stanford University
 and is currently pursuing a J.D. degree at Georgetown University Law Center,
 where he works on technology law and policy.
 {% endtrans %}</p>

ambuda/templates/proofing/projects/edit.html

@@ -15,12 +15,14 @@
 {{ m.project_nav(project=project, active='edit') }}
 
 {% set search_url = url_for("proofing.project.search", slug=project.slug)  %}
+{% set replace_url = url_for("proofing.project.replace", slug=project.slug)  %}
 {% set ocr_url = url_for("proofing.project.batch_ocr", slug=project.slug)  %}
 
 <div class="prose">
 
 <ul>
   <li><a href="{{ search_url }}">{{ _('Search the project') }}</a></li>
+  <li><a href="{{ replace_url }}">{{ _('Search & replace in the project') }}</a></li>
   <li><a href="{{ ocr_url }}">{{ _('Run batch OCR') }}</a></p>
 </ul>
 

ambuda/templates/proofing/projects/replace.html

@@ -0,0 +1,47 @@
+{% extends 'proofing/base.html' %}
+{% from "macros/forms.html" import field %}
+{% import "macros/proofing.html" as m %}
+
+{% block title %}Search and Replace {{ project.title }} | Ambuda{% endblock %}
+
+{% block content %}
+
+{{ m.project_header_nested('Search and Replace', project) }}
+{{ m.project_nav(project=project, active='edit') }}
+
+<div class="prose">
+  <p>Use this simple search and replace form to make edits across this project.</p>
+</div>
+
+<form method="GET" class="bg-slate-100 p-4 my-4">
+  {{ field(form.query) }}
+  {{ field(form.replace) }}
+  <input class="btn btn-submit" type="submit" value="Project-wide Search & Replace">
+</form>
+
+{% if query %}
+<div class="prose">
+
+{% macro sp(s, p, n) %}{% if n == 1 %}{{ s }}{% else %}{{ p }}{% endif %}{% endmacro %}
+
+{% set nr = results|length %}
+<p>Found {{ nr }} {{ sp("page", "pages", nr) }} that {{ sp("contains", "contain", nr) }} <kbd>{{ query }}</kbd>.</p>
+
+<ul>
+{% for page in results %}
+{% set page_url = url_for("proofing.page.edit", project_slug=project.slug, page_slug=page.slug) %}
+<li>
+    <a href="{{ page_url }}">{{ project.title }}/{{ page.slug }}</a>
+    <div class="p-2 border-l my-2">
+    {% for match in page.matches %}
+    <pre class="p-0.5">{{ match.query }}</pre>
+    <pre class="p-0.5">{{ match.update }}</pre>
+    {%- endfor %}
+    </div>
+</li>
+{% endfor %}
+</ul>
+
+</div>
+{% endif %}
+{% endblock %}