Merge branch 'main' into neon-pipeline-refactor

.env.example

@@ -37,4 +37,5 @@ NEON_API_TOKEN=y
 NEON_API_BASE_URL=https://data.neonscience.org/api/v0
 
 NERSC_USERNAME=replaceme
-ORCID_CLIENT_ID=replaceme
+ORCID_CLIENT_ID=replaceme
+ORCID_CLIENT_SECRET=replaceme

.github/workflows/lint.yml

@@ -0,0 +1,33 @@
+name: Lint-check + Style-normalize Python files
+
+on:
+  pull_request:
+    paths:
+      - '.github/workflows/lint.yml'
+      - '**.py'
+
+
+jobs:
+  build:
+    name:  lint-check and style-normalize Python files
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.ref }}
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+    - name: Lint with flake8 and Reformat with black
+      run: |
+        make init-lint-and-black
+        make lint
+        make black
+    - name: commit and push if reformatted
+      run: |
+        git config user.name github-actions
+        git config user.email [email protected]
+        if git status --short | grep -q '\.py$'; then git add '*.py' && git commit -m "style: reformat" && git push; fi
+        

.gitpod.yml

@@ -9,4 +9,4 @@ tasks:
   - name: Start Dev on Fresh Gitpod
     before: cp .env.example .env
     init: docker compose up mongo --detach && make mongorestore-nmdc-dev
-    command: make up-dev && docker-compose logs -f fastapi
+    command: make up-dev && docker compose logs -f fastapi

Makefile

@@ -18,14 +18,14 @@ update-deps:
 update: update-deps init
 
 up-dev:
-	docker-compose up --build --force-recreate --detach --remove-orphans
+	docker compose up --build --force-recreate --detach --remove-orphans
 
 dev-reset-db:
 	docker compose \
 		exec mongo /bin/bash -c "./app_tests/mongorestore-nmdc-testdb.sh"
 
 up-test:
-	docker-compose --file docker-compose.test.yml \
+	docker compose --file docker-compose.test.yml \
 		up --build --force-recreate --detach --remove-orphans
 
 test-build:
@@ -41,21 +41,31 @@ test-run:
 
 test: test-build test-run
 
+black:
+	black nmdc_runtime
+
 lint:
 	# Python syntax errors or undefined names
 	flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --extend-ignore=F722
 	# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
 	flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 \
 		--statistics --extend-exclude="./build/" --extend-ignore=F722
 
+PIP_PINNED_FLAKE8 := $(shell grep 'flake8==' requirements/dev.txt)
+PIP_PINNED_BLACK := $(shell grep 'black==' requirements/dev.txt)
+
+init-lint-and-black:
+	pip install $(PIP_PINNED_FLAKE8)
+	pip install $(PIP_PINNED_BLACK)
+
 down-dev:
-	docker-compose down
+	docker compose down
 
 down-test:
-	docker-compose --file docker-compose.test.yml down
+	docker compose --file docker-compose.test.yml down
 
 follow-fastapi:
-	docker-compose logs fastapi -f
+	docker compose logs fastapi -f
 
 fastapi-deploy-spin:
 	rancher kubectl rollout restart deployment/runtime-fastapi --namespace=nmdc-dev

nmdc_runtime/api/core/auth.py

@@ -6,17 +6,25 @@
 from fastapi.exceptions import HTTPException
 from fastapi.openapi.models import OAuthFlows as OAuthFlowsModel
 from fastapi.param_functions import Form
-from fastapi.security import OAuth2, HTTPBasic, HTTPBasicCredentials
+from fastapi.security import (
+    OAuth2,
+    HTTPBasic,
+    HTTPBasicCredentials,
+    HTTPBearer,
+    HTTPAuthorizationCredentials,
+)
 from fastapi.security.utils import get_authorization_scheme_param
 from jose import JWTError, jwt
 from passlib.context import CryptContext
 from pydantic import BaseModel
+from starlette import status
 from starlette.requests import Request
 from starlette.status import HTTP_400_BAD_REQUEST, HTTP_401_UNAUTHORIZED
 
 SECRET_KEY = os.getenv("JWT_SECRET_KEY")
 ALGORITHM = "HS256"
 ORCID_CLIENT_ID = os.getenv("ORCID_CLIENT_ID")
+ORCID_CLIENT_SECRET = os.getenv("ORCID_CLIENT_SECRET")
 
 # https://orcid.org/.well-known/openid-configuration
 # XXX do we want to live-load this?
@@ -129,30 +137,43 @@ async def __call__(self, request: Request) -> Optional[str]:
     tokenUrl="token", auto_error=False
 )
 
+bearer_scheme = HTTPBearer(scheme_name="bearerAuth", auto_error=False)
+
 
 async def basic_credentials(req: Request):
     return await HTTPBasic(auto_error=False)(req)
 
 
+async def bearer_credentials(req: Request):
+    return await HTTPBearer(scheme_name="bearerAuth", auto_error=False)(req)
+
+
 class OAuth2PasswordOrClientCredentialsRequestForm:
     def __init__(
         self,
         basic_creds: Optional[HTTPBasicCredentials] = Depends(basic_credentials),
+        bearer_creds: Optional[HTTPAuthorizationCredentials] = Depends(
+            bearer_credentials
+        ),
         grant_type: str = Form(None, regex="^password$|^client_credentials$"),
         username: Optional[str] = Form(None),
         password: Optional[str] = Form(None),
         scope: str = Form(""),
         client_id: Optional[str] = Form(None),
         client_secret: Optional[str] = Form(None),
     ):
-        if grant_type == "password" and (username is None or password is None):
+        if bearer_creds:
+            self.grant_type = "client_credentials"
+            self.username, self.password = None, None
+            self.scopes = scope.split()
+            self.client_id = bearer_creds.credentials
+            self.client_secret = None
+        elif grant_type == "password" and (username is None or password is None):
             raise HTTPException(
                 status_code=HTTP_400_BAD_REQUEST,
                 detail="grant_type password requires username and password",
             )
-        if grant_type == "client_credentials" and (
-            client_id is None or client_secret is None
-        ):
+        elif grant_type == "client_credentials" and (client_id is None):
             if basic_creds:
                 client_id = basic_creds.username
                 client_secret = basic_creds.password

nmdc_runtime/api/endpoints/metadata.py

@@ -239,6 +239,11 @@ async def submit_json_nmdcdb(
     Submit a NMDC JSON Schema "nmdc:Database" object.
 
     """
+    if not permitted(user.username, "/metadata/json:submit"):
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="Only specific users are allowed to submit json at this time.",
+        )
     rv = validate_json(docs, mdb)
     if rv["result"] == "errors":
         raise HTTPException(

nmdc_runtime/api/endpoints/queries.py

@@ -34,7 +34,7 @@ def check_can_update_and_delete(user: User):
     if not permitted(user.username, "/queries:run(query_cmd:DeleteCommand)"):
         raise HTTPException(
             status_code=status.HTTP_403_FORBIDDEN,
-            detail="Only specific users are allowed to issue update and delete commands."
+            detail="Only specific users are allowed to issue update and delete commands.",
         )
 
 
@@ -125,7 +125,8 @@ def _run_query(query, mdb) -> CommandResponse:
                 detail="Can only delete documents in nmdc-schema collections.",
             )
         delete_specs = [
-            {"filter": del_statement.q, "limit": del_statement.limit} for del_statement in query.cmd.deletes
+            {"filter": del_statement.q, "limit": del_statement.limit}
+            for del_statement in query.cmd.deletes
         ]
         for spec in delete_specs:
             docs = list(mdb[collection_name].find(**spec))
@@ -148,7 +149,8 @@ def _run_query(query, mdb) -> CommandResponse:
                 detail="Can only update documents in nmdc-schema collections.",
             )
         update_specs = [
-            {"filter": up_statement.q, "limit": 0 if up_statement.multi else 1} for up_statement in query.cmd.updates
+            {"filter": up_statement.q, "limit": 0 if up_statement.multi else 1}
+            for up_statement in query.cmd.updates
         ]
         for spec in update_specs:
             docs = list(mdb[collection_name].find(**spec))

nmdc_runtime/api/endpoints/users.py

@@ -2,7 +2,9 @@
 from datetime import timedelta
 
 import pymongo.database
+import requests
 from fastapi import Depends, APIRouter, HTTPException, status
+from fastapi.openapi.docs import get_swagger_ui_html
 from jose import jws, JWTError
 from starlette.requests import Request
 from starlette.responses import HTMLResponse, RedirectResponse
@@ -16,6 +18,7 @@
     ORCID_JWK,
     ORCID_JWS_VERITY_ALGORITHM,
     credentials_exception,
+    ORCID_CLIENT_SECRET,
 )
 from nmdc_runtime.api.core.auth import get_password_hash
 from nmdc_runtime.api.core.util import generate_secret
@@ -32,43 +35,28 @@
 router = APIRouter()
 
 
-@router.get("/orcid_authorize")
-async def orcid_authorize():
-    """NOTE: You want to load /orcid_authorize directly in your web browser to initiate the login redirect flow."""
-    return RedirectResponse(
-        f"https://orcid.org/oauth/authorize?client_id={ORCID_CLIENT_ID}"
-        "&response_type=token&scope=openid&"
-        f"redirect_uri={BASE_URL_EXTERNAL}/orcid_token"
-    )
-
-
-@router.get("/orcid_token")
-async def redirect_uri_for_orcid_token(req: Request):
-    """
-    Returns a web page that will display a user's orcid jwt token for copy/paste.
-
-    This route is loaded by orcid.org after a successful orcid user login.
-    """
-    return HTMLResponse(
-        """
-    <head>
-    <script>
-        function getFragmentParameterByName(name) {
-            name = name.replace(/[\[]/, "\\[").replace(/[\]]/, "\\]");
-            var regex = new RegExp("[\\#&]" + name + "=([^&#]*)"),
-            results = regex.exec(window.location.hash);
-            return results === null ? "" : decodeURIComponent(results[1].replace(/\+/g, " "));
-        }
-    </script>
-    </head>
-    <body>
-    <main id="token"></main>
-    </body>
-    <script>
-    document.getElementById("token").innerHTML = getFragmentParameterByName("id_token")
-    </script>
-    """
+@router.get("/orcid_code", response_class=RedirectResponse)
+async def receive_orcid_code(request: Request, code: str, state: str | None = None):
+    rv = requests.post(
+        "https://orcid.org/oauth/token",
+        data=(
+            f"client_id={ORCID_CLIENT_ID}&client_secret={ORCID_CLIENT_SECRET}&"
+            f"grant_type=authorization_code&code={code}&redirect_uri={BASE_URL_EXTERNAL}/orcid_code"
+        ),
+        headers={
+            "Content-type": "application/x-www-form-urlencoded",
+            "Accept": "application/json",
+        },
     )
+    token_response = rv.json()
+    response = RedirectResponse(state or request.url_for("custom_swagger_ui_html"))
+    for key in ["user_orcid", "user_name", "user_id_token"]:
+        response.set_cookie(
+            key=key,
+            value=token_response[key.replace("user_", "")],
+            max_age=2592000,
+        )
+    return response
 
 
 @router.post("/token", response_model=Token)

nmdc_runtime/api/endpoints/util.py

@@ -110,20 +110,23 @@ def list_resources(req: ListRequest, mdb: MongoDatabase, collection_name: str):
         }
         return rv
     else:
+        # the below block committed in anger. nmdc schema collections should have an 'id' field.
+        id_field = "id"
         if "id_1" not in mdb[collection_name].index_information():
             logging.warning(
                 f"list_resources: no index set on 'id' for collection {collection_name}"
             )
+            id_field = "_id"  # expected atm for functional_annotation_agg
         resources = list(
             mdb[collection_name].find(
                 filter=filter_,
                 projection=projection,
                 limit=limit,
-                sort=[("id", 1)],
+                sort=[(id_field, 1)],
                 allow_disk_use=True,
             )
         )
-        last_id = resources[-1]["id"]
+        last_id = resources[-1][id_field]
         token = generate_one_id(mdb, "page_tokens")
         mdb.page_tokens.insert_one(
             {"_id": token, "ns": collection_name, "last_id": last_id}