Add DB v6 support to grype-db-manager

config/grype-db-manager/include.d/validate.yaml

@@ -1,9 +1,23 @@
 # validate:
 
 listing:
-  image: "centos:8.2.2004"
-  minimum-packages: 85
-  minimum-vulnerabilities: 400
+  image: "alpine:3.9.2"
+  minimum-packages: 10
+  minimum-vulnerabilities: 90
+
+expected-providers:
+  - alpine
+  - amazon
+  - chainguard
+  - debian
+  - github
+  - mariner
+  - nvd
+  - oracle
+  - rhel
+  - sles
+  - ubuntu
+  - wolfi
 
 default-max-year: 2021
 gates:

manager/src/grype_db_manager/cli/config.py

@@ -81,7 +81,7 @@ class Validate:
     default_max_year: int = 2021
     gates: list[ValidateDB] = field(default_factory=list)
     listing: ValidateListing = field(default_factory=ValidateListing)
-
+    expected_providers: list[str] = field(default_factory=list)
 
 @dataclass()
 class ListingReplica:

manager/src/grype_db_manager/cli/db.py

@@ -8,7 +8,7 @@
 from yardstick.cli import config as ycfg
 from yardstick.cli.validate import validate as yardstick_validate
 
-from grype_db_manager import db, s3utils
+from grype_db_manager import db, grypedb, s3utils
 from grype_db_manager.cli import config, error
 from grype_db_manager.db.format import Format
 from grype_db_manager.grypedb import DB_DIR, DBManager, GrypeDB
@@ -54,6 +54,13 @@ def clear_dbs(cfg: config.Application) -> None:
         click.echo("no databases to clear")
 
 
+def remove_db(cfg: config.Application, db_uuid: str) -> None:
+    db_manager = DBManager(root_dir=cfg.data.root)
+    if db_manager.remove_db(db_uuid=db_uuid):
+        click.echo(f"database {db_uuid!r} removed")
+    click.echo(f"no database found with session id {db_uuid}")
+
+
 @group.command(name="build", help="build and validate a grype database")
 @click.option("--schema-version", "-s", required=True, help="the DB schema version to build")
 @click.pass_obj
@@ -96,7 +103,7 @@ def show_db(cfg: config.Application, db_uuid: str) -> None:
     "--skip-namespace-check",
     "skip_namespace_check",
     is_flag=True,
-    help="do not ensure the minimum expected namespaces are present",
+    help="do not ensure the minimum expected namespaces are present (for v6+ this is a providers-based check)",
 )
 @click.argument("db-uuid")
 @click.pass_obj
@@ -120,13 +127,41 @@ def validate_db(
         return
 
     if not skip_namespace_check:
-        # ensure the minimum number of namespaces are present
-        db_manager.validate_namespaces(db_uuid=db_uuid)
+        if db_info.schema_version < 6:
+            # ensure the minimum number of namespaces are present
+            db_manager.validate_namespaces(db_uuid=db_uuid)
+        else:
+            # ensure the minimum number of namespaces are present
+            db_manager.validate_providers(db_uuid=db_uuid, expected=cfg.validate.expected_providers)
+
+    _validate_db(ctx, cfg, db_info, images, db_uuid, verbosity, recapture)
+
+    logging.info(f"validating latest.json {db_uuid}")
+
+    if db_info.schema_version >= 6:
+        _validate_latest(cfg, db_info.latest_path, db_info.archive_path)
+
+    click.echo(f"{Format.BOLD}{Format.OKGREEN}Validation passed{Format.RESET}")
 
+
+def _validate_db(
+    ctx: click.Context,
+    cfg: config.Application,
+    db_info: grypedb.DBInfo,
+    images: list[str],
+    db_uuid: str,
+    verbosity: int,
+    recapture: bool,
+) -> None:
     # resolve tool versions and install them
     yardstick.store.config.set_values(store_root=cfg.data.yardstick_root)
 
     grype_version = db.schema.grype_version(db_info.schema_version)
+    basis_grype_version = grype_version
+
+    if db_info.schema_version >= 6:
+        # TODO: we don't have any published v6 grype databases yet
+        basis_grype_version = db.schema.grype_version(5)
 
     result_sets = {}
     for idx, rs in enumerate(cfg.validate.gates):
@@ -153,18 +188,25 @@ def validate_db(
                         label="custom-db",
                         name="grype",
                         version=grype_version + f"+import-db={db_info.archive_path}",
+                        profile="v6",
                     ),
                     ycfg.Tool(
                         name="grype",
-                        version=grype_version,
+                        version=basis_grype_version,
                     ),
                 ],
             ),
         )
 
     yardstick_cfg = ycfg.Application(
         profiles=ycfg.Profiles(
-            data={},
+            data={
+                "grype[custom-db]": {
+                    "v6": {
+                      "config_path": "./.grype-db-v6.yaml"
+                    },
+                },
+            },
         ),
         store_root=cfg.data.yardstick_root,
         default_max_year=cfg.validate.default_max_year,
@@ -193,6 +235,31 @@ def validate_db(
     )
 
 
+def _validate_latest(cfg: config.Application, latest_file: str, archive_path: str) -> None:
+    with open(latest_file) as f:
+        latest_obj = db.Latest.from_json(f.read())
+
+    if not cfg.validate.listing.image:
+        msg = "no image specified to validate against"
+        raise ValueError(msg)
+
+    if not cfg.validate.listing.minimum_packages:
+        msg = "minimum packages must be specified"
+        raise ValueError(msg)
+
+    if not cfg.validate.listing.minimum_vulnerabilities:
+        msg = "minimum vulnerabilities must be specified"
+        raise ValueError(msg)
+
+    db.latest.smoke_test(
+        latest_obj,
+        archive_path,
+        image=cfg.validate.listing.image,
+        minimum_packages=cfg.validate.listing.minimum_packages,
+        minimum_vulnerabilities=cfg.validate.listing.minimum_vulnerabilities,
+    )
+
+
 @group.command(name="upload", help="upload a grype database")
 @click.option("--ttl-seconds", "-t", default=DEFAULT_TTL_SECONDS, help="the TTL for the uploaded DB (should be relatively high)")
 @click.argument("db-uuid")
@@ -208,22 +275,35 @@ def upload_db(cfg: config.Application, db_uuid: str, ttl_seconds: int) -> None:
     db_manager = DBManager(root_dir=cfg.data.root)
     db_info = db_manager.get_db_info(db_uuid=db_uuid)
 
-    key = f"{s3_path}/{os.path.basename(db_info.archive_path)}"
+    if db_info.schema_version >= 6:
+        if not os.path.exists(db_info.archive_path):
+            msg = f"latest.json file not found for DB {db_uuid!r}"
+            raise ValueError(msg)
+
+        # /databases -> /databases/v6 , and is dynamic based on the schema version
+        s3_path = f"{s3_path}/v{db_info.schema_version}"
 
-    # TODO: we have folks that require legacy behavior, where the content type was application/x-tar
-    kwargs = {}
-    if db_info.archive_path.endswith(".tar.gz"):
-        kwargs["ContentType"] = "application/x-tar"
+    db_key = f"{s3_path}/{os.path.basename(db_info.archive_path)}"
+    latest_key = f"{s3_path}/latest.json"
 
     s3utils.upload_file(
         bucket=s3_bucket,
-        key=key,
+        key=db_key,
         path=db_info.archive_path,
         CacheControl=f"public,max-age={ttl_seconds}",
-        **kwargs,
     )
 
-    click.echo(f"DB {db_uuid!r} uploaded to s3://{s3_bucket}/{s3_path}")
+    click.echo(f"DB archive {db_uuid!r} uploaded to s3://{s3_bucket}/{s3_path}")
+
+    if db_info.schema_version >= 6:
+        s3utils.upload_file(
+            bucket=s3_bucket,
+            key=latest_key,
+            path=db_info.latest_path,
+            CacheControl="public,max-age=300",  # 5 minutes
+        )
+
+        click.echo(f"DB latest.json {db_uuid!r} uploaded to s3://{s3_bucket}/{s3_path}")
 
 
 @group.command(name="build-and-upload", help="upload a grype database")
@@ -234,7 +314,7 @@ def upload_db(cfg: config.Application, db_uuid: str, ttl_seconds: int) -> None:
     "--skip-namespace-check",
     "skip_namespace_check",
     is_flag=True,
-    help="do not ensure the minimum expected namespaces are present",
+    help="do not ensure the minimum expected namespaces are present (for v6+ this is a providers-based check)",
 )
 @click.option("--verbose", "-v", "verbosity", count=True, help="show details of all comparisons")
 @click.pass_obj

manager/src/grype_db_manager/cli/listing.py

@@ -9,7 +9,7 @@
 from grype_db_manager.db.format import Format
 
 
-@click.group(name="listing", help="manage the grype-db listing file")
+@click.group(name="listing", help="manage the grype-db listing file (only schemas v1-v5)")
 @click.pass_obj
 def group(_: config.Application) -> None:
     pass
@@ -110,7 +110,7 @@ def validate_listing(cfg: config.Application, listing_file: str) -> None:
         raise ValueError(msg)
 
     if cfg.validate.listing.override_grype_version and not cfg.validate.listing.override_db_schema_version:
-        msg = "ovrerride db schema version must be specified if override grype version is specified"
+        msg = "override db schema version must be specified if override grype version is specified"
         raise ValueError(msg)
 
     override_schema_release = None

manager/src/grype_db_manager/data/schema-info.json

@@ -24,6 +24,11 @@
       "schema": "5",
       "grype-version": "main",
       "supported": true
+    },
+    {
+      "schema": "6",
+      "grype-version": "feat/v6-query-api",
+      "supported": false
     }
   ]
 }

manager/src/grype_db_manager/db/__init__.py

@@ -1,11 +1,14 @@
 from . import listing, metadata, schema
+from .latest import Latest
 from .listing import Listing
 from .metadata import Metadata
 from .validation import capture_results
 
 __all__ = [
+    "Latest",
     "Listing",
     "Metadata",
+    "latest",
     "listing",
     "metadata",
     "schema",