Adds prototype registry storage (#168)

- Adds storage implementation
- Adds CI workflow for registry
- Addresses code review

.github/workflows/ci.yml → .github/workflows/cli.yml

@@ -1,4 +1,4 @@
-name: CI
+name: Buffrs CLI
 
 on:
   push:

.github/workflows/registry.yml

@@ -0,0 +1,72 @@
+name: Buffrs Registry
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+
+env:
+  MINIMUM_LINE_COVERAGE_PERCENT: 5
+
+jobs:
+  fmt:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - run: rustup update && rustup component add rustfmt
+      - run: cd registry && cargo fmt --check --all
+
+  clippy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - run: rustup update && rustup component add clippy
+      - uses: Swatinem/rust-cache@v2
+      - run: cd registry && cargo clippy --all-targets --workspace -- -D warnings -D clippy::all
+
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          lfs: "true"
+      - uses: isbang/[email protected]
+        with:
+          compose-file: "./registry/docker-compose.yml"
+      - run: rustup update
+      - uses: Swatinem/rust-cache@v2
+      - run: cd registry && cargo test --workspace
+        env:
+          RUST_BACKTRACE: 1
+
+  deny:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - run: rustup update
+      - uses: Swatinem/rust-cache@v2
+      - run: cargo install cargo-deny || true
+      - run: cd registry && cargo deny --workspace check
+
+  coverage:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          lfs: 'true'
+      - uses: isbang/[email protected]
+        with:
+          compose-file: "./registry/docker-compose.yml"
+      - run: rustup update
+      - run: rustup component add llvm-tools-preview
+      - uses: Swatinem/rust-cache@v2
+      - run: cargo install cargo-llvm-cov || true
+      - run: cd registry && cargo llvm-cov --workspace --fail-under-lines "$MINIMUM_LINE_COVERAGE_PERCENT"
+
+  typos:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: Swatinem/rust-cache@v2
+      - run: cargo install typos-cli || true
+      - run: cd registry && typos

registry/Cargo.toml

@@ -4,18 +4,40 @@ version = "0.1.0"
 edition = "2021"
 description = "Registry for buffrs, a modern protocol buffer package manager"
 license = "Apache-2.0"
+default-run = "buffrs-registry"
 
 [dependencies]
-buffrs = { path = "../", version = "0.6.4" }
-prost = "0.12.1"
-tonic = "0.10.2"
+async-trait = "0.1.74"
+aws-config = { version = "0.56.1", optional = true }
+aws-sdk-s3 = { version = "0.34.0", optional = true }
+axum_tonic = "0.1.0"
+buffrs = { path = "../", version = "0.7.1" }
+bytes = "1.5.0"
+chrono = "0.4.31"
 clap = { version = "4.3", features = ["cargo", "derive", "env"] }
+eyre = "0.6.8"
+miette = "5.10.0"
+prost = "0.12.1"
+semver = "1.0.20"
+sqlx = { version = "0.7.2", features = ["runtime-tokio", "postgres", "macros", "migrate"] }
+ssh-key = { version = "0.6.2", features = ["ed25519"] }
+thiserror = "1.0.49"
 tokio = { version = "1.33.0", features = ["full"] }
+tonic = "0.10.2"
 tracing = "0.1.37"
 tracing-subscriber = "0.3.17"
-sqlx = { version = "0.7.2", features = ["runtime-tokio", "postgres", "macros", "migrate"] }
-eyre = "0.6.8"
 url = "2.4.1"
 
 [build-dependencies]
-buffrs = { path = "../", version = "0.7.0" }
+buffrs = { path = "../", version = "0.7.1" }
+
+[features]
+default = ["storage-s3"]
+storage-s3 = ["dep:aws-config", "dep:aws-sdk-s3"]
+
+[dev-dependencies]
+aws-credential-types = { version = "0.56.1", features = ["hardcoded-credentials"] }
+proptest = "1.3.1"
+rand = "0.8.5"
+tempfile = "3.8.1"
+test-strategy = "0.3.1"

registry/Proto.toml

@@ -1,4 +1,4 @@
 [package]
 type = "api"
-name = "buffrs-registry"
+name = "buffrs"
 version = "0.0.1"

registry/deny.toml

@@ -0,0 +1,13 @@
+[licenses]
+unlicensed = "deny"
+allow = ["Apache-2.0", "BSD-3-Clause", "MIT", "Unicode-DFS-2016", "ISC"]
+default = "deny"
+
+[[licenses.clarify]]
+name = "ring"
+expression = "ISC"
+license-files = [{ path = "LICENSE", hash = 0xbd0eed23 }]
+
+[bans]
+multiple-versions = "warn"
+wildcards = "deny"

registry/docker-compose.yml

@@ -7,3 +7,12 @@ services:
       POSTGRES_USER: buffrs
     ports:
       - 127.0.0.1:5432:5432
+  storage:
+    image: minio/minio
+    command: server /data
+    environment:
+      MINIO_ROOT_USER: buffrs
+      MINIO_ROOT_PASSWORD: password
+      MINIO_DOMAIN: localhost
+    ports:
+      - 127.0.0.1:9000:9000

registry/migrations/20230828111733_users.up.sql

@@ -1,18 +1,37 @@
-CREATE TABLE users (
-    id            SERIAL PRIMARY KEY,
-    -- metadata
-    name          TEXT,
-    email         TEXT,
-    avatar        TEXT,
-    -- static user token here
-    token         TEXT NOT NULL UNIQUE,
-    -- timestamps
-    created_at TIMESTAMPTZ NOT NULL,
-    updated_at TIMESTAMPTZ NOT NULL,
-    deleted_at TIMESTAMPTZ
+-- table of users.
+CREATE TABLE "users" (
+    "id"          SERIAL PRIMARY KEY,
+    "handle"      TEXT NOT NULL UNIQUE,
+    "created_at"  TIMESTAMPTZ NOT NULL DEFAULT (now()),
+    "updated_at"  TIMESTAMPTZ NOT NULL DEFAULT (now()),
+    -- when users are deleted, we keep the row, but set the deleted_at field.
+    "deleted_at"  TIMESTAMPTZ
 );
 
-CREATE TABLE user_tokens (
-    user_id       INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
-    token         TEXT NOT NULL UNIQUE
+CREATE TABLE "user_tokens" (
+    "user"           INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+    "prefix"         TEXT NOT NULL UNIQUE,
+    "hash"           TEXT NOT NULL UNIQUE,
+    "allow_publish"  BOOLEAN NOT NULL DEFAULT (false),
+    "allow_update"   BOOLEAN NOT NULL DEFAULT (false),
+    "allow_yank"     BOOLEAN NOT NULL DEFAULT (false),
+    "created_at"     TIMESTAMPTZ NOT NULL DEFAULT (now()),
+    "expires_at"     TIMESTAMPTZ NOT NULL,
+    "deleted_at"     TIMESTAMPTZ
 );
+
+-- view showing only active users
+CREATE VIEW "users_active" AS
+    SELECT *
+    FROM users
+    WHERE deleted_at IS NULL;
+
+-- view showing only active tokens
+CREATE VIEW "user_tokens_active" AS
+    SELECT
+        tokens.*,
+        users.handle
+    FROM users_active users
+    JOIN user_tokens tokens on tokens.user = users.id
+    WHERE tokens.expires_at > now()
+    AND tokens.deleted_at IS NULL;

registry/migrations/20230828111844_packages.up.sql

@@ -20,3 +20,15 @@ CREATE TABLE package_owners (
     created_at TIMESTAMPTZ NOT NULL,
     deleted_at TIMESTAMPTZ
 );
+
+CREATE TABLE package_invites (
+    id SERIAL PRIMARY KEY,
+    token TEXT NOT NULL UNIQUE,
+
+    user_id    INTEGER NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
+    package_id INTEGER NOT NULL REFERENCES packages(id) ON DELETE RESTRICT,
+    created_by INTEGER NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
+
+    created_at TIMESTAMPTZ NOT NULL,
+    expires_at TIMESTAMPTZ NOT NULL
+);

registry/proto/registry.proto

@@ -14,6 +14,10 @@ service Registry {
     // Download a package
     rpc Download(DownloadRequest)
       returns (DownloadResponse);
+
+    // Get versions for a package
+    rpc Versions(VersionsRequest)
+        returns (VersionsResponse);
 }
 
 message PublishRequest {
@@ -29,3 +33,12 @@ message DownloadRequest {
 message DownloadResponse {
     buffrs.package.Compressed package = 1;
 }
+
+message VersionsRequest {
+    string name = 1;
+    string requirement = 2;
+}
+
+message VersionsResponse {
+    repeated string version = 1;
+}

registry/proto/version.proto

@@ -0,0 +1,15 @@
+
+syntax = "proto3";
+
+package buffrs.version;
+
+message VersionCore {
+    uint32 major = 1;
+    uint32 minor = 2;
+    uint32 patch = 3;
+}
+
+message SemanticVersion {
+    VersionCore version = 1;
+    optional string prerelease = 2;
+}

registry/src/storage/mod.rs → registry/src/api.rs

@@ -11,3 +11,16 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
+//! # Buffrs Registry API
+
+use crate::context::Context;
+
+mod grpc;
+
+impl Context {
+    /// Launch API
+    pub async fn launch_api(&self) {
+        todo!()
+    }
+}

registry/src/api/grpc.rs

@@ -0,0 +1,47 @@
+// Copyright 2023 Helsing GmbH
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::{
+    context::Context,
+    proto::buffrs::registry::{
+        registry_server::Registry, DownloadRequest, DownloadResponse, PublishRequest,
+        PublishResponse, VersionsRequest, VersionsResponse,
+    },
+};
+use async_trait::async_trait;
+use tonic::{Request, Response, Status};
+
+#[async_trait]
+impl Registry for Context {
+    async fn publish(
+        &self,
+        _request: Request<PublishRequest>,
+    ) -> Result<Response<PublishResponse>, Status> {
+        todo!()
+    }
+
+    async fn download(
+        &self,
+        _request: Request<DownloadRequest>,
+    ) -> Result<Response<DownloadResponse>, Status> {
+        todo!()
+    }
+
+    async fn versions(
+        &self,
+        _request: Request<VersionsRequest>,
+    ) -> Result<Response<VersionsResponse>, Status> {
+        todo!()
+    }
+}