feat: DB driver and cache

.gitignore

@@ -18,8 +18,6 @@ bin
 go.work.sum
 
 # Config
-config.yaml
-config.*.yaml
 .config.yaml
 .config.test.yaml
 
@@ -32,3 +30,7 @@ config.*.yaml
 .env
 .idea/
 .mise.local.toml
+
+# tmp dir
+tmp
+tmp/*

Makefile

@@ -15,15 +15,15 @@ help: ## Prints all the targets in all the Makefiles
 
 .PHONY: path_up
 path_up: ## Run docker compose up
-	docker compose up -d
+	docker compose -f ./docker/docker-compose.yml up -d
 
 .PHONY: path_up_build
 path_up_build: ## Run docker compose up with build
-	docker compose up -d --build
+	docker compose -f ./docker/docker-compose.yml up -d --build
 
 .PHONY: path_down
 path_down: ## Run docker compose down
-	docker compose down
+	docker compose -f ./docker/docker-compose.yml down
 
 #########################
 ### Test Make Targets ###
@@ -59,3 +59,12 @@ copy_test_config: ## copies the example test configuration yaml file to .gitigno
 	else \
 		echo ".config.test.yaml already exists, not overwriting."; \
 	fi
+
+#######################
+### SQLC Generation ###
+#######################
+
+.PHONY: sqlc_generate
+sqlc_generate: ## Generate SQLC code from db/driver/sqlc/*.sql files
+	sqlc generate -f ./db/driver/sqlc/sqlc.yaml
+

README.md

@@ -5,26 +5,38 @@
 </div>
 <br/>
 
-# Table of Contents <!-- omit in toc -->
-
-- [Introduction](#introduction)
-- [Quickstart (Shannon)](#quickstart-shannon)
-- [Configuration](#configuration)
-  - [Configuration File](#configuration-file)
-    - [Example Configuration Format](#example-configuration-format)
-- [Running PATH](#running-path)
-  - [Setup Config YAML](#setup-config-yaml)
-  - [Start the Container](#start-the-container)
-- [E2E Tests](#e2e-tests)
-  - [Running Tests](#running-tests)
-
-## Introduction
+- [1. Introduction](#1-introduction)
+  - [1.1 Prerequisites](#11-prerequisites)
+- [2. Quickstart (Shannon)](#2-quickstart-shannon)
+- [3. Configuration](#3-configuration)
+  - [3.1 Configuration File](#31-configuration-file)
+  - [3.2 Example Configuration Format](#32-example-configuration-format)
+- [4. Running PATH](#4-running-path)
+  - [4.1 Setup Config YAML](#41-setup-config-yaml)
+  - [4.2 Start the Container](#42-start-the-container)
+- [5. E2E Tests](#5-e2e-tests)
+  - [5.1 Running Tests](#51-running-tests)
+- [6. User Data](#6-user-data)
+  - [6.1 Updated Endpoint](#61-updated-endpoint)
+  - [6.2 Database Configuration](#62-database-configuration)
+  - [6.3 Database Schema](#63-database-schema)
+
+## 1. Introduction
 
 **PATH** (Path API & Toolkit Harness) is an open source framework for enabling access to a decentralized supply network.
 
 It provides various tools and libraries to streamline the integration and interaction with decentralized protocols.
 
-## Quickstart (Shannon)
+### 1.1 Prerequisites
+
+- [Docker](https://docs.docker.com/get-docker/)
+
+**Required For Development:**
+
+- [SQLC](https://docs.sqlc.dev/)
+- [Mockgen](https://github.com/uber-go/mock)
+
+## 2. Quickstart (Shannon)
 
 1. Stake Apps and Gateway
 
@@ -54,9 +66,9 @@ It provides various tools and libraries to streamline the integration and intera
 
    _For detailed instructions on running PATH, see the [Running PATH](#running-path) section._
 
-## Configuration
+## 3. Configuration
 
-### Configuration File
+### 3.1 Configuration File
 
 The configuration for PATH is defined in a YAML file, which should be named `.config.yaml`.
 
@@ -91,7 +103,7 @@ The configuration is divided into several sections:
    - _Required only if the gateway operator wishes to associate user data with requests._
    - Configures the PostgreSQL database connection string.
 
-#### Example Configuration Format
+### 3.2 Example Configuration Format
 
 ```yaml
 shannon_config:
@@ -115,9 +127,9 @@ services:
   - [Shannon](./cmd/config/testdata/shannon.example.yaml)
 - [Config YAML Schema](./config/config.schema.yaml)
 
-## Running PATH
+## 4. Running PATH
 
-#### Setup Config YAML
+### 4.1 Setup Config YAML
 
 - The PATH service requires the config YAML file to be populated.
 
@@ -133,7 +145,7 @@ services:
 
    **⚠️ IMPORTANT: The data required to populate the `.config.yaml` file is sensitive and the contents of this file must never be shared outside of your organization. ⚠️**
 
-#### Start the Container
+### 4.2 Start the Container
 
 1. Once the `.config.yaml` file is populated, to start the PATH service for a specific protocol, use the `make` target:
 
@@ -153,7 +165,7 @@ services:
    make path_down
    ```
 
-## E2E Tests
+## 5. E2E Tests
 
 This repository contains end-to-end (E2E) tests for the Shannon relay protocol. The tests ensure that the protocol behaves as expected under various conditions.
 
@@ -171,7 +183,7 @@ Currently, the E2E tests are configured to run against the Shannon testnet.
 
 Future work will include adding support for other protocols.
 
-#### Running Tests
+### 5.1 Running Tests
 
 To run the tests, use the following `make` targets:
 
@@ -185,3 +197,74 @@ make test_unit
 # Shannon E2E test only
 make test_e2e_shannon_relay
 ```
+
+## 6. User Data
+
+By default, PATH does not associate user data with service requests (i.e. relays).
+
+You may opt to enable user data config to unlock the ability to associate a specific user with a particular service request.
+
+This is required for:
+
+- User-specified app settings (e.g. api auth keys, etc.)
+- Metering and billing of service requests (e.g. charging users $2 per 1 million requests)
+- Rate limiting of service requests by throughput (e.g. 30 req / second) and/or capacity (e.g. 1M req / month)
+
+### 6.1 Updated Endpoint
+
+Enabling user data will modify the endpoint for service requests to require a gateway endpoint ID at the end of the URL path.
+
+For example:
+
+```bash
+http://eth-mainnet.localhost:3000/v1/{gateway_endpoint_id}
+```
+
+The default endpoint of `/v1` will no longer function without a gateway endpoint ID.
+
+### 6.2 Database Configuration
+
+To enable user data, you must set up a Postgres database and populate the `.config.yaml` file's `user_data_config` field with the connection string.
+
+```yaml
+user_data_config:
+  postgres_connection_string: "postgres://user:password@localhost:5432/database"
+```
+
+An example Postgres Docker configuration is included in the [docker-compose.yml](./docker/docker-compose.yml) file at the root of this repository. **However, this configuration is not recommended for production use.**
+
+### 6.3 Database Schema
+
+[Base Schema SQL File](./db/driver/sqlc/schema.sql)
+
+```mermaid
+erDiagram
+    PLANS {
+        int id
+        varchar type
+        int rate_limit_throughput
+        int rate_limit_capacity
+        enum rate_limit_capacity_period
+    }
+
+    USER_ACCOUNTS {
+        varchar id
+        varchar plan_type
+    }
+
+    GATEWAY_ENDPOINTS {
+        varchar id
+        varchar account_id
+        varchar api_key
+        boolean api_key_required
+    }
+
+    PLANS ||--o{ USER_ACCOUNTS : "plan_type"
+    USER_ACCOUNTS ||--o{ GATEWAY_ENDPOINTS : "account_id"
+```
+
+A base schema is provided with the minimal tables and columns required to enable user data handling in PATH.
+
+These tables should not be modified; instead, any additional functionality required by the gateway operator for managing user data should be added by extending the base tables and columns provided in this schema.
+
+For example, it is up to the gateway operator to decide how they wish to manage their gateway's user data, user accounts, subscription plans, authentication, etc.

config/config.go

@@ -28,7 +28,9 @@ type (
 
 		Services map[relayer.ServiceID]ServiceConfig `yaml:"services"`
 		Router   RouterConfig                        `yaml:"router_config"`
-		UserData UserDataConfig                      `yaml:"user_data_config"`
+		// UserDataConfig is optional and only used if user data handling is enabled
+		// for the gateway by setting the 'user_data_config' field in the config YAML file.
+		UserData *UserDataConfig `yaml:"user_data_config"`
 
 		// A map from human readable aliases (e.g. eth-mainnet) to service ID (e.g. 0021)
 		serviceAliases map[string]relayer.ServiceID
@@ -55,6 +57,9 @@ func LoadGatewayConfigFromYAML(path string) (GatewayConfig, error) {
 	// hydrate required fields and set defaults for optional fields
 	config.hydrateServiceAliases()
 	config.hydrateRouterConfig()
+	if config.IsUserDataEnabled() {
+		config.hydrateUserDataConfig()
+	}
 
 	return config, config.validate()
 }
@@ -73,7 +78,12 @@ func (c GatewayConfig) GetRouterConfig() RouterConfig {
 	return c.Router
 }
 
-func (c GatewayConfig) GetUserDataConfig() UserDataConfig {
+// UserDataEnabled returns true if user data handling is enabled for the Gateway.
+func (c GatewayConfig) IsUserDataEnabled() bool {
+	return c.UserData != nil
+}
+
+func (c GatewayConfig) GetUserDataConfig() *UserDataConfig {
 	return c.UserData
 }
 
@@ -117,6 +127,10 @@ func (c *GatewayConfig) hydrateRouterConfig() {
 	c.Router.hydrateRouterDefaults()
 }
 
+func (c *GatewayConfig) hydrateUserDataConfig() {
+	c.UserData.hydrateDefaults()
+}
+
 /* --------------------------------- Gateway Config Validation Helpers -------------------------------- */
 
 func (c GatewayConfig) validate() error {
@@ -126,6 +140,11 @@ func (c GatewayConfig) validate() error {
 	if err := c.validateServiceConfig(); err != nil {
 		return err
 	}
+	if c.IsUserDataEnabled() {
+		if err := c.UserData.validate(); err != nil {
+			return err
+		}
+	}
 
 	return nil
 }