Validate against a remote schema and by version tag.

Provides the functionality to validate against a remote schema hosted on github by github reference (e.g. master HEAD, version tag, etc.). It also pulls out a "spec" as a class object to make it easier to pass around and abstract the differences between getting a spec from Github vs Local files. This PR also: - formats code per pre-commit - refactors apply_schema_to_df into smaller functions - updates errors and warning to be more helpful (pointing to files, etc) - adds tests for field constraints in constraint_tests.py - adds assert statements to basic_tests.py Closes #25 Closes #26 NOTE: Tests failing due to zephyr-data-specs/GMNS#70
e-lo · Jun 14, 2023 · 2918316 · 2918316
2 parents c18d13a + 13fc5b1
commit 2918316
Show file tree

Hide file tree

Showing 24 changed files with 923 additions and 381 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md
@@ -2,7 +2,7 @@
 name: Bug report
 about: Create a report to help us improve the project code
 title: '🐛  '
-labels: ['🐛 bug']
+labels: ['🐛bug']
 assignees: ''
 
 ---

diff --git a/docs/api.md b/docs/api.md
@@ -5,43 +5,80 @@
 Returns a dataframe that conforms to the specified schema and have been
 validated.
 
-```python
-df = gmnspy.in_out.read_gmns_csv(data_filename, schema_file=schemafilename)
-```
+=== "Specify Local Schema File"
+
+    ```python
+    df = gmnspy.in_out.read_gmns_csv(data_filename, schema_path=schemafilename)
+    ```
+
+=== "Use local spec and schema name"
+
+    ```python
+    spec = gmnspy.SpecConfig(gmnspy.defaults.LOCAL_SPEC)
+    df = gmnspy.in_out.read_gmns_csv(data_filename, spec=spec, schema_name = "link")
+    ```
+
+=== "Use canonical spec and infer schema from filename"
+
+    ```python
+    spec = gmnspy.SpecConfig()
+    df = gmnspy.in_out.read_gmns_csv(data_filename, spec=spec)
+    ```
 
 ## Read a network
 
 Returns a dictionary of dataframes that conform to the specified schema
-and have been validated.
+and have been validated. Also checks foreign keys between files.
+
+=== "Use local spec"
 
-Checks foreign keys between files.
+    ```python
+    net = gmnspy.in_out.read_gmns_network(data_directory, config_path: gmnspy.defaults.LOCAL_SPEC)
+    ```
 
-```python
-net = gmnspy.in_out.read_gmns_network(data_directory, config: "gmns.spec.json")
-```
+=== "Use canonical spec"
+
+    ```python
+    net = gmnspy.in_out.read_gmns_network(data_directory, official_version="master")
+    ```
 
 ## API
 
+::: gmnspy.schema.json_from_path
+
+::: gmnspy.schema.GithubFile
+
+
 ### Read/Write
 
 ::: gmnspy.in_out.read_gmns_csv
 
 ::: gmnspy.in_out.read_gmns_network
 
+### Schema
+
+::: gmnspy.schema.SpecConfig
+
+::: gmnspy.schema.official_spec_config
+
+::: gmnspy.schema.local_spec_config
+
 ### Validation
 
-::: gmnspy.validate
+::: gmnspy.validation.apply_schema_to_df
+
+::: gmnspy.validation.update_resources_based_on_existance
 
-::: gmnspy.schema.read_schema
+::: gmnspy.validation.check_required_files
 
-::: gmnspy.schema.read_config
+::: gmnspy.validation.apply_schema_to_df
 
 ### Conversions
 
 TKTK
 
 ### Auto Documentation
 
-::: gmnspy.schema.document_schema
+::: gmnspy.schema.document_schemas_to_md
 
 ::: gmnspy.utils.list_to_md_table
diff --git a/docs/architecture.md b/docs/architecture.md
@@ -1,2 +1,49 @@
 
-To come.
+# General Architecture of GMNS
+
+```mermaid
+erDiagram
+    link |o--|{ node : from_node_id
+    link |o--|{ node : to_node_id
+    link }|--o| geometry : "is optionally associated with"
+    link ||--o{ use-group : "is optionally associated with a"
+    link-tod }|--|| link : "is associated with a"
+    link-tod }|--|| time-set-definitions : "is associated with a"
+    node }o--|| zone : "is optionally associated with"
+    lane }o--|| link : "is part of a"
+    lane-tod }o--|| lane : "is part of a lane"
+    lane-tod }o--|| time-set-definitions : "is associated with a"
+    lane-tod }o--o{ use-group : "is optionally associated with a"
+    location }o--|| link : "is associated with a"
+    location }o--|| node : "is associated with a"
+    location }o--|{ zone : "is optionally associated with a"
+    location ||--o{ "gtfs.stop_id" : "is optionally associated with a"
+    segment }o--|{ link : "is associated with a"
+    segment }o--|{ node : "is associated with a"
+    segment }o--|{ use-group : "is optionally associated with a"
+    use-group }|--|{ use-definition : "includes groups of"
+```
+
+## Signals
+
+```mermaid
+erDiagram
+    signal_controller |o--|{ signal_coordination : ""
+    signal_coordination |o--|{ signal_timing_plan  : ""
+    signal_controller |o--|{ signal_timing_plan :""
+    signal_phase_mvmt }o--|| signal_controller :""
+    signal_phase_mvmt }o--|| movement :""
+    signal_phase_mvmt }o--|| link :""
+    signal_timing_plan }o--|| signal_controller :""
+    signal_timing_plan }o--|{ time_set_definitions :""
+    signal_timing_phase }|--|| signal_timing_plan :""
+    signal_detector }o--|| link :""
+    signal_detector }o--|| node :""
+    signal_detector }o--|| signal_controller : ""
+    movement }o--|| node :""
+    movement }o--|| link : "ib_link_id"
+    movement }o--|| link : "ob_link_id"
+    movement_tod }o--|| timeday_id : ""
+    movement_tod }o--|| link : ib_link_id
+    movement_tod }o--|| link : ob_link_id
+```
diff --git a/docs/spec-versions/development.md b/docs/spec-versions/development.md
@@ -0,0 +1,15 @@
+# Development Branch GMNS Specification
+
+Schema under development for General Network Feed Specification.
+
+## Files in Specification
+
+GMNS consists of a package of files as defined in the following table.
+
+*The following tables are automatically generated.*
+
+{{ official_frictionless_spec('development') }}
+
+## Data Table Schemas
+
+{{ official_frictionless_schemas('development') }}
diff --git a/docs/spec-versions/local.md b/docs/spec-versions/local.md
@@ -0,0 +1,15 @@
+# Local Branch GMNS Specification
+
+Local copy of General Network Feed Specification.
+
+## Files in Specification
+
+GMNS consists of a package of files as defined in the following table.
+
+*The following tables are automatically generated.*
+
+{{ local_frictionless_spec() }}
+
+## Data Table Schemas
+
+{{ local_frictionless_schemas() }}
diff --git a/docs/spec.md b/docs/spec.md
@@ -1,102 +1,22 @@
-# GMNS Specification
+# Official GMNS Specification
 
-Schema for General Network Feed Specification.
+The GMNS specification is managed in the following Github repo: <https://github.com/zephyr-data-specs/GMNS>
 
-## Files in Specification
+| **GMNS Version** | **Description** |
+| ----- | ----- |
+| [development](spec-versions/development.md) | Not yet in official spec. Hasn't been voted on by PMG |
+| [local](spec-versions/local.md) | Local copy of spec. May not be identical to official spec. |
 
-GMNS consists of a package of files as defined in the following table.
+GMNS is defined using the [Frictionless Framework](https://specs.frictionlessdata.io/#data-package-specs-suite) as a series of `tabular-data-resources`.
 
-**The following table is automatically generated from `gmns.spec.json`**
+## Files in Specification
 
-{{ frictionless_spec() }}
+GMNS consists of a package of files as defined in the following table.
 
-File components for GMNS are specified in `gmns.spec.json` in a format compatible with the
-[frictionless data](https://specs.frictionlessdata.io/tabular-data-package/) data package standard.
+*The following tables are automatically generated from the official spec.*
 
-Example `gmns.spec.json`:
-```JSON
-{
-  "profile": "gmns-data-package",
-  "profile_version":0.0,
-  "name": "my-dataset",
-  "resources": [
-   {
-     "name":"link",
-     "path": "link.csv",
-     "schema": "link.schema.json",
-     "required": true
-   },
-   {
-     "name":"node",
-     "path": "node.csv",
-     "schema": "node.schema.json",
-     "required": true
-   }
- ]
-}
-```
+{{ official_frictionless_spec('master') }}
 
 ## Data Table Schemas
 
-Data table schemas are specified in JSON and are compatible with the
-[frictionless data](https://specs.frictionlessdata.io/table-schema/) table
-schema standards.
-
-Example:
-
-```JSON
-{
-    "primaryKey": "segment_id",
-    "missingValues": ["NaN",""],
-    "fields": [
-        {
-            "name": "segment_id",
-            "type": "any",
-            "description": "Primary key.",
-            "constraints": {
-              "required": true,
-              "unique": true
-              }
-        },
-        {
-            "name": "road_link_id",
-            "type": "any",
-            "description": "Required. Foreign key to road_links. The link that the segment is located on.",
-            "foreign_key": "link.link_id",
-            "constraints": {
-              "required": true
-              }
-        },
-        {
-            "name": "ref_node_id",
-            "type": "any",
-            "description": "Required. Foreign key to node.",
-            "foreign_key": "node.node_id",
-            "constraints": {
-              "required": true
-              }
-        },
-        {
-            "name": "start_lr",
-            "type": "number",
-            "description": "Required. Distance from ref_node_id.",
-            "constraints": {
-              "required": true,
-              "minimum": 0
-              }
-        },
-        {
-            "name": "end_lr",
-            "type": "number",
-            "description": "Required. Distance from ref_node_id.",
-            "constraints": {
-              "required": true,
-              "minimum": 0
-              }
-          }
-    ]
-}
-
-```
-
-{{ frictionless_schemas() }}
+{{ official_frictionless_schemas('master') }}
diff --git a/gmnspy/__init__.py b/gmnspy/__init__.py
@@ -6,13 +6,14 @@
     import gmnspy
     ```
 """
+import os
 
 from .in_out import read_gmns_csv, read_gmns_network
 from .schema import (
+    SpecConfig,
     document_schemas_to_md,
-    document_spec_to_md,
-    read_config,
-    read_schema,
+    local_spec_config,
+    official_spec_config,
 )
 from .utils import list_to_md_table, logger
 
@@ -21,7 +22,9 @@
     "read_gmns_network",
     "document_schemas_to_md",
     "document_spec_to_md",
-    "read_config," "read_schema",
+    "SpecConfig",
+    "official_spec_config",
+    "local_spec_config",
     "list_to_md_table",
     "logger",
 ]
diff --git a/gmnspy/defaults.py b/gmnspy/defaults.py
@@ -0,0 +1,10 @@
+import os
+from pathlib import Path
+
+SPEC_GITHUB_USER = "zephyr-data-specs"
+SPEC_GITHUB_REPO = "GMNS"
+SPEC_GITHUB_PATH = Path("Specification")
+SPEC_GITHUB_SPEC_FILE = "gmns.spec.json"
+SPEC_GITHUB_REF = "master"
+
+LOCAL_SPEC = os.path.join(os.path.dirname(os.path.realpath(__file__)), "spec", "gmns.spec.json")