Forbid extra fields and add invalid tests. (#11)

* Forbid `extra` fields and add `invalid` tests.

* Update gen.py

.dvc/.gitignore

@@ -0,0 +1,3 @@
+/config.local
+/tmp
+/cache

.github/workflows/main.yml

@@ -23,5 +23,5 @@ jobs:
         run: |
           ./gen.py | diff schema.json -
           ./gen.py > schema.json
-          ./tests.py
+          pytest -vv tests.py
           dvc repro  # sanity check

dvc.lock

@@ -0,0 +1,24 @@
+schema: '2.0'
+stages:
+  generate:
+    cmd: ./gen.py > schema.json
+    deps:
+    - path: gen.py
+      md5: efb0a465d94a16365e9654a470bd7ef2
+      size: 5597
+    outs:
+    - path: schema.json
+      md5: d2e598efee4b54b6460956edb662a9e4
+      size: 11638
+  check_updated:
+    cmd: ./gen.py | diff schema.json -
+    deps:
+    - path: schema.json
+      md5: d2e598efee4b54b6460956edb662a9e4
+      size: 11638
+  tests:
+    cmd: pytest -vv tests.py
+    deps:
+    - path: schema.json
+      md5: d2e598efee4b54b6460956edb662a9e4
+      size: 11638

dvc.yaml

@@ -13,6 +13,6 @@ stages:
           cache: false
           persist: true
   tests:
-    cmd: ./tests.py
+    cmd: pytest -vv tests.py
     deps:
       - schema.json

examples/invalid/foreach.extra.dvc.yaml

@@ -0,0 +1,10 @@
+stages:
+  test_dict:
+    foreach:
+      first: 1
+      second: 2
+      third: 3
+    do:
+      cmd: echo ${item.value}
+    else:
+      - foo

examples/invalid/live.extra.dvc.yaml

@@ -0,0 +1,9 @@
+stages:
+  copy_multiple:
+    cmd: echo foo > foo
+    live:
+      foo:
+        summary: true
+        html: true
+        cache: false
+        extra: false

examples/invalid/metrics.extra.dvc.yaml

@@ -0,0 +1,7 @@
+stages:
+  generate-metrics:
+    cmd: echo "metric" > scores.json
+    metrics:
+      - scores.json:
+          cache: false
+          extra: false

examples/invalid/outs.extra.dvc.yaml

@@ -0,0 +1,11 @@
+stages:
+  copy_multiple:
+    cmd: cp foo bar && cp foo1 bar1
+    deps:
+      - foo
+      - foo1
+    outs:
+      - foo1
+      - bar1:
+          cache: false
+          extra: false

examples/invalid/plots.extra.dvc.yaml

@@ -0,0 +1,10 @@
+stages:
+  stage_one:
+    cmd: python train.py input plots
+    deps:
+      - input
+    plots:
+      - plot1
+      - plot2:
+          cache: true
+          extra: false

examples/invalid/stage.extra.dvc.yaml

@@ -0,0 +1,8 @@
+stages:
+  stage1:
+    cmd: python train.py input output
+    deps:
+      - input
+    outs:
+      - output
+    extra: false

gen.py

@@ -29,7 +29,8 @@ class OutFlags(BaseModel):
         description="User description for the output",
         title="Description",
     )
-
+    class Config:
+        extra = "forbid"
 
 class PlotFlags(OutFlags):
     x: str = Field(
@@ -91,8 +92,7 @@ class LiveFlags(PlotFlags):
         True, description="Signals dvclive to produce training report"
     )
     cache: Optional[bool] = Field(True, description="Cache output by DVC")
-    class Config:
-        extra = "forbid"
+
 
 class Live(BaseModel):
     __root__: Dict[FilePath, LiveFlags]
@@ -142,6 +142,7 @@ class Stage(BaseModel):
 
     class Config:
         allow_mutation = False
+        extra = "forbid"
 
 
 FOREACH_DESC = """\
@@ -164,7 +165,8 @@ class ForeachDo(BaseModel):
         ..., description=FOREACH_DESC
     )
     do: Stage = Field(..., description=DO_DESC)
-
+    class Config:
+        extra = "forbid"
 
 Definition = Union[ForeachDo, Stage]
 

requirements.txt

@@ -1,3 +1,4 @@
 dvc>1
+jsonschema
 pydantic>1,<2
-jsonschema
+pytest

schema.json

@@ -116,7 +116,8 @@
           "description": "User description for the output",
           "type": "string"
         }
-      }
+      },
+      "additionalProperties": false
     },
     "Out": {
       "title": "Out",
@@ -203,7 +204,8 @@
           "description": "Default plot template",
           "type": "string"
         }
-      }
+      },
+      "additionalProperties": false
     },
     "Plot": {
       "title": "Plot",
@@ -424,7 +426,8 @@
       },
       "required": [
         "cmd"
-      ]
+      ],
+      "additionalProperties": false
     },
     "ForeachDo": {
       "title": "ForeachDo",
@@ -460,7 +463,8 @@
       "required": [
         "foreach",
         "do"
-      ]
+      ],
+      "additionalProperties": false
     }
   }
 }

tests.py

@@ -6,27 +6,29 @@
 from pathlib import Path
 
 import jsonschema
+import pytest
 from dvc.utils.serialize import load_yaml
 
 schema = json.loads(
     (Path(__file__) / ".." / "schema.json").resolve().read_text()
 )
 validate_schema = partial(jsonschema.validate, schema=schema)
 
+VALID_DIR = (Path(__file__) / ".." / "examples" / "valid").resolve()
+INVALID_DIR = (Path(__file__) / ".." / "examples" / "invalid").resolve()
 
-def test_examples():
-    """Validate schema over examples."""
-    examples_dir = (Path(__file__) / ".." / "examples").resolve()
-    assert list(examples_dir.iterdir())
 
-    for example in examples_dir.iterdir():
-        try:
-            validate_schema(load_yaml(example))
-        except Exception:
-            print("Failed to validate:", example)
-            raise
-        print("Validated", example)
+@pytest.mark.parametrize("example", 
+    map(str, VALID_DIR.iterdir())
+)
+def test_valid_examples(example):
+    validate_schema(load_yaml(example))
+
 
+@pytest.mark.parametrize("example", 
+    map(str, INVALID_DIR.iterdir())
+)
+def test_invalid_examples(example):
+    with pytest.raises(jsonschema.ValidationError):
+        validate_schema(load_yaml(example))
 
-if __name__ == "__main__":
-    test_examples()