Wildcard matching (#397)

* Add wildcard matching for CodeLists

* Add test for wildcard matching

* Remove nonsensical test

* Apply suggestions from code review

Co-authored-by: Daniel Huppmann <[email protected]>

* Use pyam.utils.pattern_match for wildcard validation

* Update test file path

* Pin common-definitions to make tests pass

---------

Co-authored-by: Daniel Huppmann <[email protected]>

nomenclature/codelist.py

@@ -7,7 +7,7 @@
 import pandas as pd
 import yaml
 from pyam import IamDataFrame
-from pyam.utils import is_list_like, write_sheet
+from pyam.utils import is_list_like, write_sheet, pattern_match
 from pydantic import BaseModel, ValidationInfo, field_validator
 from pydantic_core import PydanticCustomError
 
@@ -118,7 +118,8 @@ def validate_items(self, items: List[str]) -> List[str]:
         list
             Returns the list of items that are **not** defined in the codelist
         """
-        return [c for c in items if c not in self.keys()]
+        matches = pattern_match(pd.Series(items), self.keys())
+        return [item for item, match in zip(items, matches) if not match]
 
     @classmethod
     def replace_tags(

tests/data/codelist/wildcard/scenario/scenario.yaml

@@ -0,0 +1 @@
+- scen_*

tests/data/config/general-config-only/nomenclature.yaml

@@ -1,6 +1,7 @@
 repositories:
   common-definitions:
     url: https://github.com/IAMconsortium/common-definitions.git/
+    hash: cb85704
 definitions:
   region:
     repository: common-definitions

tests/data/config/general-config/nomenclature.yaml

@@ -2,6 +2,7 @@ dimensions: [region, variable]
 repositories:
   common-definitions:
     url: https://github.com/IAMconsortium/common-definitions.git/
+    hash: cb85704
 definitions:
   region:
     repository: common-definitions

tests/data/region_processing/external_repo_test/nomenclature.yaml

@@ -1,6 +1,7 @@
 repositories:
   common-definitions:
     url: https://github.com/IAMconsortium/common-definitions.git/
+    hash: cb85704
 definitions:
   region:
     repository: common-definitions

tests/test_validation.py

@@ -67,14 +67,6 @@ def test_validation_fails_region(simple_definition, simple_df, caplog):
     )
 
 
-def test_validation_fails_region_as_int(simple_definition, simple_df):
-    """Using a region name as integer raises the expected error"""
-    simple_df.rename(region={"World": 1}, inplace=True)
-
-    with pytest.raises(ValueError, match=MATCH_FAIL_VALIDATION):
-        simple_definition.validate(simple_df)
-
-
 def test_validation_with_custom_dimension(simple_df):
     """Check validation with a custom DataStructureDefinition dimension"""
 
@@ -95,3 +87,15 @@ def test_validation_with_custom_dimension(simple_df):
 
     # validating against all dimensions works
     definition.validate(simple_df)
+
+
+def test_wildcard_match(simple_df):
+    definition = DataStructureDefinition(
+        TEST_DATA_DIR / "codelist" / "wildcard",
+        dimensions=["scenario"],
+    )
+
+    assert definition.validate(simple_df) is None
+
+    with pytest.raises(ValueError, match=MATCH_FAIL_VALIDATION):
+        definition.validate(simple_df.rename(scenario={"scen_a": "foo"}))