diff --git a/python/tests/test_model_load.py b/python/tests/test_model_load.py
index 70205244..0bd959f3 100644
--- a/python/tests/test_model_load.py
+++ b/python/tests/test_model_load.py
@@ -1,28 +1,52 @@
 import json
-from itertools import chain
+import re
 from pathlib import Path
 
-from importlib_resources import files
-from pytest import mark, param
+from git import Repo
+from packaging import version
+from pytest import fixture
 
 from cucumber_messages import Envelope
 
 
-@mark.parametrize(
-    "ast_path",
-    map(
-        lambda file: param(file, id=file.name),  # type: ignore[no-any-return]
-        chain.from_iterable(map(lambda p: p.glob("*.ndjson"), files("message_samples").iterdir())),
-    ),
-)
-def test_simple_load_model(ast_path: Path):
-    with ast_path.open(mode="r") as ast_file:
-        for ast_line in ast_file:
-            model_datum = json.loads(ast_line)
-            model = Envelope.model_validate(model_datum)  # type: ignore[attr-defined]
+@fixture
+def compatibility_kit_repo(tmpdir):
+    repo_path = Path(tmpdir) / "compatibility-kit"
+    repo = Repo.clone_from(
+        "https://github.com/cucumber/compatibility-kit.git",
+        str(repo_path),
+        branch="main",
+    )
+    repo_tags = list(filter(lambda tag: tag is not None, map(lambda tag: getattr(tag.tag, "tag", None), repo.tags)))
 
-            assert isinstance(model, Envelope)
+    version_pattern = re.compile(r"((.*/)?)v(\d+\.\d+\.\d+)")
+    last_version = sorted(
+        map(
+            version.parse,
+            map(
+                lambda match: match.groups()[-1],
+                filter(lambda match: match is not None, map(lambda tag: re.match(version_pattern, tag), repo_tags)),
+            ),
+        )
+    )[-1]
 
-            dumped_ast_datum = json.loads(model.model_dump_json(by_alias=True, exclude_none=True))  # type: ignore[attr-defined] # migration to pydantic2
+    last_version_tag = next(filter(lambda tag: re.search(re.escape(str(last_version)), tag), repo_tags))
 
-            assert model_datum == dumped_ast_datum
+    repo.git.checkout(last_version_tag)
+
+    return repo_path
+
+
+def test_simple_load_model(compatibility_kit_repo):
+    for ast_path in (compatibility_kit_repo / "devkit" / "samples").rglob("*.ndjson"):
+        print(f"Checking ${ast_path}")
+        with ast_path.open(mode="r") as ast_file:
+            for ast_line in ast_file:
+                model_datum = json.loads(ast_line)
+                model = Envelope.model_validate(model_datum)  # type: ignore[attr-defined]
+
+                assert isinstance(model, Envelope)
+
+                dumped_ast_datum = json.loads(model.model_dump_json(by_alias=True, exclude_none=True))  # type: ignore[attr-defined] # migration to pydantic2
+
+                assert model_datum == dumped_ast_datum