-
Notifications
You must be signed in to change notification settings - Fork 11
Commit
* Rename default SUTs global * manually remove 0.5 standards * remove most of 0.5 code from modelbench * remove irrelevant ssg tests * Remove provisional 0.5 disclaimer from CLI * Remove SSG * delete templates and cli options related to ssg * Write record to run/records/ * Print table summary of results * Modelbench SUT cleanup + testing infra improvements (#754) * SUT arg(s) is now required by CLI * Get rid of DEFFAULT_SUTS * mb tests use centralized SUT fixtures * mv conftest up to root tests dir * Modelbench does not register SUTs * print known SUT uids on newlines * Remove SUT wrapper (#758)
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
import pathlib | ||
|
||
import tomli | ||
|
||
|
||
# TODO: If we plan to keep static content in modelbench, we need to add tests to make sure static content for | ||
# relevant objects exists. | ||
class StaticContent(dict): | ||
def __init__(self, path=pathlib.Path(__file__).parent / "content"): | ||
super().__init__() | ||
self.path = path | ||
for file in (path).rglob("*.toml"): | ||
with open(file, "rb") as f: | ||
try: | ||
data = tomli.load(f) | ||
except tomli.TOMLDecodeError as e: | ||
raise ValueError(f"failure reading {file}") from e | ||
duplicate_keys = set(self.keys()) & set(data.keys()) | ||
if duplicate_keys: | ||
raise Exception(f"Duplicate tables found in content files: {duplicate_keys}") | ||
self.update(data) | ||
|
||
def update_custom_content(self, custom_content_path: pathlib.Path): | ||
custom_content = StaticContent(custom_content_path) | ||
for table in custom_content: | ||
if table not in self: | ||
raise ValueError( | ||
f"Unknown table {table} in custom content from {custom_content_path}; doesn't match {list(self.keys())} from {self.path}" | ||
) | ||
self[table].update(custom_content[table]) |
This file was deleted.
This file was deleted.
This file was deleted.
This file was deleted.
This file was deleted.
This file was deleted.
This file was deleted.
This file was deleted.
This file was deleted.
This file was deleted.
This file was deleted.
This file was deleted.
This file was deleted.
This file was deleted.
This file was deleted.
This file was deleted.
This file was deleted.
This file was deleted.
This file was deleted.
This file was deleted.
This file was deleted.
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
import pytest | ||
|
||
from modelgauge.sut_registry import SUTS | ||
from modelgauge_tests.fake_sut import FakeSUT | ||
|
||
# Need to declare global here because session start hook can't access fixtures. | ||
_SUT_UID = "fake-sut" | ||
|
||
|
||
def pytest_sessionstart(session): | ||
"""Register the fake SUT during the session start.""" | ||
SUTS.register(FakeSUT, _SUT_UID) | ||
|
||
|
||
def pytest_sessionfinish(session, exitstatus): | ||
"""Remove fake SUTs from registry.""" | ||
del SUTS._lookup[_SUT_UID] | ||
|
||
|
||
@pytest.fixture(scope="session") | ||
def sut_uid(): | ||
return _SUT_UID | ||
|
||
|
||
@pytest.fixture | ||
def sut(sut_uid): | ||
return FakeSUT(sut_uid) |
This file was deleted.
This file was deleted.
This file was deleted.
This file was deleted.
This file was deleted.
This file was deleted.
This file was deleted.
This file was deleted.
This file was deleted.
This file was deleted.