Run 'check' in warn mode with 'run'

opensafely/__init__.py

@@ -25,7 +25,9 @@ def show_help(**kwargs):
         title="available commands", description="", metavar="COMMAND"
     )
 
-    parser_help = subparsers.add_parser("help", help="Show this help message and exit")
+    parser_help = subparsers.add_parser(
+        "help", help="Show this help message and exit"
+    )
     parser_help.set_defaults(function=show_help)
 
     # Add `run` subcommand
@@ -34,7 +36,9 @@ def show_help(**kwargs):
     local_run.add_arguments(parser_run)
 
     # Add `codelists` subcommand
-    parser_codelists = subparsers.add_parser("codelists", help=codelists.DESCRIPTION)
+    parser_codelists = subparsers.add_parser(
+        "codelists", help=codelists.DESCRIPTION
+    )
     parser_codelists.set_defaults(function=codelists.main)
     codelists.add_arguments(parser_codelists)
 
@@ -64,6 +68,11 @@ def show_help(**kwargs):
     kwargs = vars(args)
     function = kwargs.pop("function")
     success = function(**kwargs)
+
+    # if `run`ning locally, run `check` in warn mode
+    if function == local_run.main and "format-output-for-github" not in kwargs:
+        check.main(continue_on_error=True)
+
     sys.exit(0 if success is not False else 1)
 
 

opensafely/check.py

@@ -17,7 +17,7 @@ def add_arguments(parser):
     pass
 
 
-def main():
+def main(continue_on_error=False):
     permissions_url = (
         os.environ.get("OPENSAFELY_PERMISSIONS_URL") or PERMISSIONS_URL
     )
@@ -39,9 +39,13 @@ def main():
 
     if found_datasets:
         violations = "\n".join(format_violations(found_datasets))
-        sys.exit(violations)
+        if not continue_on_error:
+            sys.exit(violations)
+        print("*** WARNING ***\n")
+        print(violations)
     else:
-        print("Success")
+        if not continue_on_error:
+            print("Success")
 
 
 def format_violations(found_datasets):

tests/test_check.py

@@ -25,6 +25,7 @@
 
 PERMISSIONS_URL = "https://raw.githubusercontent.com/opensafely-core/opensafely-cli/main/tests/fixtures/permissions/repository_permisisons.yaml"
 
+
 class Repo(Enum):
     PERMITTED = "opensafely/dummy_icnarc"
     PERMITTED_MULTIPLE = "opensafely/dummy_icnarc_ons"
@@ -85,19 +86,20 @@ def git_init(url):
     subprocess.run(["git", "remote", "add", "origin", url])
 
 
-def validate_pass(capsys):
-    check.main()
+def validate_pass(capsys, continue_on_error):
+    check.main(continue_on_error)
     stdout, stderr = capsys.readouterr()
-    assert stderr == ""
-    assert stdout == "Success\n"
+    if not continue_on_error:
+        assert stderr == ""
+        assert stdout == "Success\n"
+    else:
+        assert stdout == ""
 
 
-def validate_fail(capsys):
-    with pytest.raises(SystemExit):
-        check.main()
-        stdout, stderr = capsys.readouterr()
+def validate_fail(capsys, continue_on_error):
+    def validate_fail_output(stdout, stderr):
         assert stdout != "Success\n"
-        assert stderr.startswith("Usage of restricted datasets found:")
+        assert "Usage of restricted datasets found:" in stderr
         assert "icnarc" in stderr
         assert "admitted_to_icu" in stderr
         assert "3:" in stderr
@@ -109,6 +111,17 @@ def validate_fail(capsys):
         assert "study_definition_restricted_1.py" in stderr
         assert "study_definition_restricted_2.py" in stderr
 
+    if not continue_on_error:
+        with pytest.raises(SystemExit):
+            check.main(continue_on_error)
+            stdout, stderr = capsys.readouterr()
+            validate_fail_output(stdout, stderr)
+
+    else:
+        check.main(continue_on_error)
+        stdout, stderr = capsys.readouterr()
+        validate_fail_output(stdout, stdout)
+
 
 @pytest.fixture
 def repo_path(tmp_path):
@@ -119,17 +132,21 @@ def repo_path(tmp_path):
 
 
 @pytest.mark.parametrize(
-    "repo, protocol, dataset",
+    "repo, protocol, dataset, continue_on_error",
     itertools.chain(
-        itertools.product(list(Repo), list(Protocol), list(Dataset)),
-        itertools.product([None], [None], list(Dataset)),
+        itertools.product(
+            list(Repo), list(Protocol), list(Dataset), [True, False]
+        ),
+        itertools.product([None], [None], list(Dataset), [True, False]),
     ),
 )
-def test_check(repo_path, capsys, monkeypatch, repo, protocol, dataset):
+def test_check(
+    repo_path, capsys, monkeypatch, repo, protocol, dataset, continue_on_error
+):
     if "GITHUB_REPOSITORY" in os.environ:
         monkeypatch.delenv("GITHUB_REPOSITORY")
 
-    monkeypatch.setenv("OPENSAFELY_PERMISSIONS_URL",PERMISSIONS_URL)
+    monkeypatch.setenv("OPENSAFELY_PERMISSIONS_URL", PERMISSIONS_URL)
 
     write_study_def(repo_path, dataset)
 
@@ -150,14 +167,15 @@ def test_check(repo_path, capsys, monkeypatch, repo, protocol, dataset):
         Repo.PERMITTED,
         Repo.PERMITTED_MULTIPLE,
     ]:
-        validate_fail(capsys)
+        validate_fail(capsys, continue_on_error)
     else:
-        validate_pass(capsys)
+        validate_pass(capsys, continue_on_error)
+
 
 def test_repository_permissions_yaml():
     permissions = check.get_datasource_permissions(check.PERMISSIONS_URL)
     assert permissions, "empty permissions file"
-    assert type(permissions) == CommentedMap , "invalid permissions file"
-    for k,v in permissions.items():
+    assert type(permissions) == CommentedMap, "invalid permissions file"
+    for k, v in permissions.items():
         assert len(v.keys()) == 1, f"multiple keys specified for {k}"
-        assert "allow" in v.keys(), f"allow key not present for {k}"
+        assert "allow" in v.keys(), f"allow key not present for {k}"