fix: remove experimental and fix up model version test (#4162)

* fix: remove experimental and fix up model version test

* pyformat

* clang

* fix model loading

* make dires

* test

* check experimental

* manually remove automl and epsilon_decay

* test build wheel

* test2

* test against most recent commit

* move test

* change pip

* container

* v1 actions

* python v1

* use container python

* python -m pip

* fix wheel install

* remove experimental from forward model check CI

* change comment to master branch

.github/workflows/forward_model_load_check.yml

@@ -1,7 +1,7 @@
 name: Model checks
 
 # This file is for checking that models generated from newer VW code do not cause crashes when loaded into older VW code
-# Generate a model using latest wheel and load the same model using the latest vw python release
+# Generate a model using latest wheel and load the same model using the latest vw master branch
 
 on:
   push:
@@ -63,21 +63,27 @@ jobs:
           path: ~/.vw_runtests_model_gen_working_dir/test_models/*
           if-no-files-found: error
   test-latest-model:
-    name: Test latest model with previous release
+    name: Test latest model with current master
+    container:
+      image: vowpalwabbit/manylinux2010-build:latest
     needs: generate-latest-model
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v1
         with:
-          python-version: '3.10' 
+          submodules: recursive
       - uses: actions/download-artifact@v1
         with:
           name: vw_generated_models
           path: .vw_runtests_model_gen_working_dir/test_models
-      - name: Test loading model with latest release
+      - name: Test loading model with current master
         shell: bash
         run: |
           mv .vw_runtests_model_gen_working_dir ~
-          pip install vowpalwabbit
-          python ./test/run_tests_model_gen_and_load.py --load_models
+          git clone --recursive https://github.com/VowpalWabbit/vowpal_wabbit.git
+          cd vowpal_wabbit
+          /opt/python/cp310-cp310/bin/pip wheel . -w wheel_last_commit/ --global-option --cmake-options="-DSTATIC_LINK_VW_JAVA=On;-DPython_INCLUDE_DIR='/opt/python/cp310-cp310/include/python3.10/'" --verbose
+          auditwheel repair wheel_last_commit/*whl -w audit_last_commit/
+          /opt/python/cp310-cp310/bin/python3.10 -m pip install wheel_last_commit/*.whl
+          cd ..
+          /opt/python/cp310-cp310/bin/python3.10 ./test/run_tests_model_gen_and_load.py --load_models

test/run_tests_model_gen_and_load.py

@@ -74,9 +74,8 @@ def generate_model(
     working_dir: Path,
     color_enum: Type[Union[Color, NoColor]] = Color,
 ) -> None:
-    command = command + " --quiet "
     print(f"{color_enum.LIGHT_CYAN}id: {test_id}, command: {command}{color_enum.ENDC}")
-    vw = vowpalwabbit.Workspace(command)
+    vw = vowpalwabbit.Workspace(command, quiet=True)
 
     vw.save(str(working_dir / f"model_{test_id}.vw"))
     vw.finish()
@@ -88,7 +87,6 @@ def load_model(
     working_dir: Path,
     color_enum: Type[Union[Color, NoColor]] = Color,
 ) -> None:
-    command = command + " --quiet "
     model_file = str(working_dir / f"model_{test_id}.vw")
     command = command + f" -i {model_file}"
 
@@ -106,15 +104,17 @@ def load_model(
     )
 
     try:
-        vw = vowpalwabbit.Workspace(command)
+        vw = vowpalwabbit.Workspace(command, quiet=True)
         vw.finish()
     except Exception as e:
         print(f"{color_enum.LIGHT_RED} FAILURE!! id: {test_id} {str(e)}")
         raise e
 
 
 def get_tests(
-    working_dir: Path, explicit_tests: Optional[List[int]] = None
+    model_working_dir: Path,
+    working_dir: Path,
+    explicit_tests: Optional[List[int]] = None,
 ) -> List[TestData]:
     test_ref_dir: Path = Path(__file__).resolve().parent
 
@@ -138,11 +138,10 @@ def get_tests(
             not test.depends_on
             and not test.is_shell
             and not test.skip
-            and not " -i " in test.command_line
+            and not "-i " in test.command_line
             and not "--no_stdin" in test.command_line
-            and not "bfgs" in test.command_line
-            and not "--flatbuffer" in test.command_line
             and not "--help" in test.command_line
+            and not "--flatbuffer" in test.command_line
         ):
             test.command_line = re.sub("-f [:a-zA-Z0-9_.\\-/]*", "", test.command_line)
             test.command_line = re.sub("-f=[:a-zA-Z0-9_.\\-/]*", "", test.command_line)
@@ -153,18 +152,35 @@ def get_tests(
                 "--final_regressor=[:a-zA-Z0-9_.\\-/]*", "", test.command_line
             )
             test.command_line = test.command_line.replace("--onethread", "")
+            create_test_dir(
+                test_id=test.id,
+                input_files=test.input_files,
+                test_base_dir=working_dir,
+                test_ref_dir=test_ref_dir,
+            )
+            # Check experimental and loadable reductions
+            os.chdir(model_working_dir.parent)
+            vw = vowpalwabbit.Workspace(test.command_line, quiet=True)
+            skip_cmd = False
+            for groups in vw.get_config().values():
+                for group in groups:
+                    for opt in group[1]:
+                        if opt.value_supplied and (
+                            opt.experimental or opt.name == "bfgs"
+                        ):
+                            skip_cmd = True
+                            break
+                if skip_cmd:
+                    break
+            vw.finish()
+            if skip_cmd:
+                continue
+
             filtered_tests.append(test)
 
     if explicit_tests:
         filtered_tests = list(filter(lambda x: x.id in explicit_tests, filtered_tests))
 
-    for test in filtered_tests:
-        create_test_dir(
-            test_id=test.id,
-            input_files=test.input_files,
-            test_base_dir=working_dir,
-            test_ref_dir=test_ref_dir,
-        )
     return filtered_tests
 
 
@@ -253,7 +269,7 @@ def main():
     else:
         temp_working_dir.mkdir(parents=True, exist_ok=True)
         test_model_dir.mkdir(parents=True, exist_ok=True)
-        tests = get_tests(temp_working_dir, args.test)
+        tests = get_tests(test_model_dir, temp_working_dir, args.test)
 
         if args.generate_models:
             generate_all(tests, test_model_dir, color_enum)