From b3fea1c7eaa7058d91d717c12889c217542c413b Mon Sep 17 00:00:00 2001 From: David Hart Date: Mon, 4 Nov 2024 13:36:15 -0700 Subject: [PATCH 1/9] fix(dat-convert): change default conversion behavior to match sansmic UM (2015) --- src/python/sansmic/io.py | 31 ++++++++++++++++++++++++++----- tests/test_app.py | 3 +++ tests/test_io.py | 12 +++++++++++- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/src/python/sansmic/io.py b/src/python/sansmic/io.py index db1ada0..11f5779 100644 --- a/src/python/sansmic/io.py +++ b/src/python/sansmic/io.py @@ -137,6 +137,7 @@ def read_dat(str_or_buffer: Union[str, Path], *, ignore_errors=False) -> Scenari else str_or_buffer ) as fin: scenario.title = "Converted from DAT-file" + scenario.comments = """\nConversion report:\n""" first = True logger.debug('Reading DAT-formatted file "{}"'.format(Path(fin.name).name)) while True: @@ -241,14 +242,11 @@ def read_dat(str_or_buffer: Union[str, Path], *, ignore_errors=False) -> Scenari + "\n tdlay [depr.]: {}".format(tDelay) + "\n sep : {}".format(coallescing_well_separation) ) - if float(tDelay) != 0: - logger.warning("The TDLAY option should not be used. Ignoring.") # First stage - block 9 if first: logger.debug("First stage initialization") logger.debug("Record 9 - Geometry\n data:") geometry_data = list() - first = False geometry_format = GeometryFormat(int(geometry_format)) if geometry_format == GeometryFormat.RADIUS_LIST: radii = list() @@ -300,6 +298,7 @@ def read_dat(str_or_buffer: Union[str, Path], *, ignore_errors=False) -> Scenari logger.warning( "The REFDEP is no longer used, only DEPTH. Ignoring REFDEP." ) + scenario.comments += "* The REFDEP value (field 3 in record 10) is unused; DEPTH (field 4) is used instead.\n" if float(dissolution_factor) != 1.0: logger.warning("The ZDIS should almost always be 1.0 for NaCl(s)") scenario.geometry_format = geometry_format @@ -310,8 +309,14 @@ def read_dat(str_or_buffer: Union[str, Path], *, ignore_errors=False) -> Scenari df = float(dissolution_factor) if df != 1.0: scenario.advanced.dissolution_factor = df + scenario.comments += "* Atypical dissolution factor set to {} in [advanced] options table.\n".format( + dissolution_factor + ) scenario.insolubles_ratio = float(insoluble_fraction) scenario.floor_depth = float(depth) + scenario.comments += "* Floor depth was {} ft MD; all heights converted to MD using this value.\n".format( + depth + ) scenario.cavern_height = float(cavern_height) scenario.ullage_standoff = float(ullage_standoff) else: @@ -326,12 +331,25 @@ def read_dat(str_or_buffer: Union[str, Path], *, ignore_errors=False) -> Scenari ) ) raise TypeError("Invalid data in DAT file: RESETGEO not 0") - if not first and isinstance(cavern_sg, (float, int)) and cavern_sg > 1.0: + if float(tDelay) != 0: + logger.warning("The TDLAY option should not be used. Ignoring.") + scenario.comments += "* The TDLAY option ({}) was removed from stage {}, as it has been deprecated.\n".format( + tDelay, len(scenario.stages) + 1 + ) + if not first and (cavern_sg is not None and float(cavern_sg) > 1.0): + scenario.comments += "* Initial cavern SG was changed from {} to 0 in subsequent stage number {}.\n".format( + cavern_sg, len(scenario.stages) + 1 + ) + scenario.comments += " If you truly meant to reset the cavern SG at the beginning of the stage,\n" + scenario.comments += " please add ``set-cavern-sg = {}`` into [[stages]] definition number {}\n".format( + cavern_sg, len(scenario.stages) + 1 + ) logger.warning( - "The REPEAT option was supposed to turn off the cavern SG; it did not do so. sansmic is currently mimicing SANSMIC behavior and resetting the cavern brine to {} sg in stage {}. \n\nIf this is not what is intended, please manually remove the 'set-cavern-sg' entry from stages after stage 1. This behavior will change in future releases.".format( + "The REPEAT option now disables cavern SG initialization that is non-zero. Please add ``set-initial-conditions = true`` and ``set-cavern-sg = {}`` to [[stages]] {}".format( cavern_sg, len(scenario.stages) + 1 ) ) + cavern_sg = 0.0 stage.title = title stage.simulation_mode = SimulationMode(int(mode)) stage.save_frequency = int(print_interval) @@ -363,6 +381,7 @@ def read_dat(str_or_buffer: Union[str, Path], *, ignore_errors=False) -> Scenari stage.injection_duration = float(injection_duration) stage.product_injection_rate = float(fill_rate) scenario.stages.append(stage) + first = False logger.debug("Finished reading stage") logger.debug(".DAT file converted successfully") return scenario @@ -429,6 +448,8 @@ def write_scenario(scenario: Scenario, filename: str, *, redundant=False, format continue if isinstance(v, bool): v = str(v).lower() + elif k == "comments": + v = '"""' + v + '"""' elif isinstance(v, str): v = repr(v) elif isinstance(v, IntEnum): diff --git a/tests/test_app.py b/tests/test_app.py index 382b380..f9cf25b 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -185,17 +185,20 @@ def test_convert_app(self): """Test the 'sansmic-convert' command line application.""" scenario0 = sansmic.io.read_scenario(self.withdrawal_dat) scenario0.title = "" + scenario0.comments = "" sansmic.app.convert( [self.withdrawal_dat, self.withdrawal_toml], standalone_mode=False ) scenario1 = sansmic.io.read_scenario(self.withdrawal_toml) scenario1.title = "" + scenario1.comments = "" self.assertEqual(scenario0, scenario1) sansmic.app.convert( [self.withdrawal_dat, self.withdrawal_toml, "--full"], standalone_mode=False ) scenario2 = sansmic.io.read_scenario(self.withdrawal_toml) scenario2.title = "" + scenario2.comments = "" self.maxDiff = None self.assertDictEqual(scenario0.to_dict(), scenario2.to_dict()) diff --git a/tests/test_io.py b/tests/test_io.py index 3ce3f67..da8c072 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -259,7 +259,7 @@ def test_read_scenario_ordinary_reverse_multistage(self): self.assertEqual(scenario.stages[2].outer_csg_inside_diam / 2, 8.925) self.assertEqual(scenario.stages[2].outer_csg_outside_diam / 2, 9.375) self.assertEqual(scenario.stages[2].brine_injection_sg, 1.03) - self.assertEqual(scenario.stages[2].set_cavern_sg, 1.2019) + self.assertIsNone(scenario.stages[2].set_cavern_sg) # Changed in version 1.1 self.assertEqual(scenario.stages[2].solver_timestep, 0.1) self.assertEqual(scenario.stages[2].injection_duration, 72) self.assertEqual(scenario.stages[2].product_injection_rate, 0.0) @@ -267,15 +267,18 @@ def test_read_scenario_ordinary_reverse_multistage(self): def test_roundtrip_toml(self): scenario = sansmic.io.read_scenario(self.withdrawal_dat) scenario.title = "" + scenario.comments = "" sansmic.io.write_scenario(scenario, self.withdrawal_toml) scenario2 = sansmic.io.read_scenario(self.withdrawal_toml) scenario2.title = "" + scenario2.comments = "" self.assertEqual(scenario, scenario2) def test_roundtrip_yaml(self): self.maxDiff = None scenario = sansmic.io.read_scenario(self.ordinary_rev_dat) scenario.title = "" + scenario.comments = "" scenario.stages[1].brine_interface_depth = 0 scenario.stages[2].brine_interface_depth = 0 scenario.stages[1].set_initial_conditions = None @@ -284,6 +287,7 @@ def test_roundtrip_yaml(self): sansmic.io.write_scenario(scenario, self.ordinary_rev_yaml, redundant=True) scenario2 = sansmic.io.read_scenario(self.ordinary_rev_yaml) scenario2.title = "" + scenario2.comments = "" self.assertDictEqual( scenario.to_dict(keep_empty=True), scenario2.to_dict(keep_empty=True) ) @@ -291,28 +295,34 @@ def test_roundtrip_yaml(self): def test_roundtrip_json(self): scenario = sansmic.io.read_scenario(self.ordinary_dir_dat) scenario.title = "" + scenario.comments = "" sansmic.io.write_scenario(scenario, self.ordinary_dir_json) scenario2 = sansmic.io.read_scenario(self.ordinary_dir_json) scenario2.title = "" + scenario2.comments = "" self.assertEqual(scenario, scenario2) def test_roundtrip_spec_format(self): scenario = sansmic.io.read_scenario(self.leach_fill_dat) scenario.title = "" + scenario.comments = "" sansmic.io.write_scenario(scenario, self.leach_fill_format, format="toml") scenario2 = sansmic.io.read_scenario(self.leach_fill_format, format="toml") scenario2.title = "" + scenario2.comments = "" self.assertEqual(scenario, scenario2) sansmic.io.write_scenario(scenario, self.leach_fill_format, format="json") scenario3 = sansmic.io.read_scenario(self.leach_fill_format, format="json") scenario3.title = "" + scenario3.comments = "" self.assertEqual(scenario, scenario3) sansmic.io.write_scenario(scenario, self.leach_fill_format, format="yaml") scenario4 = sansmic.io.read_scenario(self.leach_fill_format, format="yaml") scenario4.title = "" + scenario4.comments = "" self.assertEqual(scenario, scenario4) @classmethod From d28bdf4254af3d437b1dfd6bcce2372d5ab37d7a Mon Sep 17 00:00:00 2001 From: David Hart Date: Mon, 4 Nov 2024 14:17:10 -0700 Subject: [PATCH 2/9] docs: Update scenario documentation Update to include both -depth and -height configuraiton options for brine injection, production, and interface. --- docs/um/scenarios.rst | 64 +++++++++++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/docs/um/scenarios.rst b/docs/um/scenarios.rst index cd4ea5f..780479f 100644 --- a/docs/um/scenarios.rst +++ b/docs/um/scenarios.rst @@ -95,15 +95,15 @@ be defined for **each** stage * :confval:`injection-duration` * :confval:`rest-duration` * :confval:`brine-injection-sg` -* :confval:`brine-injection-depth` -* :confval:`brine-production-depth` +* :confval:`brine-injection-depth` xor :confval:`brine-injection-height` +* :confval:`brine-production-depth` xor :confval:`brine-production-height` * :confval:`brine-injection-rate` The keys which are required for at least the **first** stage are * :confval:`set-cavern-sg` -* :confval:`brine-interface-depth` +* :confval:`brine-interface-depth` xor :confval:`brine-interface-height` The keys which can be set using the ``[defaults]`` section @@ -528,25 +528,37 @@ number of stages. A stage **requires** a title key. .. confval:: brine-injection-depth +.. confval:: brine-injection-height - The *height* above the original :confval:`floor-depth` of the cavern + The *depth* (preferred) or *height* within the cavern where the EOT for the injection string is located. If above the production depth, it will be assigned to the outer casing, otherwise it will apply to the inner casing. - This value is specified in :term:`foot` above :confval:`floor-depth`. - Changing depth values to handle measured depths below surface - is the second-highest priority update. + + The :confval:`brine-injection-depth` is specified in :term:`foot` + below the :term:`ZDP` at the surface. + + The :confval:`brine-injection-height` is specified in :term:`foot` + above the original :confval:`floor-depth` + + Do not use both -depth and -height options. .. confval:: brine-production-depth +.. confval:: brine-production-height - The *height* above the original :confval:`floor-depth` of the cavern + The *depth* (preferred) or *height* within the cavern where the EOT for the injection string is located. If above the injection depth, it will be assigned to the outer casing, otherwise it will apply to the inner casing. - This value is specified in :term:`foot` above :confval:`floor-depth`. - Changing depth values to handle measured depths below surface - is the second-highest priority update. + + The :confval:`brine-production-depth` is specified in :term:`foot` + below the :term:`ZDP` at the surface. + + The :confval:`brine-production-height` is specified in :term:`foot` + above the original :confval:`floor-depth` + + Do not use both -depth and -height options. .. confval:: brine-injection-rate @@ -558,18 +570,23 @@ number of stages. A stage **requires** a title key. .. confval:: brine-interface-depth +.. confval:: brine-interface-height - The initial brine interface depth in *height* above the - :confval:`floor-depth`. This is only required for the first stage, and + The initial brine interface *depth* (preferred) or *height* + within the cavern. This is only required for the first stage, and should not be set in subsequent stages unless you *want* to manually reset the interface position. If so, you should also set :confval:`set-initial-conditions` to true. If left blank or set to 0, it will behave as if unset and use the value from the previous stage. - Specified in :term:`foot` above :confval:`floor-depth`. - Changing depth values to handle measured depths below surface - is the second-highest priority update. + The :confval:`brine-interface-depth` is specified in :term:`foot` + below the :term:`ZDP` at the surface. + + The :confval:`brine-interface-height` is specified in :term:`foot` + above the original :confval:`floor-depth` + + Do not use both -depth and -height options. .. confval:: set-cavern-sg @@ -580,16 +597,17 @@ number of stages. A stage **requires** a title key. for the first stage, it will be set to the maximum brine specific gravity. + *Changed: v1.0.6* + The SANSMIC user guide from 2015 had an error, and stated that subsequent stages did not use this value. In fact, this value - was still being set unless it was set to 1.0 or less. This - version of sansmic will give a warning when converting files - but behave *as was described in the user manual*. If you want - to replicate the old behavior, set :confval:`set-initial-conditions` + was still being set unless it was set to 1.0 or less. + + sansmic will behave *as was described in the user manual*. To + replicate the old behavior, set :confval:`set-initial-conditions` to true to force initialization of the cavern brine to the value - you specify in this key. You can also set this to 0.0 in an - old DAT file to force the proper behavior of continuing between - stages. + you specify in this key. (You can also set this to 0.0 in an + old DAT file to force this behavior in the FORTRAN version) .. confval:: set-initial-conditions From bfe3b1d54aecf66111ffcd2dc3486886e6b3d62f Mon Sep 17 00:00:00 2001 From: David Hart Date: Tue, 5 Nov 2024 09:04:06 -0700 Subject: [PATCH 3/9] ci: update codecov config --- .github/codecov.yml | 31 +++++----- .github/workflows/test-continuous.yml | 3 +- .github/workflows/test-matrix.yml | 87 +++++++++++++++------------ 3 files changed, 67 insertions(+), 54 deletions(-) diff --git a/.github/codecov.yml b/.github/codecov.yml index 5103007..b53704b 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -1,32 +1,35 @@ coverage: status: project: + default: false + tests: + target: 100% + paths: + - "tests/" + app: + target: auto + threshold: 5% + paths: + - "src/" + patch: default: - # basic target: auto - threshold: 5 - base: auto + threshold: 5% + only_pulls: true paths: - "src/" - "tests/" - # advanced settings - branches: - - main - - dev - if_ci_failed: error #success, failure, error, ignore - informational: false - only_pulls: false comment: - layout: "header, diff, components" # show component info in the PR comment + layout: "condensed_header, condensed_files, condensed_footer" # show component info in the PR comment + behavior: new + require_changes: "coverage_drop AND uncovered_patch" component_management: default_rules: statuses: - type: project - target: auto - branches: - - "!main" + - type: patch individual_components: - component_id: module_python name: python code diff --git a/.github/workflows/test-continuous.yml b/.github/workflows/test-continuous.yml index a36c88d..e1bc606 100644 --- a/.github/workflows/test-continuous.yml +++ b/.github/workflows/test-continuous.yml @@ -67,12 +67,11 @@ jobs: - name: Upload coverage reports to Codecov if: success() || failure() + continue-on-error: true uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 with: token: ${{ secrets.CODECOV_TOKEN }} - flags: ${{ inputs.os }} - name: Test uninstall - if: success() || failure() # Allow upload to codecov to fail but not fail all tests run: python3 -m pip uninstall -y sansmic diff --git a/.github/workflows/test-matrix.yml b/.github/workflows/test-matrix.yml index 3823dbd..bac1b59 100644 --- a/.github/workflows/test-matrix.yml +++ b/.github/workflows/test-matrix.yml @@ -14,17 +14,12 @@ on: - '!docs/conf.py' workflow_dispatch: - inputs: - skip_coverage: - default: true - required: false - type: boolean permissions: contents: read jobs: - test-all-versions-oses: + platforms: strategy: fail-fast: false matrix: @@ -60,52 +55,68 @@ jobs: run: | python3 -m pip install --upgrade pip - - name: Action | Test editable build - run: python3 -m pip install -e .[formats,examples,tests] + - name: Action | Test installation + run: python3 -m pip install .[formats,examples,tests] - - name: Action | Test without coverage on Python 3.9 or if requested - if: ${{ (github.event == 'workflow_dispatch' && inputs.skip_coverage) || (matrix.os != 'ubuntu-latest' && matrix.version == '3.9') }} + - name: Action | Test package and examples run: | python3 -m pytest --nbmake --disable-warnings --no-header --color=auto examples/ tests/ >> $GITHUB_STEP_SUMMARY - - name: Action | Coverage and testing Linux - Python and C++ coverage - if: ${{ (github.event == 'pull_request' || ! inputs.skip_coverage ) && matrix.os == 'ubuntu-latest' && matrix.version != '3.9'}} + - name: Action | Test uninstall process + # Allow upload to codecov to fail but not fail all tests + run: python3 -m pip uninstall -y sansmic + + coverage: + runs-on: ubuntu-latest + steps: + - name: Setup | Harden Runner + uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + api.codecov.io:443 + api.github.com:443 + cli.codecov.io:443 + codecov.io:443 + files.pythonhosted.org:443 + github.com:443 + pypi.org:443 + storage.googleapis.com:443 + uploader.codecov.io:443 + + - name: Setup | Check out the commit + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Setup | Set up Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: '3.12' + + - name: Setup | Install dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install -r requirements.txt + python3 -m pip install gcovr==5.0 + + - name: Setup | Install editable build + run: python3 -m pip install -e .[formats,examples,tests] + + - name: Action | Test package and examples and get coverage results run: | export CPPFLAGS="-coverage" - python3 -m pip install setuptools pybind11 gcovr python3 setup.py build_ext --inplace --force echo '### Run tests' >> $GITHUB_STEP_SUMMARY echo '```bash' >> $GITHUB_STEP_SUMMARY python3 -m pytest --nbmake --disable-warnings --cov=sansmic --cov=tests --no-header --color=auto examples/ tests/ | tee -a $GITHUB_STEP_SUMMARY find src -type f -name '*.cpp' | xargs -I{} gcov -o build/temp*/src/ext_modules/libsansmic {} - gcovr --txt | tee -a $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - - - name: Action | Coverage and testing on MacOS - Python coverage only - if: ${{ (github.event == 'pull_request' || ! inputs.skip_coverage ) && matrix.os != 'windows-latest' && matrix.os != 'ubuntu-latest' }} - run: | - echo '### Run tests' >> $GITHUB_STEP_SUMMARY - echo '```bash' >> $GITHUB_STEP_SUMMARY - python3 -m pytest --nbmake --disable-warnings --cov=sansmic --cov=tests --no-header --color=auto examples/ tests/ | tee -a $GITHUB_STEP_SUMMARY + echo ' ' >> $GITHUB_STEP_SUMMARY + gcovr --txt -s -k | tee -a $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY - - name: Action | Coverage and testing on Windows - Python coverage only - if: ${{ (github.event == 'pull_request' || ! inputs.skip_coverage ) && matrix.os == 'windows-latest' }} - shell: powershell - run: | - echo '### Run tests' >> $Env:GITHUB_STEP_SUMMARY - echo '```bash' >> $Env:GITHUB_STEP_SUMMARY - python3 -m pytest --nbmake --disable-warnings --cov=sansmic --cov=tests --no-header --color=auto examples/ tests/ >> $Env:GITHUB_STEP_SUMMARY - echo '```' >> $Env:GITHUB_STEP_SUMMARY - - name: Action | Upload coverage reports to Codecov - if: ${{ (github.event == 'pull_request' || ! inputs.skip_coverage) && success() || failure() }} + if: success() || failure() + continue-on-error: true uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 with: token: ${{ secrets.CODECOV_TOKEN }} - flags: ${{ matrix.os }} - - - name: Action | Test uninstall process - if: success() || failure() - # Allow upload to codecov to fail but not fail all tests - run: python3 -m pip uninstall -y sansmic From ce34b3762f4f6eed9e411194ec571db143d50252 Mon Sep 17 00:00:00 2001 From: David Hart Date: Tue, 5 Nov 2024 09:15:44 -0700 Subject: [PATCH 4/9] ci: always comment on pull requests Signed-off-by: David Hart --- .github/codecov.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/codecov.yml b/.github/codecov.yml index b53704b..b69f9ec 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -22,8 +22,7 @@ coverage: comment: layout: "condensed_header, condensed_files, condensed_footer" # show component info in the PR comment - behavior: new - require_changes: "coverage_drop AND uncovered_patch" + require_changes: false component_management: default_rules: From c34a894f2ed2215ffa6c064d748a7225de2911b5 Mon Sep 17 00:00:00 2001 From: David Hart Date: Tue, 5 Nov 2024 09:30:09 -0700 Subject: [PATCH 5/9] ci: update codecov configuration --- .github/codecov.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/codecov.yml b/.github/codecov.yml index b69f9ec..2f8ea2d 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -1,16 +1,12 @@ coverage: status: project: - default: false - tests: - target: 100% - paths: - - "tests/" - app: + default: target: auto threshold: 5% paths: - "src/" + - "tests/" patch: default: target: auto @@ -23,6 +19,7 @@ coverage: comment: layout: "condensed_header, condensed_files, condensed_footer" # show component info in the PR comment require_changes: false + behavior: new component_management: default_rules: @@ -31,14 +28,16 @@ component_management: - type: patch individual_components: - component_id: module_python - name: python code + name: python package paths: - src/python/** - component_id: module_ext - name: C++ code + name: external modules paths: - - src/ext_modules/** + - src/ext_modules/**/*.cpp + - src/ext_modules/**/*.c - component_id: module_tests - name: tests + name: tests and examples + target: 100% paths: - tests/** From 58496b15de340afaa704d3b6c34cbd5bdd4bcbfe Mon Sep 17 00:00:00 2001 From: David Hart Date: Tue, 5 Nov 2024 09:40:19 -0700 Subject: [PATCH 6/9] ci: update codecov config again --- .github/codecov.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/codecov.yml b/.github/codecov.yml index 2f8ea2d..ff35bd5 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -1,7 +1,7 @@ coverage: status: project: - default: + default: target: auto threshold: 5% paths: @@ -17,9 +17,8 @@ coverage: - "tests/" comment: - layout: "condensed_header, condensed_files, condensed_footer" # show component info in the PR comment + layout: "condensed_header, condensed_diff, condensed_files" # show component info in the PR comment require_changes: false - behavior: new component_management: default_rules: From 982217af2200852a3e6fc959be3e54a9e24ea516 Mon Sep 17 00:00:00 2001 From: David Hart Date: Tue, 5 Nov 2024 09:52:38 -0700 Subject: [PATCH 7/9] ci: update codecov config for better comments --- .github/codecov.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/codecov.yml b/.github/codecov.yml index ff35bd5..a226959 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -7,6 +7,21 @@ coverage: paths: - "src/" - "tests/" + libsansmic: + target: auto + threshold: 5% + paths: + - "src/ext_modules/**/*.cpp" + - "src/ext_modules/**/*.c" + sansmic-py: + target: auto + threshold: 5% + paths: + - "src/python/**" + tests: + target: 100% + paths: + - "tests/**" patch: default: target: auto @@ -19,6 +34,8 @@ coverage: comment: layout: "condensed_header, condensed_diff, condensed_files" # show component info in the PR comment require_changes: false + hide_project_coverage: false + behavior: new component_management: default_rules: From c1a65f1d4778c9f40e5b6fae129e13503aaa7b46 Mon Sep 17 00:00:00 2001 From: David Hart Date: Tue, 5 Nov 2024 09:55:13 -0700 Subject: [PATCH 8/9] ci: update codecov to show component info --- .github/codecov.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/codecov.yml b/.github/codecov.yml index a226959..7da3ff9 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -32,7 +32,7 @@ coverage: - "tests/" comment: - layout: "condensed_header, condensed_diff, condensed_files" # show component info in the PR comment + layout: "condensed_header, condensed_diff, condensed_files, components" # show component info in the PR comment require_changes: false hide_project_coverage: false behavior: new From 785716d985f05a47c81d3ac226dce9916dc43f48 Mon Sep 17 00:00:00 2001 From: David Hart Date: Tue, 5 Nov 2024 10:42:37 -0700 Subject: [PATCH 9/9] test: rename regression outputs as txt Rename regression testing output files to be explicitly .txt to avoid filetype confusion. --- examples/{baseline.tst => baseline.tst.txt} | 0 examples/regression.ipynb | 8 ++++---- src/python/sansmic/io.py | 10 +++++----- tests/{baseline.ddl => baseline.ddl.txt} | 0 tests/{baseline.dra => baseline.dra.txt} | 0 tests/{baseline.out => baseline.out.txt} | 0 tests/{baseline.rad => baseline.rad.txt} | 0 tests/{baseline.tst => baseline.tst.txt} | 0 tests/test_regression.py | 9 ++++++--- 9 files changed, 15 insertions(+), 12 deletions(-) rename examples/{baseline.tst => baseline.tst.txt} (100%) rename tests/{baseline.ddl => baseline.ddl.txt} (100%) rename tests/{baseline.dra => baseline.dra.txt} (100%) rename tests/{baseline.out => baseline.out.txt} (100%) rename tests/{baseline.rad => baseline.rad.txt} (100%) rename tests/{baseline.tst => baseline.tst.txt} (100%) diff --git a/examples/baseline.tst b/examples/baseline.tst.txt similarity index 100% rename from examples/baseline.tst rename to examples/baseline.tst.txt diff --git a/examples/regression.ipynb b/examples/regression.ipynb index 8d83bf7..420d797 100644 --- a/examples/regression.ipynb +++ b/examples/regression.ipynb @@ -19,7 +19,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -28,7 +28,7 @@ "import matplotlib.pylab as plt\n", "\n", "model = sansmic.io.read_scenario(\"baseline.dat\")\n", - "tstF = sansmic.io.read_tst_file(\"baseline\")\n", + "tstF = sansmic.io.read_tst_file(\"baseline.tst.txt\")\n", "resF = sansmic.io.read_json_results(\"baseline.json\")" ] }, @@ -98,7 +98,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -107,7 +107,7 @@ ") as sim:\n", " sim.run_sim()\n", "resPy = sim.results\n", - "tstPy = sansmic.io.read_tst_file(\"regression\")" + "tstPy = sansmic.io.read_tst_file(\"regression.tst\")" ] }, { diff --git a/src/python/sansmic/io.py b/src/python/sansmic/io.py index 11f5779..f8c6ae9 100644 --- a/src/python/sansmic/io.py +++ b/src/python/sansmic/io.py @@ -502,7 +502,7 @@ def write_scenario(scenario: Scenario, filename: str, *, redundant=False, format logger.info(f'Scenario file written to "{file_path.name}"') -def read_classic_out_ddl(file_prefix): +def read_classic_out_ddl(out_filename, ddl_filename): """ Read in the data from ".out" and ".ddl" files from a classic SANSMIC run. @@ -528,7 +528,7 @@ def read_classic_out_ddl(file_prefix): z_0 = list() h_0 = list() - with open(file_prefix + ".ddl", "r") as fddl: + with open(ddl_filename, "r") as fddl: for ct, line in enumerate(fddl.readlines()): if ct == 0: continue @@ -544,7 +544,7 @@ def read_classic_out_ddl(file_prefix): h = z - z_0[0] h_0.append(h) - with open(file_prefix + ".out", "r") as fout: + with open(out_filename, "r") as fout: for line in fout.readlines(): words = line.split() if skip1: @@ -671,7 +671,7 @@ def read_classic_out_ddl(file_prefix): return res -def read_tst_file(file_prefix: str): +def read_tst_file(tst_filename: str): """Read a .tst output file into a DataFrame. Parameters @@ -695,7 +695,7 @@ def read_tst_file(file_prefix: str): Q_fill = list() V_injTot = list() V_fillTot = list() - with open(file_prefix + ".tst", "r") as fout: + with open(tst_filename, "r") as fout: for line in fout.readlines(): words = line.split() if len(words) == 0: diff --git a/tests/baseline.ddl b/tests/baseline.ddl.txt similarity index 100% rename from tests/baseline.ddl rename to tests/baseline.ddl.txt diff --git a/tests/baseline.dra b/tests/baseline.dra.txt similarity index 100% rename from tests/baseline.dra rename to tests/baseline.dra.txt diff --git a/tests/baseline.out b/tests/baseline.out.txt similarity index 100% rename from tests/baseline.out rename to tests/baseline.out.txt diff --git a/tests/baseline.rad b/tests/baseline.rad.txt similarity index 100% rename from tests/baseline.rad rename to tests/baseline.rad.txt diff --git a/tests/baseline.tst b/tests/baseline.tst.txt similarity index 100% rename from tests/baseline.tst rename to tests/baseline.tst.txt diff --git a/tests/test_regression.py b/tests/test_regression.py index 8fa6bc9..e69c47b 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -16,8 +16,11 @@ class TestRegression(unittest.TestCase): def setUpClass(cls): cls.nameF = "baseline" cls.datF = join(testdir, cls.nameF + ".dat") - cls.tstF = sansmic.io.read_tst_file(join(testdir, cls.nameF)) - cls.resF = sansmic.io.read_classic_out_ddl(join(testdir, cls.nameF)) + cls.tstF = sansmic.io.read_tst_file(join(testdir, cls.nameF) + ".tst.txt") + cls.resF = sansmic.io.read_classic_out_ddl( + join(testdir, cls.nameF) + ".out.txt", + join(testdir, cls.nameF) + ".ddl.txt", + ) cls.namePy = "regression" def test_regression_results(self): @@ -26,7 +29,7 @@ def test_regression_results(self): with model.new_simulation(join(testdir, self.namePy)) as sim: sim.run_sim() resPy = sim.results - tstPy = sansmic.io.read_tst_file(join(testdir, self.namePy)) + tstPy = sansmic.io.read_tst_file(join(testdir, self.namePy) + ".tst") Stats = sansmic.model.pd.DataFrame.from_dict( dict( rmse=np.sqrt(np.mean((self.tstF - tstPy) ** 2, 0)),