Merge pull request #157 from tjgalvin/leakage

Calculate leakage from component catalogue and polarised image

.github/workflows/python-lint-test.yml

@@ -5,35 +5,34 @@ name: Python application
 
 on:
   push:
-    branches: [ "main" ]
+    branches: ["main"]
   pull_request:
-    branches: [ "main" ]
+    branches: ["main"]
 
 permissions:
   contents: read
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v3
-      with:
-        python-version: "3.8"
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip uv
-        uv pip install --system pytest pytest-cov pre-commit black mypy .
-    - name: Lint with pre-commit
-      run: |
-        pre-commit run --all-files
-    - name: Test with pytest
-      run: |
-        pytest --cov flint
-    - name: Upload coverage reports to Codecov
-      uses: codecov/[email protected]
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
+      - uses: actions/checkout@v3
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.8"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip uv
+          uv pip install --system pytest pytest-cov pre-commit black mypy .
+      - name: Lint with pre-commit
+        run: |
+          pre-commit run --all-files
+      - name: Test with pytest
+        run: |
+          pytest --cov flint
+      - name: Upload coverage reports to Codecov
+        uses: codecov/[email protected]
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}

.pre-commit-config.yaml

@@ -1,28 +1,88 @@
-repos:
--   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
-    hooks:
-    -   id: check-yaml
-    -   id: end-of-file-fixer
-    -   id: trailing-whitespace
--   repo: https://github.com/PyCQA/isort
-    rev: 5.13.2
-    hooks:
-    -   id: isort
-        args: ["--profile=black"]
--   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.5.4
-    hooks:
-    -   id: ruff
-    -   id: ruff-format
 ci:
-    autofix_commit_msg: |
-        [pre-commit.ci] auto fixes from pre-commit.com hooks
-
-        for more information, see https://pre-commit.ci
-    autofix_prs: true
-    autoupdate_branch: ''
-    autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate'
-    autoupdate_schedule: weekly
-    skip: []
-    submodules: false
+  autoupdate_commit_msg: "chore: update pre-commit hooks"
+  autofix_commit_msg: "style: pre-commit fixes"
+
+repos:
+  - repo: https://github.com/adamchainz/blacken-docs
+    rev: "1.18.0"
+    hooks:
+      - id: blacken-docs
+        additional_dependencies: [black==24.*]
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: "v4.6.0"
+    hooks:
+      - id: check-added-large-files
+      - id: check-case-conflict
+      - id: check-merge-conflict
+      - id: check-symlinks
+      - id: check-yaml
+      - id: debug-statements
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+      - id: name-tests-test
+        args: ["--pytest-test-first"]
+      - id: requirements-txt-fixer
+      - id: trailing-whitespace
+
+  - repo: https://github.com/pre-commit/pygrep-hooks
+    rev: "v1.10.0"
+    hooks:
+      - id: rst-backticks
+      - id: rst-directive-colons
+      - id: rst-inline-touching-normal
+
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: "v3.1.0"
+    hooks:
+      - id: prettier
+        types_or: [yaml, markdown, html, css, scss, javascript, json]
+        args: [--prose-wrap=always]
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: "v0.5.2"
+    hooks:
+      - id: ruff
+        args: ["--fix", "--show-fixes"]
+      - id: ruff-format
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: "v1.10.1"
+    hooks:
+      - id: mypy
+        files: src|tests
+        args: ["--ignore-missing-imports"]
+        additional_dependencies:
+          - pytest
+          - types-PyYAML
+
+  - repo: https://github.com/codespell-project/codespell
+    rev: "v2.3.0"
+    hooks:
+      - id: codespell
+
+  - repo: https://github.com/shellcheck-py/shellcheck-py
+    rev: "v0.10.0.1"
+    hooks:
+      - id: shellcheck
+
+  - repo: local
+    hooks:
+      - id: disallow-caps
+        name: Disallow improper capitalization
+        language: pygrep
+        entry: PyBind|Numpy|Cmake|CCache|Github|PyTest
+        exclude: .pre-commit-config.yaml
+
+  - repo: https://github.com/abravalheri/validate-pyproject
+    rev: "v0.18"
+    hooks:
+      - id: validate-pyproject
+        additional_dependencies: ["validate-pyproject-schema-store[all]"]
+
+  - repo: https://github.com/python-jsonschema/check-jsonschema
+    rev: "0.29.0"
+    hooks:
+      - id: check-dependabot
+      - id: check-github-workflows
+      - id: check-readthedocs

CHANGELOG.md

@@ -1,35 +1,55 @@
 # Change log
 
 # dev
-- if `-temp-dir` used in wsclean then imaging products are produced here and then copied over to the same directory as the MS. This is intended to make use of compute nodes and fast local storage, like memory tmpfs or local disks.
+
+- if `-temp-dir` used in wsclean then imaging products are produced here and
+  then copied over to the same directory as the MS. This is intended to make use
+  of compute nodes and fast local storage, like memory tmpfs or local disks.
 - added stokes-v imaging. This includes a couple of minor / largish changes
-    - the `name` attribute of `WSCleanOptions` is not recognised
-    - the `pol` attribute of `WSCleanOptions` now needs to be a `str`
-    - the `-name` CLI argument of `wsclean` is auto-generated and always provided, and will now always contain the `pol` values (i.e. has a `polVALUE` field in output filenames)
-    - the strategy format now has a `operations` set of keywords, including `stokesv` to drawn options from
-    - naming format of output linmos files could contain the pol field
-    - `stokesv` imaging will not linmos the cleaning residuals together, even if the `--linmos-residuals` CLI is provided
-- Capture `CleanDivergenceError` from `wsclean` and rerun with larger image size and lower gain
-- Added `flint.catalogue`, which aims to collect all the catalogue related operations
+  - the `name` attribute of `WSCleanOptions` is not recognised
+  - the `pol` attribute of `WSCleanOptions` now needs to be a `str`
+  - the `-name` CLI argument of `wsclean` is auto-generated and always provided,
+    and will now always contain the `pol` values (i.e. has a `polVALUE` field in
+    output filenames)
+  - the strategy format now has a `operations` set of keywords, including
+    `stokesv` to drawn options from
+  - naming format of output linmos files could contain the pol field
+  - `stokesv` imaging will not linmos the cleaning residuals together, even if
+    the `--linmos-residuals` CLI is provided
+- Capture `CleanDivergenceError` from `wsclean` and rerun with larger image size
+  and lower gain
+- Added `flint.catalogue`, which aims to collect all the catalogue related
+  operations
   - a `flint_catalogue` CLI program to:
     - download reference catalogues that are known and expected from vizier
     - verify reference catalogues conform to expectations
     - list the reference catalogues that are expected
+- Added `flint.leakage` CLI program to attempt to characterise leakage over
+  polarisations, e.g. V/I
 
 # 0.2.5
+
 - added in skip rounds for masking and selfcal
 - Basic handling of CASDA measurement sets (preprocessing)
-- Basic handling of environment variables in Options, only supported in WSCleanOptions (no need for others yet)
-- basic renaming of MS and shuffling column names in place of straight up copying the MS
-- added CLI argument `--fixed-beam-shape` to specify a fixed final resolution, overwritng the optimal beam shape that otherwise would be computed
-- Added a SlurmInfo class to help with debugging crashed jobs. Primative and likely to change.
-- Made the `calibrated_bandpass_path` and optional CLI argument so that CASDA MSs can be better handled
+- Basic handling of environment variables in Options, only supported in
+  WSCleanOptions (no need for others yet)
+- basic renaming of MS and shuffling column names in place of straight up
+  copying the MS
+- added CLI argument `--fixed-beam-shape` to specify a fixed final resolution,
+  overwritng the optimal beam shape that otherwise would be computed
+- Added a SlurmInfo class to help with debugging crashed jobs. Primitive and
+  likely to change.
+- Made the `calibrated_bandpass_path` and optional CLI argument so that CASDA
+  MSs can be better handled
 - copying folders in `copy_files_into` when archiving
-- added reading of ArchiveOptions from strategy file for continuum pipeline and `flint_archive`
+- added reading of ArchiveOptions from strategy file for continuum pipeline and
+  `flint_archive`
 - Adaptive colour bar scaling in the rms validation plot
-- Create multiple linmos images if `--fixed-beam-shape` specified, one at an optimal common resolution and another at the specified resolution
+- Create multiple linmos images if `--fixed-beam-shape` specified, one at an
+  optimal common resolution and another at the specified resolution
 - Dump the `FieldOptions` to the output science directory
-- Weights produced by `linmos` are also trimmed in the same way as the corresponding image
+- Weights produced by `linmos` are also trimmed in the same way as the
+  corresponding image
 
 ## 0.2.4