Update test framework and support msgspec serialization.

.github/workflows/main.yml

@@ -14,70 +14,98 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     runs-on: ubuntu-latest
 
     steps:
       - name: Checking out repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Install and upgrade dependencies
         run: |
-          python -m pip install -U pip setuptools wheel
-          python3 -m pip install -U .[dev]
+          python -m pip install -U pip poetry
+          python3 -m poetry install
 
-      - name: Analysing the code with pylint
-        run: pylint singer
+      - name: Analysing the code with tox tests
+        run: poetry run tox -e py
 
       - name: Runs tests with coverage
-        run: coverage run --parallel -m pytest
+        run: poetry run coverage run --parallel -m pytest
 
       - name: Upload coverage data
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: coverage-data
+          name: coverage-data-${{ matrix.python-version }}
           path: ".coverage.*"
 
   coverage:
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+      repository-projects: write
     needs: build
     steps:
     - name: Check out the repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: "3.8"
 
     - name: Install and upgrade dependencies
       run: |
-        python -m pip install -U pip setuptools wheel
-        python3 -m pip install -U .[dev]
+        python -m pip install -U pip poetry
+        python3 -m poetry install
 
     - name: Download coverage data
-      uses: actions/download-artifact@v3.0.0
+      uses: actions/download-artifact@v4
       with:
-        name: coverage-data
+        pattern: coverage-data-*
+        merge-multiple: true
 
     - name: Combine coverage data
       run: |
-        coverage combine
+        poetry run coverage combine
 
     - name: Generate XML coverage report
       run: |
-        coverage xml
+        poetry run coverage xml --fail-under=75
 
     - name: Display human readable report
       run: |
-        coverage report
+        poetry run coverage report --fail-under=75
+
+    - name: Code Coverage Summary Report
+      uses: irongut/[email protected]
+      with:
+        filename: coverage.xml
+        badge: true
+        fail_below_min: true
+        format: markdown
+        hide_branch_rate: false
+        hide_complexity: true
+        indicators: true
+        output: both
+        thresholds: '60 80'
+
+    - name: Add Coverage PR Comment
+      uses: marocchino/sticky-pull-request-comment@v2
+      if: github.event_name == 'pull_request'
+      with:
+        recreate: true
+        path: code-coverage-results.md
+
+    - name: Write to Job Summary
+      run: cat code-coverage-results.md >> $GITHUB_STEP_SUMMARY
 
     # Optional if you want to use codecov.io
     # - name: Upload coverage report
-    #   uses: codecov/codecov-action@v3
+    #   uses: codecov/codecov-action@v4

.github/workflows/pythonpublish.yml

@@ -2,25 +2,28 @@ name: Upload Python Package to PyPi
 
 on:
   release:
-    types: [created]
+    types: [published]
 
 jobs:
   deploy:
+    environment:
+      name: Production
+      url: https://pypi.org/p/realit-singer-python
+    permissions:
+      contents: read
+      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.x'
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install setuptools wheel twine
-    - name: Build and publish
-      env:
-        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-      run: |
-        python setup.py sdist bdist_wheel
-        twine upload dist/*
+        pip install build
+    - name: Build package
+      run: python -m build
+    - name: Publish package
+      uses: pypa/gh-action-pypi-publish@ec4db0b4ddc65acdf4bff5fa45ac92d78b56bdf0

CHANGELOG.md

@@ -1,5 +1,46 @@
 # Changelog
 
+## 5.0.3 (2024-08-19)
+ * Moving to a trusted push to pypi
+
+## 5.0.2 (2024-08-13)
+ * Bump black from 24.4.2 to 24.8.0
+ * Bump pylint from 3.2.5 to 3.2.6
+ * Bump mypy from 1.10.1 to 1.11.1
+ * Bump flake8 from 7.1.0 to 7.1.1 
+ * Bump actions/setup-python from 3 to 5
+ * Bump pypa/gh-action-pypi-publish from 1.4.2 to 1.9.0
+
+## 5.0.1 (2024-08-12)
+ * Updating the README.md to have the badges point to the correctly published version on PyPI.
+
+## 5.0.0 (2024-08-09)
+ * Renaming repo to realit-singer-python to allow required publishing to PyPI
+ * NOTE: Will need to urgently update tap-oracle, tap-mssql, tap-s3-csv which are dependent on this.
+
+## 4.0.1 (2024-08-08)
+ * Correcting Supported Python version constraint in pyproject.toml >=3.8,<3.13
+ * Bump pytest from 7.4.4 to 8.3.2
+ * Bump pytz from 2018.9 to 2024.1
+ * Bump jsonschema from 4.22.0 to 4.23.0
+ * Bump tox from 4.15.1 to 4.17.1 #35
+ * Bump coverage from 7.5.4 to 7.6.1 #33
+
+## 4.0.0 (2024-07-08)
+ * Moving from setup.py to pyproject.toml and poetry for installation
+ * Introducing tox for platform independent testing of pep8 compliance.
+ * Introducing black for formatting
+ * Introducing isort for compliant and order includes
+ * Introducing flake8 for linting alongside pylint
+ * Introducing mypy for type checking
+ * Updating github actions to run tox tests rather than directly running pytests.
+ * Code formatting updates and changes to pass / met the standards above.
+
+## 3.0.0 (2023-08-23)
+ * Using msgspec instead of orjson or other serializers for speed benefit
+ * Support for Python 3.12
+ * Deprecating Python 3.7
+
 ## 2.0.2 (2022-03-23)
  * Using orjson instead of simplejson or other serializers for speed benefit
  * Fix: Output decimal.Decimal as int or float not str

CONTRIBUTING.md

@@ -4,5 +4,5 @@
 
 1. Install and use [pre-commit](https://pre-commit.com/) to keep your changes in the style of the project.
 2. Write tests to cover any new code or code changes.
-3. Please make sure that all tests pass and that the code passes linting with `make`.
+3. Please make sure that all tests pass and that the code passes linting with `poetry run tox`.
 4. Open up the PR.

Makefile

@@ -2,7 +2,7 @@
 
 check_prereqs:
 	bash -c '[[ -n $$VIRTUAL_ENV ]]'
-	bash -c '[[ $$(python3 --version) == *3.[5-7]* ]]'
+	bash -c '[[ $$(python3 --version) == *3.[8-12]* ]]'
 
 install: check_prereqs
 	python3 -m pip install -e '.[dev]'

README.md

@@ -1,12 +1,12 @@
-pipelinewise-singer-python
+realit-singer-python
 ===================
-[![PyPI version](https://badge.fury.io/py/pipelinewise-singer-python.svg)](https://badge.fury.io/py/pipelinewise-singer-python)
-[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/pipelinewise-singer-python.svg)](https://pypi.org/project/pipelinewise-singer-python/)
+[![PyPI version](https://badge.fury.io/py/realit-singer-python.svg)](https://badge.fury.io/py/realit-singer-python)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/realit-singer-python.svg)](https://pypi.org/project/realit-singer-python/)
 [![License: MIT](https://img.shields.io/badge/License-Apache2-yellow.svg)](https://opensource.org/licenses/Apache-2.0)
 
 Writes the Singer format from Python.
 
-This is a fork of [Singer's singer-python](https://github.com/singer-io/singer-python) made for [PipelineWise](https://transferwise.github.io/pipelinewise).
+This is a fork of [Singer's singer-python](https://github.com/singer-io/singer-python) made for [PipelineWise](https://transferwise.github.io/pipelinewise) and [Meltano](https://meltano.com/)
 
 Usage
 ---
@@ -25,7 +25,7 @@ Next, install this library:
 source ~/.virtualenvs/singer-python/bin/activate
 git clone http://github.com/singer-io/singer-python
 cd singer-python
-make install
+pip install
 ```
 
 ### Usage example
@@ -44,10 +44,60 @@ singer.write_state({'my_table': 'd'})
 
 ### Logging configuration
 
-**pipelinewise-singer-python** by default doesn't use any predefined logging configuration, it's up to the calling 
-library to define it. However, if the environment variable `LOGGING_CONF_FILE` is found and set then the **pipelinewise-singer-python** 
+**realit-singer-python** by default doesn't use any predefined logging configuration, it's up to the calling 
+library to define it. However, if the environment variable `LOGGING_CONF_FILE` is found and set then the **realit-singer-python** 
 would use the path provided in the env variable as the logging configuration for the logger. 
 
+### Singer Decimal
+
+Enabling the use_singer_decimal = True in a tap will output **decimal** and **floats** as a string
+rather than their numeric representation.
+
+**Optional Setting**:
+
+A boolean setting: when enabled `true` in the config will outputs decimal and floating point numbers as strings to avoid loss of precision and scale.
+For supporting taps, there are hints in the schema message, format = "singer.decimal", and additionalProperties scale_precision dictionary providing precision and scale. For decimal data, the target can use this 
+information to correctly replicate decimal data without loss. For the Floats and Number data type without precision and scale it is recommended that post processing formats the datatype based on an inspection of the data because the true data size is unknown / dynamic.
+
+```json
+{
+  "use_singer_decimal": true,
+}
+```
+
+## Developer Resources
+
+### Initialize your Development Environment
+
+```bash
+pip install poetry
+poetry install
+```
+
+### Create and Run Tests
+
+Create tests within the `tests/` directory and
+then run:
+
+```bash
+poetry run pytest
+```
+
+or 
+
+```bash
+poetry run coverage run --parallel -m pytest
+```
+
+### Continuous Integration
+Run through the full suite of tests and linters by running
+
+```bash
+poetry run tox
+```
+
+These must pass in order for PR's to be merged.
+
 
 License
 -------