chore: Refresh tap (#69)

This was suggested as a project sample in
meltano/sdk#1588, so we might as well make it
a bit shinier:

- Drop support for Python 3.7
- Use Ruff
- Use pre-commit.ci
- Update dependencies

Related:

- meltano/sdk#2604

.github/workflows/ci_workflow.yml

@@ -12,46 +12,60 @@ jobs:
     strategy:
       matrix:
         # Only lint using the primary version used for dev
-        python-version: ["3.9"]
+        python-version: ["3.12"]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install Poetry
+    - name: Install tools
+      env:
+        PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
       run: |
         python -m pip install --upgrade pip
-        pip install poetry==1.2.*
+        pipx install poetry
+        pipx install tox
     - name: Install dependencies
       run: |
         poetry install
-    - name: Run lint command from tox.ini
+    - name: Lint
       run: |
-        poetry run tox -e lint
+        tox -e lint
 
   pytest:
 
     runs-on: ubuntu-latest
     env:
       GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
     strategy:
+      fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version:
+        - "3.13"
+        - "3.12"
+        - "3.11"
+        - "3.10"
+        - "3.9"
+        - "3.8"
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install Poetry
+        allow-prereleases: true
+    - name: Install tools
+      env:
+        PIP_CONSTRAINT: ${{ github.workspace }}/.github/workflows/constraints.txt
       run: |
         python -m pip install --upgrade pip
-        pip install poetry==1.2.*
+        pipx install poetry
     - name: Install dependencies
       run: |
+        poetry env use python${{ matrix.python-version }}
         poetry install
     - name: Test with pytest
       run: |

.github/workflows/constraints.txt

@@ -0,0 +1,3 @@
+pip==24.2
+poetry==1.8.3
+tox==4.18.0

.pre-commit-config.yaml

@@ -0,0 +1,27 @@
+ci:
+  autofix_prs: true
+  autofix_commit_msg: '[pre-commit.ci] auto fixes'
+  autoupdate_schedule: weekly
+  autoupdate_commit_msg: 'chore(deps): pre-commit autoupdate'
+
+repos:
+- repo: https://github.com/pre-commit/pre-commit
+  rev: v3.8.0
+  hooks:
+  - id: validate_manifest
+
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.6.0
+  hooks:
+  - id: check-json
+    exclude: "\\.vscode/.*.json"
+  - id: check-toml
+  - id: check-yaml
+  - id: end-of-file-fixer
+  - id: trailing-whitespace
+
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.5.7
+  hooks:
+  - id: ruff
+  - id: ruff-format

pyproject.toml

@@ -7,35 +7,35 @@ keywords = [
     "ELT",
     "CloudWatch",
 ]
-license = "Apache 2.0"
+license = "Apache-2.0"
 
 [tool.poetry.dependencies]
-python = "<3.11,>=3.7.1"
-requests = "^2.28.2"
-singer-sdk = "^0"
-fs-s3fs = { version = "^1.1.1", optional = true}
-boto3 = "^1.33.13"
-pytz = "^2024.1"
-
-[tool.poetry.extras]
-s3 = ["fs-s3fs"]
+python = ">=3.8"
+boto3 = "~=1.33.13"
+singer-sdk = "~=0.39.0"
+pytz = "==2024.1"
 
 [tool.poetry.group.dev.dependencies]
-pytest = "^6.2.5"
-tox = "^3.28.0"
-flake8 = "^3.9.2"
-pydocstyle = "^6.3.0"
-mypy = "^0.910"
-types-requests = "^2.28.11"
-isort = "^5.11.5"
-freezegun = "^1.2.2"
-black = "^22.12.0"
 coverage = "^7.2.1"
+freezegun = "^1.2.2"
+mypy = ">=1"
+pytest = ">=8"
+ruff = ">=0.5.7"
+types-pytz = "^2024.1.0.20240417"
 
-[tool.isort]
-profile = "black"
-multi_line_output = 3 # Vertical Hanging Indent
-src_paths = "tap_cloudwatch"
+[tool.ruff]
+line-length = 88
+
+[tool.ruff.lint]
+select = [
+    "B",   # flake8-bugbear
+    "E",   # pycodestyle errors
+    "F",   # Pyflakes
+    "FA",  # flake8-future-annotations
+    "I",   # isort
+    "UP",  # pyupgrade
+    "W",   # pycodestyle warnings
+]
 
 [build-system]
 requires = ["poetry-core>=1.0.8"]

tap_cloudwatch/client.py

@@ -1,11 +1,16 @@
 """Custom client handling, including CloudWatchStream base class."""
 
-from typing import Iterable, Optional
+from __future__ import annotations
+
+import typing as t
 
 from singer_sdk.streams import Stream
 
 from tap_cloudwatch.cloudwatch_api import CloudwatchAPI
 
+if t.TYPE_CHECKING:
+    from singer_sdk.helpers.types import Context
+
 
 class CloudWatchStream(Stream):
     """Stream class for CloudWatch streams."""
@@ -47,7 +52,7 @@ def check_sorted(self) -> bool:
         # `2023-02-20 06:01:57.009`. For that reason it is disabled.
         return False
 
-    def get_records(self, context: Optional[dict]) -> Iterable[dict]:
+    def get_records(self, context: Context | None) -> t.Iterable[dict]:
         """Return a generator of record-type dictionary objects.
 
         The optional `context` argument is used to identify a specific slice of the

tap_cloudwatch/cloudwatch_api.py

@@ -1,10 +1,11 @@
 """Class for interacting with Cloudwatch API."""
 
+from __future__ import annotations
+
 import os
 from collections import deque
 from datetime import datetime, timedelta, timezone
 from math import ceil
-from typing import Deque
 
 import boto3
 
@@ -106,7 +107,7 @@ def _get_completed_query(queue):
         return queue.popleft()
 
     def _iterate_batches(self, batch_windows, log_group, query):
-        queue: Deque["Subquery"] = deque()
+        queue: deque[Subquery] = deque()
 
         for start_ts, end_ts in batch_windows:
             if self._queue_is_full(queue):

tap_cloudwatch/streams.py

@@ -1,6 +1,6 @@
 """Stream type classes for tap-cloudwatch."""
 
-from typing import List
+from __future__ import annotations
 
 from singer_sdk import typing as th
 
@@ -11,13 +11,13 @@ class LogStream(CloudWatchStream):
     """Log stream."""
 
     name = "log"
-    primary_keys: List[str] = ["ptr"]
+    primary_keys: list[str] = ["ptr"]
     replication_key = "timestamp"
 
     @property
     def schema(self):
         """Dynamically detect the json schema for the stream."""
-        properties: List[th.Property] = []
+        properties: list[th.Property] = []
 
         # TODO: handle parse and unmask syntax
         # | parse @message "[*] *" as loggingType, loggingMessage

tap_cloudwatch/subquery.py

@@ -1,4 +1,7 @@
 """Class for managing a Subquery."""
+
+from __future__ import annotations
+
 import logging
 import time
 from datetime import datetime
@@ -23,11 +26,9 @@ def __init__(self, client, start_ts, end_ts, log_group, query):
     def execute(self):
         """Run the query."""
         self.logger.info(
-            (
-                "Submitting query for batch from:"
-                f" `{datetime.utcfromtimestamp(self.start_ts).isoformat()} UTC` -"
-                f" `{datetime.utcfromtimestamp(self.end_ts).isoformat()} UTC`"
-            )
+            "Submitting query for batch from:"
+            f" `{datetime.utcfromtimestamp(self.start_ts).isoformat()} UTC` -"
+            f" `{datetime.utcfromtimestamp(self.end_ts).isoformat()} UTC`"
         )
         start_query_response = self.client.start_query(
             logGroupName=self.log_group,
@@ -42,11 +43,9 @@ def execute(self):
     def get_results(self, prev_start=None):
         """Get results from query and recurse if needed."""
         self.logger.info(
-            (
-                "Retrieving results for batch from:"
-                f" `{datetime.utcfromtimestamp(self.start_ts).isoformat()} UTC` -"
-                f" `{datetime.utcfromtimestamp(self.end_ts).isoformat()} UTC`"
-            )
+            "Retrieving results for batch from:"
+            f" `{datetime.utcfromtimestamp(self.start_ts).isoformat()} UTC` -"
+            f" `{datetime.utcfromtimestamp(self.end_ts).isoformat()} UTC`"
         )
         response = None
         retry = True

tap_cloudwatch/tap.py

@@ -1,6 +1,6 @@
 """CloudWatch tap class."""
 
-from typing import List
+from __future__ import annotations
 
 from singer_sdk import Stream, Tap
 from singer_sdk import typing as th
@@ -109,7 +109,7 @@ class TapCloudWatch(Tap):
         ),
     ).to_dict()
 
-    def discover_streams(self) -> List[Stream]:
+    def discover_streams(self) -> list[Stream]:
         """Return a list of discovered streams."""
         return [stream_class(tap=self) for stream_class in STREAM_TYPES]
 

tap_cloudwatch/tests/test_cloudwatch_api.py → tests/test_cloudwatch_api.py

@@ -7,7 +7,8 @@
 
 from tap_cloudwatch.cloudwatch_api import CloudwatchAPI
 from tap_cloudwatch.exception import InvalidQueryException
-from tap_cloudwatch.tests.utils import datetime_from_str
+
+from .utils import datetime_from_str
 
 
 @pytest.mark.parametrize(

tap_cloudwatch/tests/test_core.py → tests/test_core.py

@@ -9,7 +9,8 @@
 
 from tap_cloudwatch.cloudwatch_api import CloudwatchAPI
 from tap_cloudwatch.tap import TapCloudWatch
-from tap_cloudwatch.tests.utils import datetime_from_str
+
+from .utils import datetime_from_str
 
 SAMPLE_CONFIG = {
     "log_group_name": "my_log_group_name",

tox.ini

@@ -2,55 +2,42 @@
 
 [tox]
 envlist = py38
-; envlist = py37, py38, py39
 isolated_build = true
 
 [testenv]
-whitelist_externals = poetry
+allowlist_externals = poetry
 
 commands =
     poetry install -v
     poetry run coverage run -m pytest --capture=no
-    poetry run coverage html -d tap_cloudwatch/tests/codecoverage
-    poetry run black --check tap_cloudwatch/
-    poetry run isort --check tap_cloudwatch
-    poetry run flake8 tap_cloudwatch
-    poetry run pydocstyle tap_cloudwatch
-    poetry run mypy tap_cloudwatch --exclude='tap_cloudwatch/tests'
+    poetry run coverage html -d tests/codecoverage
+    poetry run ruff check .
+    poetry run ruff format --check .
+    poetry run mypy . --exclude='tests'
 
 [testenv:pytest]
 # Run the python tests.
 # To execute, run `tox -e pytest`
-envlist = py37, py38, py39, py310
+envlist = py{38,39,310,311,312,313}
 commands =
     poetry install -v
     poetry run coverage run -m pytest --capture=no
-    poetry run coverage html -d tap_cloudwatch/tests/codecoverage
+    poetry run coverage html -d tests/codecoverage
 
 [testenv:format]
 # Attempt to auto-resolve lint errors before they are raised.
 # To execute, run `tox -e format`
 commands =
     poetry install -v
-    poetry run black tap_cloudwatch/
-    poetry run isort tap_cloudwatch
+    poetry run ruff check --fix .
+    poetry run ruff format .
 
 [testenv:lint]
 # Raise an error if lint and style standards are not met.
 # To execute, run `tox -e lint`
 commands =
     poetry install -v
-    poetry run black --check --diff tap_cloudwatch/
-    poetry run isort --check tap_cloudwatch
-    poetry run flake8 tap_cloudwatch
-    poetry run pydocstyle tap_cloudwatch
+    poetry run ruff format --check --diff .
+    poetry run ruff check --diff .
     # refer to mypy.ini for specific settings
-    poetry run mypy tap_cloudwatch --exclude='tap_cloudwatch/tests'
-
-[flake8]
-ignore = W503
-max-line-length = 88
-max-complexity = 10
-
-[pydocstyle]
-ignore = D105,D203,D213
+    poetry run mypy . --exclude='tests'