Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: Batch storage split_url to work with Windows paths #1043

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions docs/CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,20 @@ To view the code coverage report in HTML format:
nox -rs coverage -- html && open ./htmlcov/index.html
```

### Platform-specific Testing

To mark a test as platform-specific, use the `@pytest.mark.<platform>` decorator:

```python
import pytest

@pytest.mark.windows
def test_windows_only():
pass
```

Supported platform markers are `windows`, `darwin`, and `linux`.

## Testing Updates to Docs

Documentation runs on Sphinx, using ReadtheDocs style template, and hosting from
Expand Down
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ known_first_party = ["tests", "samples"]
addopts = '-vvv --ignore=singer_sdk/helpers/_simpleeval.py -m "not external"'
markers = [
"external: Tests relying on external resources",
"windows: Tests that only run on Windows",
]

[tool.commitizen]
Expand Down
9 changes: 8 additions & 1 deletion singer_sdk/helpers/_batch.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from __future__ import annotations

import enum
import platform
from contextlib import contextmanager
from dataclasses import asdict, dataclass, field
from typing import IO, TYPE_CHECKING, Any, ClassVar, Generator
Expand Down Expand Up @@ -129,7 +130,13 @@ def split_url(url: str) -> tuple[str, str]:
Returns:
A tuple of the head and tail parts of the URL.
"""
return fs.path.split(url)
if platform.system() == "Windows" and "\\" in url:
# Original code from pyFileSystem split
# Augemnted slitly to properly Windows paths
split = url.rsplit("\\", 1)
return (split[0] or "\\", split[1])
else:
return fs.path.split(url)

@classmethod
def from_url(cls, url: str) -> StorageTarget:
Expand Down
10 changes: 10 additions & 0 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,14 @@

import os
import pathlib
import platform
import shutil

import pytest
from _pytest.config import Config

SYSTEMS = {"linux", "darwin", "windows"}


def pytest_collection_modifyitems(config: Config, items: list[pytest.Item]):
rootdir = pathlib.Path(config.rootdir)
Expand All @@ -21,6 +24,13 @@ def pytest_collection_modifyitems(config: Config, items: list[pytest.Item]):
item.add_marker("external")


def pytest_runtest_setup(item):
supported_systems = SYSTEMS.intersection(mark.name for mark in item.iter_markers())
system = platform.system().lower()
if supported_systems and system not in supported_systems:
pytest.skip(f"cannot run on platform {system}")


@pytest.fixture(scope="class")
def outdir() -> str:
"""Create a temporary directory for cookiecutters and target output."""
Expand Down
32 changes: 32 additions & 0 deletions tests/core/test_batch.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,3 +47,35 @@ def test_storage_from_url(file_url: str, root: str):
head, _ = StorageTarget.split_url(file_url)
target = StorageTarget.from_url(head)
assert target.root == root


@pytest.mark.parametrize(
"file_url,expected",
[
pytest.param(
"file:///Users/sdk/path/to/file",
("file:///Users/sdk/path/to", "file"),
id="local",
),
pytest.param(
"s3://bucket/path/to/file",
("s3://bucket/path/to", "file"),
id="s3",
),
pytest.param(
"file://C:\\Users\\sdk\\path\\to\\file",
("file://C:\\Users\\sdk\\path\\to", "file"),
marks=(pytest.mark.windows,),
id="windows-local",
),
pytest.param(
"file://\\\\remotemachine\\C$\\batches\\file",
("file://\\\\remotemachine\\C$\\batches", "file"),
marks=(pytest.mark.windows,),
id="windows-local",
),
],
)
def test_storage_split_url(file_url: str, expected: tuple):
"""Test storage target split URL."""
assert StorageTarget.split_url(file_url) == expected