Skip to content

Commit

Permalink
Deal with empty and bad dataset names (#314)
Browse files Browse the repository at this point in the history
ServiceX no longer returns a *Fatal* error when it is asked to look up a bad dataset name or a dataset name that has zero files.

* Detect a completed transform with no files

Fixes #313
Fixes #312
  • Loading branch information
gordonwatts authored Sep 22, 2023
1 parent 9341628 commit c61bdad
Show file tree
Hide file tree
Showing 2 changed files with 39 additions and 3 deletions.
12 changes: 12 additions & 0 deletions servicex/servicex_adaptor.py
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,18 @@ async def get_transform_status(
f"Transform status for {request_id}" ' is marked "Fatal".'
)

if (
"status" in info
and info["status"] == "Complete"
and info["files-remaining"] is None
and info["files-processed"] == 0
):
raise ServiceXFatalTransformException(
f"Transform status for {request_id}"
' is marked "Complete" but no files were processed. Either the dataset '
"is empty, or the dataset name is invalid."
)

files_remaining = self._get_transform_stat(info, "files-remaining")
files_failed = self._get_transform_stat(info, "files-skipped")
files_processed = self._get_transform_stat(info, "files-processed")
Expand Down
30 changes: 27 additions & 3 deletions tests/test_servicex_adaptor.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,23 @@ def servicex_status_negative(mocker):
)


@pytest.fixture
def servicex_no_files(mocker):
"Returns a result when there are no files in the dataset"
result = {
"files-processed": 0,
"files-remaining": None,
"files-skipped": 0,
"request-id": "24e59fa2-e1d7-4831-8c7e-82b2efc7c658",
"stats": None,
"status": "Complete",
}
mocker.patch(
"aiohttp.ClientSession.get",
return_value=ClientSessionMocker(dumps(result), 200),
)


@pytest.fixture
def good_submit(mocker):
client = mocker.MagicMock()
Expand Down Expand Up @@ -191,7 +208,6 @@ async def test_status_no_auth(servicex_status_request):

@pytest.mark.asyncio
async def test_status_fatal_status(servicex_status_fatal):

sa = ServiceXAdaptor("http://localhost:500/sx")
async with aiohttp.ClientSession() as client:
with pytest.raises(ServiceXFatalTransformException) as e:
Expand All @@ -202,7 +218,6 @@ async def test_status_fatal_status(servicex_status_fatal):

@pytest.mark.asyncio
async def test_status_negative_status(servicex_status_negative):

sa = ServiceXAdaptor("http://localhost:500/sx")
async with aiohttp.ClientSession() as client:
with pytest.raises(ServiceXFatalTransformException) as e:
Expand All @@ -211,6 +226,16 @@ async def test_status_negative_status(servicex_status_negative):
assert "negative" in str(e.value)


@pytest.mark.asyncio
async def test_status_empty(servicex_no_files):
sa = ServiceXAdaptor("http://localhost:500/sx")
async with aiohttp.ClientSession() as client:
with pytest.raises(ServiceXFatalTransformException) as e:
await sa.get_transform_status(client, "123-123-123-444")

assert "empty" in str(e.value)


@pytest.mark.asyncio
async def test_status_with_auth(mocker):
client = mocker.MagicMock()
Expand Down Expand Up @@ -243,7 +268,6 @@ async def test_status_with_auth(mocker):

@pytest.mark.asyncio
async def test_status_unknown_request(servicex_status_unknown):

sa = ServiceXAdaptor("http://localhost:5000/sx")
with pytest.raises(ServiceXUnknownRequestID) as e:
async with aiohttp.ClientSession() as client:
Expand Down

0 comments on commit c61bdad

Please sign in to comment.