Skip to content

Commit

Permalink
Rebase and cleanup up
Browse files Browse the repository at this point in the history 
Improve `contents` functional test to verify the URI values. Use `location`
response header to return the tarball URI in upload, and verify that.
  • Loading branch information
@dbutenhof
dbutenhof committed Jul 6, 2023
1 parent 66c8c5a commit 91debda
Show file tree
Hide file tree
Showing 5 changed files with 97 additions and 36 deletions.
22 changes: 12 additions & 10 deletions lib/pbench/server/api/resources/intake_base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,9 @@ def _intake(
username: Optional[str] = None
tmp_dir: Optional[Path] = None

prefix = current_app.server_config.rest_uri
origin = f"{self._get_uri_base(request).host}{prefix}/datasets/"

try:
try:
authorized_user = Auth.token_auth.current_user()
Expand Down Expand Up @@ -284,7 +287,14 @@ def _intake(
f"Duplicate dataset {intake.md5!r} ({dataset_name!r}) is missing"
) from e
else:
response = jsonify(dict(message="Dataset already exists"))
response = jsonify(
{
"message": "Dataset already exists",
"name": dataset_name,
"resource_id": intake.md5,
}
)
response.headers["location"] = f"{origin}{intake.md5}/inventory/"
response.status_code = HTTPStatus.OK
return response
except APIAbort:
Expand Down Expand Up @@ -500,21 +510,13 @@ def _intake(
except Exception as e:
current_app.logger.warning("Error removing {}: {}", tmp_dir, str(e))

prefix = current_app.server_config.rest_uri
origin = (
f"{self._get_uri_base(request).host}{prefix}/datasets/{dataset.resource_id}"
)

response = jsonify(
{
"message": "File successfully uploaded",
"name": dataset.name,
"resource_id": dataset.resource_id,
"uris": {
"tarball": origin + "/inventory/",
"visualize": origin + "/visualize",
},
}
)
response.headers["location"] = f"{origin}{dataset.resource_id}/inventory/"
response.status_code = HTTPStatus.CREATED
return response
5 changes: 3 additions & 2 deletions lib/pbench/server/api/resources/query_apis/datasets/datasets_contents.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,19 +203,20 @@ def postprocess(self, es_json: JSON, context: ApiContext) -> JSON:

prefix = current_app.server_config.rest_uri
origin = f"{self._get_uri_base(request).host}{prefix}/datasets/{resource_id}"
path = "" if target == "/" else target

dir_list = []
file_list = []
for val in es_json["hits"]["hits"]:
if val["_source"]["directory"] == target:
# Retrieve files list if present else add an empty list.
for f in val["_source"].get("files", []):
f["uri"] = f"{origin}/inventory{target}/{f['name']}"
f["uri"] = f"{origin}/inventory{path}/{f['name']}"
file_list.append(f)
elif val["_source"]["parent"] == target:
name = val["_source"]["name"]
dir_list.append(
{"name": name, "uri": f"{origin}/contents{target}/{name}"}
{"name": name, "uri": f"{origin}/contents{path}/{name}"}
)

return {"directories": dir_list, "files": file_list}
63 changes: 52 additions & 11 deletions lib/pbench/test/functional/server/test_datasets.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,15 +72,10 @@ def test_upload_all(self, server_client: PbenchServerClient, login_user):
"message": "File successfully uploaded",
"name": name,
"resource_id": md5,
"uris": {
"tarball": server_client._uri(
API.DATASETS_INVENTORY, {"dataset": md5, "target": ""}
),
"visualize": server_client._uri(
API.DATASETS_VISUALIZE, {"dataset": md5}
),
},
}
assert response.headers["location"] == server_client._uri(
API.DATASETS_INVENTORY, {"dataset": md5, "target": ""}
)
print(f"\t... uploaded {name}: {a}")

datasets = server_client.get_list(
Expand Down Expand Up @@ -109,10 +104,20 @@ def test_upload_again(self, server_client: PbenchServerClient, login_user):
but with an OK (200) response instead of CREATED (201)
"""
duplicate = next(iter(TARBALL_DIR.glob("*.tar.xz")))
name = Dataset.stem(duplicate)
md5 = Dataset.md5(duplicate)
response = server_client.upload(duplicate)
assert (
response.status_code == HTTPStatus.OK
), f"upload returned unexpected status {response.status_code}, {response.text}"
assert response.json() == {
"message": "Dataset already exists",
"name": name,
"resource_id": md5,
}
assert response.headers["location"] == server_client._uri(
API.DATASETS_INVENTORY, {"dataset": md5, "target": ""}
)

@staticmethod
def test_bad_md5(server_client: PbenchServerClient, login_user):
Expand Down Expand Up @@ -149,11 +154,20 @@ def test_archive_only(server_client: PbenchServerClient, login_user):
validate that it doesn't get enabled for unpacking or indexing.
"""
tarball = SPECIAL_DIR / "fio_mock_2020.01.19T00.18.06.tar.xz"
name = Dataset.stem(tarball)
md5 = Dataset.md5(tarball)
response = server_client.upload(tarball, metadata={"server.archiveonly:y"})
assert (
response.status_code == HTTPStatus.CREATED
), f"upload {Dataset.stem(tarball)} returned unexpected status {response.status_code}, {response.text}"
), f"upload {name} returned unexpected status {response.status_code}, {response.text}"
assert response.json() == {
"message": "File successfully uploaded",
"name": name,
"resource_id": md5,
}
assert response.headers["location"] == server_client._uri(
API.DATASETS_INVENTORY, {"dataset": md5, "target": ""}
)
metadata = server_client.get_metadata(
md5, ["dataset.operations", "server.archiveonly"]
)
Expand All @@ -180,11 +194,20 @@ def test_no_metadata(server_client: PbenchServerClient, login_user):
validate that it doesn't get enabled for unpacking or indexing.
"""
tarball = SPECIAL_DIR / "nometadata.tar.xz"
name = Dataset.stem(tarball)
md5 = Dataset.md5(tarball)
response = server_client.upload(tarball)
assert (
response.status_code == HTTPStatus.CREATED
), f"upload {Dataset.stem(tarball)} returned unexpected status {response.status_code}, {response.text}"
), f"upload {name} returned unexpected status {response.status_code}, {response.text}"
assert response.json() == {
"message": "File successfully uploaded",
"name": name,
"resource_id": md5,
}
assert response.headers["location"] == server_client._uri(
API.DATASETS_INVENTORY, {"dataset": md5, "target": ""}
)
metadata = server_client.get_metadata(
md5, ["dataset.operations", "dataset.metalog", "server.archiveonly"]
)
Expand Down Expand Up @@ -545,9 +568,27 @@ def test_contents(self, server_client: PbenchServerClient, login_user):
archive = dataset.metadata["server.archiveonly"]
assert json["directories"] or json["files"] or archive

# Unless archiveonly, we need a metadata.log
# Even if they're empty, both values must be lists
assert isinstance(json["directories"], list)
assert isinstance(json["files"], list)

# Unless archiveonly, we need at least a metadata.log
assert archive or "metadata.log" in (f["name"] for f in json["files"])

for d in json["directories"]:
uri = server_client._uri(
API.DATASETS_CONTENTS,
{"dataset": dataset.resource_id, "target": d["name"]},
)
assert d["uri"] == uri, f"{d['name']} uri is incorrect: {d['uri']}"

for f in json["files"]:
uri = server_client._uri(
API.DATASETS_INVENTORY,
{"dataset": dataset.resource_id, "target": f["name"]},
)
assert f["uri"] == uri, f"{f['name']} uri is incorrect: {f['uri']}"

@pytest.mark.dependency(name="visualize", depends=["upload"], scope="session")
def test_visualize(self, server_client: PbenchServerClient, login_user):
"""Check that we can generate visualization data from a dataset
Expand Down
8 changes: 4 additions & 4 deletions lib/pbench/test/unit/server/test_relay.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,11 +127,11 @@ def test_relay(self, client, server_config, pbench_drb_token, tarball):
"message": "File successfully uploaded",
"name": name,
"resource_id": md5,
"uris": {
"tarball": f"https://localhost/api/v1/datasets/{md5}/inventory/",
"visualize": f"https://localhost/api/v1/datasets/{md5}/visualize",
},
}
assert (
response.headers["location"]
== f"https://localhost/api/v1/datasets/{md5}/inventory/"
)

audit = Audit.query()
assert len(audit) == 2
Expand Down
35 changes: 26 additions & 9 deletions lib/pbench/test/unit/server/test_upload.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -424,11 +424,11 @@ def test_upload(self, client, pbench_drb_token, server_config, tarball):
"message": "File successfully uploaded",
"name": name,
"resource_id": md5,
"uris": {
"tarball": f"https://localhost/api/v1/datasets/{md5}/inventory/",
"visualize": f"https://localhost/api/v1/datasets/{md5}/visualize",
},
}
assert (
response.headers["location"]
== f"https://localhost/api/v1/datasets/{md5}/inventory/"
)

dataset = Dataset.query(resource_id=md5)
assert dataset is not None
Expand Down Expand Up @@ -520,6 +520,15 @@ def test_upload_duplicate(self, client, server_config, pbench_drb_token, tarball
)

assert response.status_code == HTTPStatus.CREATED, repr(response)
assert response.json == {
"message": "File successfully uploaded",
"name": Dataset.stem(datafile),
"resource_id": md5,
}
assert (
response.headers["location"]
== f"https://localhost/api/v1/datasets/{md5}/inventory/"
)

# Reset manually since we upload twice in this test
TestUpload.cachemanager_created = None
Expand All @@ -532,7 +541,15 @@ def test_upload_duplicate(self, client, server_config, pbench_drb_token, tarball
)

assert response.status_code == HTTPStatus.OK, repr(response)
assert response.json.get("message") == "Dataset already exists"
assert response.json == {
"message": "Dataset already exists",
"name": Dataset.stem(datafile),
"resource_id": md5,
}
assert (
response.headers["location"]
== f"https://localhost/api/v1/datasets/{md5}/inventory/"
)

# We didn't get far enough to create a CacheManager
assert TestUpload.cachemanager_created is None
Expand Down Expand Up @@ -681,11 +698,11 @@ def test_upload_nometa(self, client, pbench_drb_token, server_config, tarball):
"message": "File successfully uploaded",
"name": name,
"resource_id": md5,
"uris": {
"tarball": f"https://localhost/api/v1/datasets/{md5}/inventory/",
"visualize": f"https://localhost/api/v1/datasets/{md5}/visualize",
},
}
assert (
response.headers["location"]
== f"https://localhost/api/v1/datasets/{md5}/inventory/"
)

dataset = Dataset.query(resource_id=md5)
assert dataset is not None
Expand Down

0 comments on commit 91debda

Please sign in to comment.