Provide dataset summary info on upload

PBENCH-1204 This stems from a request from the UI to help optimize a partial refresh after using the relay upload dialog, by providing identification of the new dataset. Although a client using the traditional `PUT /upload` already knows the name and MD5, the returned information may be helpful. This is a DRAFT partly because I added a validation of the new information to the functional test, which I've renamed in #3473 ... I'll do that merge here after it's gone in. It's also DRAFT because while I like the idea of including URIs (and in particular this addresses a certain request regarding accessibility of the tarball), I'm not really sure which URIs to include or in what form. I'm certain that one or two people might possibly have opinions on this subject!
distributed-system-analysis · Jul 6, 2023 · 66c8c5a · 66c8c5a
1 parent 61be9df
commit 66c8c5a
Show file tree

Hide file tree

Showing 6 changed files with 70 additions and 11 deletions.
diff --git a/lib/pbench/server/api/resources/intake_base.py b/lib/pbench/server/api/resources/intake_base.py
@@ -500,6 +500,21 @@ def _intake(
                 except Exception as e:
                     current_app.logger.warning("Error removing {}: {}", tmp_dir, str(e))
 
-        response = jsonify(dict(message="File successfully uploaded"))
+        prefix = current_app.server_config.rest_uri
+        origin = (
+            f"{self._get_uri_base(request).host}{prefix}/datasets/{dataset.resource_id}"
+        )
+
+        response = jsonify(
+            {
+                "message": "File successfully uploaded",
+                "name": dataset.name,
+                "resource_id": dataset.resource_id,
+                "uris": {
+                    "tarball": origin + "/inventory/",
+                    "visualize": origin + "/visualize",
+                },
+            }
+        )
         response.status_code = HTTPStatus.CREATED
         return response
diff --git a/lib/pbench/server/api/resources/query_apis/datasets/datasets_contents.py b/lib/pbench/server/api/resources/query_apis/datasets/datasets_contents.py
@@ -201,7 +201,8 @@ def postprocess(self, es_json: JSON, context: ApiContext) -> JSON:
                 f"No directory {target!r} in {resource_id!r} contents.",
             )
 
-        origin = f"{self._get_uri_base(request).host}/datasets/{resource_id}"
+        prefix = current_app.server_config.rest_uri
+        origin = f"{self._get_uri_base(request).host}{prefix}/datasets/{resource_id}"
 
         dir_list = []
         file_list = []

diff --git a/lib/pbench/test/functional/server/test_datasets.py b/lib/pbench/test/functional/server/test_datasets.py
@@ -61,12 +61,27 @@ def test_upload_all(self, server_client: PbenchServerClient, login_user):
                 metadata = None
 
             cur_access = 0 if cur_access else 1
-            tarballs[Dataset.stem(t)] = Tarball(t, a)
+            name = Dataset.stem(t)
+            md5 = Dataset.md5(t)
+            tarballs[name] = Tarball(t, a)
             response = server_client.upload(t, access=a, metadata=metadata)
             assert (
                 response.status_code == HTTPStatus.CREATED
             ), f"upload returned unexpected status {response.status_code}, {response.text} ({t})"
-            print(f"\t... uploaded {t.name}: {a}")
+            assert response.json() == {
+                "message": "File successfully uploaded",
+                "name": name,
+                "resource_id": md5,
+                "uris": {
+                    "tarball": server_client._uri(
+                        API.DATASETS_INVENTORY, {"dataset": md5, "target": ""}
+                    ),
+                    "visualize": server_client._uri(
+                        API.DATASETS_VISUALIZE, {"dataset": md5}
+                    ),
+                },
+            }
+            print(f"\t... uploaded {name}: {a}")
 
         datasets = server_client.get_list(
             metadata=["dataset.access", "server.tarball-path", "dataset.operations"]

diff --git a/lib/pbench/test/unit/server/query_apis/test_datasets_contents.py b/lib/pbench/test/unit/server/query_apis/test_datasets_contents.py
@@ -162,7 +162,7 @@ def test_query(
                 "directories": [
                     {
                         "name": "sample1",
-                        "uri": "https://localhost/datasets/random_md5_string1/contents/1-default/sample1",
+                        "uri": "https://localhost/api/v1/datasets/random_md5_string1/contents/1-default/sample1",
                     }
                 ],
                 "files": [
@@ -173,7 +173,7 @@ def test_query(
                         "mode": "0o777",
                         "type": "sym",
                         "linkpath": "sample1",
-                        "uri": "https://localhost/datasets/random_md5_string1/inventory/1-default/reference-result",
+                        "uri": "https://localhost/api/v1/datasets/random_md5_string1/inventory/1-default/reference-result",
                     }
                 ],
             }
@@ -280,7 +280,7 @@ def test_subdirectory_query(
                 "directories": [
                     {
                         "name": "sample1",
-                        "uri": "https://localhost/datasets/random_md5_string1/contents/1-default/sample1",
+                        "uri": "https://localhost/api/v1/datasets/random_md5_string1/contents/1-default/sample1",
                     }
                 ],
                 "files": [],
@@ -367,7 +367,7 @@ def test_files_query(
                         "size": 122,
                         "mode": "0o644",
                         "type": "reg",
-                        "uri": "https://localhost/datasets/random_md5_string1/inventory/1-default/default.csv",
+                        "uri": "https://localhost/api/v1/datasets/random_md5_string1/inventory/1-default/default.csv",
                     }
                 ],
             }

diff --git a/lib/pbench/test/unit/server/test_relay.py b/lib/pbench/test/unit/server/test_relay.py
@@ -95,6 +95,7 @@ def test_relay(self, client, server_config, pbench_drb_token, tarball):
         relay manifest referencing a secondary relay URI containing a tarball.
         """
         file, md5file, md5 = tarball
+        name = Dataset.stem(file)
         responses.add(
             responses.GET,
             "https://relay.example.com/uri1",
@@ -122,6 +123,15 @@ def test_relay(self, client, server_config, pbench_drb_token, tarball):
         assert (
             response.status_code == HTTPStatus.CREATED
         ), f"Unexpected result, {response.text}"
+        assert response.json == {
+            "message": "File successfully uploaded",
+            "name": name,
+            "resource_id": md5,
+            "uris": {
+                "tarball": f"https://localhost/api/v1/datasets/{md5}/inventory/",
+                "visualize": f"https://localhost/api/v1/datasets/{md5}/visualize",
+            },
+        }
 
         audit = Audit.query()
         assert len(audit) == 2
@@ -132,7 +142,7 @@ def test_relay(self, client, server_config, pbench_drb_token, tarball):
         assert audit[0].name == "relay"
         assert audit[0].object_type == AuditType.DATASET
         assert audit[0].object_id == md5
-        assert audit[0].object_name == Dataset.stem(file)
+        assert audit[0].object_name == name
         assert audit[0].user_id == DRB_USER_ID
         assert audit[0].user_name == "drb"
         assert audit[0].reason is None

diff --git a/lib/pbench/test/unit/server/test_upload.py b/lib/pbench/test/unit/server/test_upload.py
@@ -410,6 +410,7 @@ def test_upload(self, client, pbench_drb_token, server_config, tarball):
         information.
         """
         datafile, _, md5 = tarball
+        name = Dataset.stem(datafile)
         with datafile.open("rb") as data_fp:
             response = client.put(
                 self.gen_uri(server_config, datafile.name),
@@ -419,7 +420,15 @@ def test_upload(self, client, pbench_drb_token, server_config, tarball):
             )
 
         assert response.status_code == HTTPStatus.CREATED, repr(response.text)
-        name = Dataset.stem(datafile)
+        assert response.json == {
+            "message": "File successfully uploaded",
+            "name": name,
+            "resource_id": md5,
+            "uris": {
+                "tarball": f"https://localhost/api/v1/datasets/{md5}/inventory/",
+                "visualize": f"https://localhost/api/v1/datasets/{md5}/visualize",
+            },
+        }
 
         dataset = Dataset.query(resource_id=md5)
         assert dataset is not None
@@ -658,6 +667,7 @@ def test_upload_nometa(self, client, pbench_drb_token, server_config, tarball):
         """Test a successful upload of a dataset without metadata.log."""
         datafile, _, md5 = tarball
         TestUpload.create_metadata = False
+        name = Dataset.stem(datafile)
         with datafile.open("rb") as data_fp:
             response = client.put(
                 self.gen_uri(server_config, datafile.name),
@@ -667,7 +677,15 @@ def test_upload_nometa(self, client, pbench_drb_token, server_config, tarball):
             )
 
         assert response.status_code == HTTPStatus.CREATED, repr(response.data)
-        name = Dataset.stem(datafile)
+        assert response.json == {
+            "message": "File successfully uploaded",
+            "name": name,
+            "resource_id": md5,
+            "uris": {
+                "tarball": f"https://localhost/api/v1/datasets/{md5}/inventory/",
+                "visualize": f"https://localhost/api/v1/datasets/{md5}/visualize",
+            },
+        }
 
         dataset = Dataset.query(resource_id=md5)
         assert dataset is not None