From ebcc9cff203c0fc0ac920266ef0574cdd917faad Mon Sep 17 00:00:00 2001 From: Gilles Filippini Date: Mon, 22 Feb 2021 08:01:56 +0000 Subject: [PATCH 1/2] API: add endpoint POST /v1/containers --- VERSION | 2 +- docs/_docs/client.md | 112 +++++++++++++++++++++++++---- shub/apps/library/views/helpers.py | 14 ++++ shub/apps/library/views/images.py | 104 +++++++++++++++++++++++---- 4 files changed, 205 insertions(+), 27 deletions(-) diff --git a/VERSION b/VERSION index ba7b2f76..c5676407 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.32 +1.1.33 diff --git a/docs/_docs/client.md b/docs/_docs/client.md index 04f5922a..b8324760 100644 --- a/docs/_docs/client.md +++ b/docs/_docs/client.md @@ -125,21 +125,14 @@ We use `-U` for unsigned. ### Push Logic + - If you push an image to a non existing collection, the collection will be created first (version `1.1.32`) + - If you push an image to a non existing container, the container will be created first (version `1.1.33`) (\*) + - If you push a new image, it will be added. + - If you push a new tag, it will be added. - If you push an existing tag, if the container is unfrozen, it will be replaced - If you push an existing tag and the container is frozen (akin to protected) you'll get permission denied. - - If you push a new tag, it will be added. - - If you push a new image, it will also be added. - - If you push an image to a non existing collection, the collection will be created first, then the image will be added (version `1.1.32`). - -Unlike the Sylabs API, when the GET endpoint is made to `v1/containers` and the image doesn't exist, -we return a response for the collection (and not 404). In other words, [this response](https://github.com/sylabs/scs-library-client/blob/acb520c8fe6456e4223af6fbece956449d790c79/client/push.go#L140) is always returned. We do this because -the Sylabs library client has a strange logic where it doesn't tag images until after the fact, -and doesn't send the user's requested tag to any of the get or creation endpoints. This means -that we are forced on the registry to create a dummy holder tag (that is guaranteed to be unique) -and then to find the container at the end to [set tags](https://github.com/sylabs/scs-library-client/blob/acb520c8fe6456e4223af6fbece956449d790c79/client/push.go#L187) based on the id of the image -that is created with the [upload request](https://github.com/sylabs/scs-library-client/blob/acb520c8fe6456e4223af6fbece956449d790c79/client/push.go#L174). I didn't see a logical way to create the container using the POST endpoint to -"v1/containers" given that we do not know the tag or version, and would need to know the exact container id -to return later when the container push is requested. + +(\*) The Singularity Registry Server data model is different from this of Sylabs library. In the former, a container is created from an image. In the latter, containers are just placeholders for images with the same name, and can be empty. Hence Singularity Registry Server just [mimics the container creation without actually doing it](#mimic-empty-container-creation). The container is created afterwards at the upload step. ### Push Size @@ -328,7 +321,7 @@ As of version `1.1.32` it is possible to create a new collection via the API. It First retrieve the numeric `id` associated with your username with a GET request to the endpoint `/v1/entities/`. ```bash -$ curl -s -H 'Authorization: Bearer ' /v1/entities/ +$ curl -s -H 'Authorization: Bearer ' http://127.0.0.1/v1/entities/ ``` Here is a response made pretty by piping into json_pp: ``` @@ -396,3 +389,94 @@ You can then see the response that the collection was created, and it will appea The `private` key is optional. If not provided, it defaults to the servers's configured default for collection creation. In case of a `singularity push` to a non existing collection, the client triggers the collection creation first, using this endpoint, then pushes the image. + +## Mimic empty container creation + +As of version `1.1.33` it is possible to mimic the Sylabs library endpoint for new container creation. It requires authentication. + +First retrieve the numeric `id` associated with the holding collection with a GET request to the endpoint `/v1/collections//`, where `` is your username. + +```bash +$ curl -s -H 'Authorization: Bearer ' http://127.0.0.1/v1/collections// +``` + +This is the value associated with the `id` key of the answer. For example: +``` +{ + "data": { + "deletedAt": "0001-01-01T00:00:00Z", + "entityName": "pini", + "deleted": false, + "name": "pini-private", + "owner": "1", + "size": 4, + "description": "My private collection", + "customData": "", + "entity": "1", + "updatedBy": "1", + "id": "6", + "createdBy": "1", + "private": true, + "createdAt": "2021-02-20T20:21:13.692215Z", + "containers": [ + "5", + "6", + "7", + "8" + ], + "updatedAt": "2021-02-20T20:21:13.777940Z" + } +} +``` + +Then we can issue a POST request to the endpoint `/v1/containers` with the payload: +``` +{ + "collection": "" + "name": "" +} +``` +```bash +$ curl -X POST -H 'Authorization: Bearer ' -H "Content-Type: application/json" --data '{"collection": 6, "name": "fish"}' http://127.0.0.1/v1/containers +``` + +There is no container creation, because it is not needed with the Singularity Registry Server data model. But the response is as if the container was created so that any workflow using this endpoint could work the same as with Sylabs library: + +```bash +{ + "data": { + "collection": "6", + "description": "My private collection", + "collectionName": "pini-private", + "owner": "1", + "readOnly": false, + "id": "6", + "size": 4, + "imageTags": {}, + "createdBy": "1", + "private": true, + "downloadCount": null, + "updatedAt": "2021-02-20T20:21:13.777940Z", + "fullDescription": "Test-private Collection", + "deletedAt": "0001-01-01T00:00:00Z", + "entityName": "pini", + "deleted": false, + "name": "test-private", + "stars": 0, + "customData": "", + "images": [], + "entity": "1", + "updatedBy": "1", + "archTags": { + "amd64": {} + }, + "createdAt": "2021-02-20T20:21:13.692215Z", + "containers": [ + "5", + "6", + "7", + "8" + ] + } +} +``` diff --git a/shub/apps/library/views/helpers.py b/shub/apps/library/views/helpers.py index cb3ed0e1..32bf8816 100644 --- a/shub/apps/library/views/helpers.py +++ b/shub/apps/library/views/helpers.py @@ -341,3 +341,17 @@ def get_collection(name, retry=True): if retry is True and "/" in name: name = name.split("/")[0] return get_collection(name, retry=False) + + +def get_collection_by_id(id, retry=True): + """get a collection by id. + + Parameters + ========== + id: the numerical id of the collection to look up + """ + try: + collection = Collection.objects.get(id=id) + return collection + except Collection.DoesNotExist: + return None diff --git a/shub/apps/library/views/images.py b/shub/apps/library/views/images.py index 21374e9a..737a5cf8 100644 --- a/shub/apps/library/views/images.py +++ b/shub/apps/library/views/images.py @@ -13,7 +13,7 @@ from sregistry.utils import parse_image_name from shub.apps.logs.utils import generate_log -from shub.apps.main.utils import format_collection_name +from shub.apps.main.utils import format_collection_name, format_container_name from shub.apps.main.models import Collection, Container from shub.settings import ( MINIO_BUCKET, @@ -47,6 +47,7 @@ get_token, get_container, get_collection, + get_collection_by_id, validate_token, ) @@ -708,18 +709,19 @@ def post(self, request, collection_id): class ContainersView(RatelimitMixin, APIView): - """Return a simple list of containers - GET /v1/containers - """ renderer_classes = (JSONRenderer,) ratelimit_key = "ip" ratelimit_rate = settings.VIEW_RATE_LIMIT ratelimit_block = settings.VIEW_RATE_LIMIT_BLOCK - ratelimit_method = "GET" + ratelimit_method = ("GET", "POST") renderer_classes = (JSONRenderer,) def get(self, request): + """Return a simple list of containers. + + GET /v1/containers + """ print("GET ContainersView") print(request.data) @@ -732,6 +734,79 @@ def get(self, request): # collections = generate_collections_list(token.user) return Response(data={}, status=200) + def post(self, request): + """Mimic the creation a new container. + + POST /v1/coontainers + + Body parameters: + * collection: collection numeric id as a string + * name: new container name + + Sylabs library has an optional 'private' parameter, which we + ignore here because containers' privacy is inherited from the + collection they belong to. + + Return the newly created container. + + This endpoint only mimics the Sylabs library's one, without actually + creating the container object. Because in the sregistry data model a + container cannot exist with no images. + """ + + print("POST ContainersView") + if not validate_token(request): + message = {"error": {"code": 403, "message": "Token not valid"}} + return Response(message, status=403) + + # body should have {'collection': collection_id, 'name': new_container_name} + # 'private' is optional and unused + # {"collection": "22", "name": "my_container"} + body = json.loads(request.body.decode("utf-8")) + if not ("collection" in body and "name" in body): + message = {"error": {"code": 400, "message": "Invalid payload."}} + return Response(message, status=400) + + try: + collection_id = int(body["collection"]) + except ValueError: + message = {"error": {"code": 400, "message": "Invalid payload."}} + return Response(message, status=400) + + # check permissions + # return 403 when collection does not exist or user is not an owner + collection = Collection.objects.get(id=collection_id) + token = get_token(request) + if (not collection) or (token.user not in collection.owners.all()): + message = { + "error": { + "code": 403, + "message": "Permission denied {0} {1}".format( + token.user.id, body["entity"] + ), + } + } + return Response(message, status=403) + + # does a container with the same name exist already? + name = format_container_name(body["name"]) + containers = collection.containers.filter(name=name) + + if containers: + message = { + "error": { + "code": 403, + "message": "A container named '{0}' exists already for collection id '{1}'!".format( + name, collection_id + ), + } + } + return Response(message, status=403) + + # We don't need to create the specific container here + details = generate_collection_details(collection, [], token.user) + return Response(data={"data": details}, status=200) + class GetNamedCollectionView(RatelimitMixin, APIView): """Given a collection, return the associated metadata. @@ -806,10 +881,15 @@ def get(self, request, username, name, container): return Response(status=403) # We don't need to create the specific container here - containers = collection.containers.filter(name=container) or [] - - # Even if the container doesn't exist, we return response that it does, - # And it's created in the next view. - - data = generate_collection_details(collection, containers, token.user) - return Response(data={"data": data}, status=200) + containers = collection.containers.filter(name=container) + if containers: + data = generate_collection_details(collection, containers, token.user) + return Response(data={"data": data}, status=200) + else: + message = { + "error": { + "code": 404, + "message": "Error retrieving container: not found", + } + } + return Response(message, status=404) From e43ecf2a1e067cdd99b023c4ead8bc7b79da0076 Mon Sep 17 00:00:00 2001 From: Gilles Filippini Date: Mon, 22 Feb 2021 17:58:12 +0000 Subject: [PATCH 2/2] Fixes from PR conversations --- shub/apps/library/views/helpers.py | 14 -------------- shub/apps/library/views/images.py | 13 +++++++++---- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/shub/apps/library/views/helpers.py b/shub/apps/library/views/helpers.py index 32bf8816..cb3ed0e1 100644 --- a/shub/apps/library/views/helpers.py +++ b/shub/apps/library/views/helpers.py @@ -341,17 +341,3 @@ def get_collection(name, retry=True): if retry is True and "/" in name: name = name.split("/")[0] return get_collection(name, retry=False) - - -def get_collection_by_id(id, retry=True): - """get a collection by id. - - Parameters - ========== - id: the numerical id of the collection to look up - """ - try: - collection = Collection.objects.get(id=id) - return collection - except Collection.DoesNotExist: - return None diff --git a/shub/apps/library/views/images.py b/shub/apps/library/views/images.py index 737a5cf8..e5921e98 100644 --- a/shub/apps/library/views/images.py +++ b/shub/apps/library/views/images.py @@ -47,7 +47,6 @@ get_token, get_container, get_collection, - get_collection_by_id, validate_token, ) @@ -737,7 +736,7 @@ def get(self, request): def post(self, request): """Mimic the creation a new container. - POST /v1/coontainers + POST /v1/containers Body parameters: * collection: collection numeric id as a string @@ -747,11 +746,14 @@ def post(self, request): ignore here because containers' privacy is inherited from the collection they belong to. - Return the newly created container. + Return new container's data (but it is not created actually). This endpoint only mimics the Sylabs library's one, without actually creating the container object. Because in the sregistry data model a container cannot exist with no images. + + It is provided to improve compatibility with other singularity + clients. """ print("POST ContainersView") @@ -775,8 +777,11 @@ def post(self, request): # check permissions # return 403 when collection does not exist or user is not an owner - collection = Collection.objects.get(id=collection_id) token = get_token(request) + try: + collection = Collection.objects.get(id=collection_id) + except Collection.DoesNotExist: + collection = None if (not collection) or (token.user not in collection.owners.all()): message = { "error": {