From 832a2353aae0fd7d76a4c97a6a51194a773a6a61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miha=20Ple=C5=A1ko?= Date: Mon, 14 Mar 2016 11:33:34 +0100 Subject: [PATCH] Implemented the API Refers: * https://trello.com/c/IScPloRZ/32-dopolnitev-api-za-deployment-service --- .../cfy_wrapper/migrations/0002_container.py | 27 +++ dice_deploy_django/cfy_wrapper/models.py | 34 +++ dice_deploy_django/cfy_wrapper/serializers.py | 10 +- dice_deploy_django/cfy_wrapper/urls.py | 11 + dice_deploy_django/cfy_wrapper/views.py | 103 ++++++--- dice_deploy_django/requirements.in | 1 + dice_deploy_django/requirements.txt | 15 +- dice_deploy_django/unit_tests/factories.py | 34 +++ .../unit_tests/test_cfy_wrapper_api.py | 199 ++++++++++++++++-- 9 files changed, 387 insertions(+), 47 deletions(-) create mode 100644 dice_deploy_django/cfy_wrapper/migrations/0002_container.py create mode 100644 dice_deploy_django/unit_tests/factories.py diff --git a/dice_deploy_django/cfy_wrapper/migrations/0002_container.py b/dice_deploy_django/cfy_wrapper/migrations/0002_container.py new file mode 100644 index 0000000..e2d4d16 --- /dev/null +++ b/dice_deploy_django/cfy_wrapper/migrations/0002_container.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.4 on 2016-03-10 09:41 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('cfy_wrapper', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='Container', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('blueprint', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='cfy_wrapper.Blueprint')), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/dice_deploy_django/cfy_wrapper/models.py b/dice_deploy_django/cfy_wrapper/models.py index 5ca636f..71a49c2 100644 --- a/dice_deploy_django/cfy_wrapper/models.py +++ b/dice_deploy_django/cfy_wrapper/models.py @@ -2,6 +2,7 @@ from django.db import models from rest_framework.exceptions import NotFound from enum import Enum +from django.db import IntegrityError class Base(models.Model): @@ -42,3 +43,36 @@ def cfy_id(self): @property def state_name(self): return Blueprint.State(self.state).name + + def pipe_deploy_blueprint(self): + """ Defines and starts async pipeline for deploying blueprint to cloudify """ + from cfy_wrapper import tasks + pipe = ( + tasks.upload_blueprint.si(self.cfy_id) | + tasks.create_deployment.si(self.cfy_id) | + tasks.install.si(self.cfy_id) + ) + pipe.apply_async() + + def pipe_undeploy_blueprint(self): + """ Defines and starts async pipeline for undeploying blueprint from cloudify """ + from cfy_wrapper import tasks + pipe = ( + tasks.uninstall.si(self.cfy_id) | + tasks.delete_deployment.si(self.cfy_id) | + tasks.delete_blueprint.si(self.cfy_id) + ) + pipe.apply_async() + + +class Container(Base): + # Fields + id = models.UUIDField( + primary_key=True, default=uuid.uuid4, editable=False + ) + blueprint = models.ForeignKey(Blueprint, null=True) + + def delete(self, using=None, keep_parents=False): + if self.blueprint is not None: + raise IntegrityError('Cannot delete container with existing blueprint') + super(Container, self).delete(using, keep_parents) diff --git a/dice_deploy_django/cfy_wrapper/serializers.py b/dice_deploy_django/cfy_wrapper/serializers.py index c8dfdcc..36975c0 100644 --- a/dice_deploy_django/cfy_wrapper/serializers.py +++ b/dice_deploy_django/cfy_wrapper/serializers.py @@ -1,9 +1,17 @@ from rest_framework import serializers -from .models import Blueprint +from .models import Blueprint, Container class BlueprintSerializer(serializers.ModelSerializer): class Meta: model = Blueprint fields = ("state_name", "cfy_id") + + +class ContainerSerializer(serializers.ModelSerializer): + blueprint = BlueprintSerializer() + + class Meta: + model = Container + fields = ("id", "blueprint") diff --git a/dice_deploy_django/cfy_wrapper/urls.py b/dice_deploy_django/cfy_wrapper/urls.py index 615fb76..d79d295 100644 --- a/dice_deploy_django/cfy_wrapper/urls.py +++ b/dice_deploy_django/cfy_wrapper/urls.py @@ -4,13 +4,24 @@ DebugView, BlueprintsView, BlueprintIdView, + ContainersView, + ContainerIdView, + ContainerBlueprint ) urlpatterns = [ url(r"^debug/?$", DebugView.as_view(), name="debug"), + # blueprint url(r"^blueprints/?$", BlueprintsView.as_view(), name="blueprints"), url(r"^blueprints/(?P[0-9a-f-]+)/?$", BlueprintIdView.as_view(), name="blueprint_id"), + # container + url(r"^containers/?$", + ContainersView.as_view(), name="containers"), + url(r"^containers/(?P[0-9a-f-]+)/?$", + ContainerIdView.as_view(), name="container_id"), + url(r"^containers/(?P[0-9a-f-]+)/blueprints?$", + ContainerBlueprint.as_view(), name="container_blueprint"), ] diff --git a/dice_deploy_django/cfy_wrapper/views.py b/dice_deploy_django/cfy_wrapper/views.py index 0337d7c..2ee12d6 100644 --- a/dice_deploy_django/cfy_wrapper/views.py +++ b/dice_deploy_django/cfy_wrapper/views.py @@ -3,10 +3,12 @@ from rest_framework.views import APIView from rest_framework.response import Response from rest_framework.parsers import MultiPartParser, FileUploadParser +from rest_framework import status +from django.db import IntegrityError from . import tasks -from .models import Blueprint -from .serializers import BlueprintSerializer +from .models import Blueprint, Container +from .serializers import BlueprintSerializer, ContainerSerializer logger = logging.getLogger("views") @@ -23,43 +25,90 @@ class BlueprintsView(APIView): def get(self, request): """ - # List all available blueprints + List all available blueprints """ s = BlueprintSerializer(Blueprint.objects.all(), many=True) return Response(s.data) - def put(self, request): - """ - # Upload new blueprint archive - """ - b = Blueprint.objects.create(archive=request.data["file"]) - s = BlueprintSerializer(b) - pipe = ( - tasks.upload_blueprint.si(b.cfy_id) | - tasks.create_deployment.si(b.cfy_id) | - tasks.install.si(b.cfy_id) - ) - pipe.apply_async() - return Response(s.data, status=201) - class BlueprintIdView(APIView): def get(self, request, blueprint_id): """ - # Return selected blueprint details + Return selected blueprint details """ s = BlueprintSerializer(Blueprint.get(blueprint_id)) return Response(s.data) def delete(self, request, blueprint_id): """ - # Delete selected blueprint + Delete selected blueprint + """ + blueprint = Blueprint.get(blueprint_id) + blueprint.pipe_undeploy_blueprint() + return Response(status=status.HTTP_202_ACCEPTED) + + +class ContainersView(APIView): + def get(self, request): + """ + List all virtual containers and their status information + """ + contaiers = Container.objects.all() + s = ContainerSerializer(contaiers, many=True) + return Response(data=s.data) + + def post(self, request): + container = Container() + container.save() # all default is good + s = ContainerSerializer(container) + return Response(data=s.data, status=status.HTTP_201_CREATED) + + +class ContainerIdView(APIView): + def get(self, request, container_id): + """ + Display the status information about the selected virtual container + """ + container = Container.get(container_id) + s = ContainerSerializer(container) + return Response(s.data) + + def delete(self, request, container_id): + """ + Remove virtual container and undeploy its blueprint. """ - b = Blueprint.get(blueprint_id) - pipe = ( - tasks.uninstall.si(b.cfy_id) | - tasks.delete_deployment.si(b.cfy_id) | - tasks.delete_blueprint.si(b.cfy_id) - ) - pipe.apply_async() - return Response(status=202) + container = Container.get(container_id) + try: + container.delete() + except IntegrityError, e: + return Response({'msg': e.message}, status=status.HTTP_400_BAD_REQUEST) + + return Response(status=status.HTTP_204_NO_CONTENT) + + +class ContainerBlueprint(APIView): + def post(self, request, container_id): + cont = Container.get(container_id) + blueprint_old = cont.blueprint + blueprint_new = Blueprint.objects.create(archive=request.data["file"]) + + # bind new blueprint to this container + cont.blueprint = blueprint_new + cont.save() + + # deploy the new blueprint + blueprint_new.pipe_deploy_blueprint() + + # undeploy the old blueprint + if blueprint_old: + # TODO: keep container-blueprint binding to old blueprint until cloudify undeploys it + blueprint_old.pipe_undeploy_blueprint() + + cont_ser = ContainerSerializer(cont).data + return Response(cont_ser, status=status.HTTP_202_ACCEPTED) + + + + + + diff --git a/dice_deploy_django/requirements.in b/dice_deploy_django/requirements.in index 15dfa19..b330cd4 100644 --- a/dice_deploy_django/requirements.in +++ b/dice_deploy_django/requirements.in @@ -5,3 +5,4 @@ cloudify_rest_client celery enum34 flower +django-factory-boy \ No newline at end of file diff --git a/dice_deploy_django/requirements.txt b/dice_deploy_django/requirements.txt index d3fc257..f489e20 100644 --- a/dice_deploy_django/requirements.txt +++ b/dice_deploy_django/requirements.txt @@ -6,14 +6,25 @@ # amqp==1.4.7 # via kombu anyjson==0.3.3 # via kombu +babel==2.2.0 # via flower +backports.ssl-match-hostname==3.5.0.1 # via tornado billiard==3.3.0.20 # via celery celery==3.1.18 +certifi==2016.2.28 # via tornado cloudify-rest-client==3.2.1 +django-factory-boy==1.0.0 django==1.9.4 djangorestframework==3.2.4 enum34==1.0.4 +factory-boy==2.6.1 # via django-factory-boy +fake-factory==0.5.7 # via factory-boy +flower==0.8.4 +futures==3.0.5 # via flower +ipaddress==1.0.16 # via fake-factory kombu==3.0.28 # via celery markdown==2.6.2 -pytz==2015.6 # via celery +python-dateutil==2.5.0 # via fake-factory +pytz==2015.7 # via babel, celery, flower requests==2.7.0 # via cloudify-rest-client -flower==0.8.4 +six==1.10.0 # via fake-factory, python-dateutil +tornado==4.2.0 # via flower diff --git a/dice_deploy_django/unit_tests/factories.py b/dice_deploy_django/unit_tests/factories.py new file mode 100644 index 0000000..8d3860f --- /dev/null +++ b/dice_deploy_django/unit_tests/factories.py @@ -0,0 +1,34 @@ +import factory +from cfy_wrapper.models import Container, Blueprint +from django.conf import settings + + +class BlueprintPendingFactory(factory.DjangoModelFactory): + state = Blueprint.State.pending.value + archive = factory.django.FileField(from_path=settings.TEST_FILE_BLUEPRINT_EXAMPLE) + + class Meta: + model = Blueprint + + +class BlueprintDeployedFactory(factory.DjangoModelFactory): + state = Blueprint.State.deployed.value + archive = factory.django.FileField(from_path=settings.TEST_FILE_BLUEPRINT_EXAMPLE) + + class Meta: + model = Blueprint + + +class ContainerEmptyFactory(factory.DjangoModelFactory): + class Meta: + model = Container + + +class ContainerFullFactory(factory.DjangoModelFactory): + blueprint = factory.SubFactory(BlueprintDeployedFactory) + + class Meta: + model = Container + + + diff --git a/dice_deploy_django/unit_tests/test_cfy_wrapper_api.py b/dice_deploy_django/unit_tests/test_cfy_wrapper_api.py index f546eb2..707102f 100644 --- a/dice_deploy_django/unit_tests/test_cfy_wrapper_api.py +++ b/dice_deploy_django/unit_tests/test_cfy_wrapper_api.py @@ -4,12 +4,14 @@ from django.conf import settings from rest_framework.exceptions import NotFound import os -from cfy_wrapper.models import Blueprint +from cfy_wrapper.models import Blueprint, Container import shutil from django.core import management +import factories +from cfy_wrapper import serializers -class AccountTests(APITestCase): +class WrapperAPITests(APITestCase): def setUp(self): # prevent data loss due to not setting settings_tests.py as settings module try: @@ -30,33 +32,168 @@ def setUp(self): def tearDown(self): shutil.rmtree(settings.MEDIA_ROOT) - # stop celery - # management.call_command('celery-service', 'stop', '--unit_tests') - - def test_upload_blueprint(self): - """ Test uploading a new blueprint """ + def test_blueprint_list(self): url = reverse('blueprints') - with open(os.path.join(settings.TEST_FILE_BLUEPRINT_EXAMPLE), 'rb') as f: - # example = f.read() - data = {'file': f} - response = self.client.put(url, data) + blue_pending = factories.BlueprintPendingFactory() + blue_pending_ser = serializers.BlueprintSerializer(blue_pending).data + + blue_deployed = factories.BlueprintDeployedFactory() + blue_deployed_ser = serializers.BlueprintSerializer(blue_deployed).data + + response = self.client.get(url) + + # check HTTP response + self.assertEqual( + response.status_code, status.HTTP_200_OK, + msg='Recieved bad status: %d. Response was: %s' % (response.status_code, response.data) + ) + + self.assertListEqual([blue_pending_ser, blue_deployed_ser], response.data) + + def test_blueprint_details(self): + blue_deployed = factories.BlueprintDeployedFactory() + blue_deployed_ser = serializers.BlueprintSerializer(blue_deployed).data + + url = reverse('blueprint_id', kwargs={'blueprint_id': blue_deployed.id}) + + response = self.client.get(url) + + # check HTTP response + self.assertEqual( + response.status_code, status.HTTP_200_OK, + msg='Recieved bad status: %d. Response was: %s' % (response.status_code, response.data) + ) + + self.assertEqual(blue_deployed_ser, response.data) + + def test_blueprint_remove(self): + blue_deployed = factories.BlueprintDeployedFactory() + blue_deployed_ser = serializers.BlueprintSerializer(blue_deployed).data + + url = reverse('blueprint_id', kwargs={'blueprint_id': blue_deployed.id}) + + response = self.client.delete(url) + + # check HTTP response + self.assertEqual( + response.status_code, status.HTTP_202_ACCEPTED, + msg='Recieved bad status: %d. Response was: %s' % (response.status_code, response.data) + ) + + def test_container_create(self): + url = reverse('containers') + + response = self.client.post(url) # check HTTP response self.assertEqual( response.status_code, status.HTTP_201_CREATED, - msg='Recieved bad status: %d. Data was: %s' % (response.status_code, response.data) + msg='Recieved bad status: %d. Response was: %s' % (response.status_code, response.data) + ) + self.assertDictContainsSubset({'blueprint': None}, response.data) + + if 'id' not in response.data: + raise AssertionError('Missing "id" key in response') + + # check DB state + try: + container = Container.get(response.data['id']) + except NotFound: + raise AssertionError('Container was not found in DB, but it should be there.') + + def test_container_remove_empty(self): + cont_empty = factories.ContainerEmptyFactory() + + url = reverse('container_id', kwargs={'container_id': cont_empty.id}) + response = self.client.delete(url) + + # check HTTP response + self.assertEqual( + response.status_code, status.HTTP_204_NO_CONTENT, + msg='Recieved bad status: %d. Response was: %s' % (response.status_code, response.data) + ) + + def test_container_remove_full(self): + cont_full = factories.ContainerFullFactory() + + url = reverse('container_id', kwargs={'container_id': cont_full.id}) + response = self.client.delete(url) + + # check HTTP response + self.assertEqual( + response.status_code, status.HTTP_400_BAD_REQUEST, + msg='Recieved bad status: %d. Response was: %s' % (response.status_code, response.data) + ) + + def test_container_list(self): + url = reverse('containers') + + cont_empty = factories.ContainerEmptyFactory() + cont_empty_ser = serializers.ContainerSerializer(cont_empty).data + + cont_full = factories.ContainerFullFactory() + cont_full_ser = serializers.ContainerSerializer(cont_full).data + + response = self.client.get(url) + + # check HTTP response + self.assertEqual( + response.status_code, status.HTTP_200_OK, + msg='Recieved bad status: %d. Response was: %s' % (response.status_code, response.data) ) - self.assertDictContainsSubset({'state_name': 'pending'}, response.data) - if 'cfy_id' not in response.data: - raise AssertionError('Missing "cfy" key in response') + self.assertListEqual([cont_empty_ser, cont_full_ser], response.data) + + def test_container_details(self): + cont_full = factories.ContainerFullFactory() + cont_full_ser = serializers.ContainerSerializer(cont_full).data + + url = reverse('container_id', kwargs={'container_id': cont_full.id}) + response = self.client.get(url) + + # check HTTP response + self.assertEqual( + response.status_code, status.HTTP_200_OK, + msg='Recieved bad status: %d. Response was: %s' % (response.status_code, response.data) + ) + + self.assertEqual(cont_full_ser, response.data) + + def test_container_details_non_existent(self): + cont_full = factories.ContainerFullFactory.build() # unsaved + + url = reverse('container_id', kwargs={'container_id': cont_full.id}) + response = self.client.get(url) + + # check HTTP response + self.assertEqual( + response.status_code, status.HTTP_404_NOT_FOUND, + msg='Recieved bad status: %d. Response was: %s' % (response.status_code, response.data) + ) + + def test_container_blueprint_upload_to_empty(self): + cont_empty = factories.ContainerEmptyFactory() + url = reverse('container_blueprint', kwargs={'container_id': cont_empty.id}) + + with open(os.path.join(settings.TEST_FILE_BLUEPRINT_EXAMPLE), 'rb') as f: + data = {'file': f} + response = self.client.post(url, data) + + # check HTTP response + self.assertEqual( + response.status_code, status.HTTP_202_ACCEPTED, + msg='Recieved bad status: %d. Response was: %s' % (response.status_code, response.data) + ) # check DB state try: - blueprint = Blueprint.get(response.data['cfy_id']) + cont = Container.get(response.data['id']) + blueprint = Blueprint.get(response.data['blueprint']['cfy_id']) except NotFound: - raise AssertionError('Blueprint was not found in DB, but it should be there.') + raise AssertionError('Container or Blueprint was not found in DB, but it should be there.') + + self.assertEqual(cont.blueprint, blueprint) # check filesystem with open(blueprint.archive.path) as f: @@ -64,4 +201,32 @@ def test_upload_blueprint(self): if not data: raise AssertionError('Blueprint .tar.gz file could not be found/opened on filesystem') + def test_container_blueprint_upload_to_full(self): + cont_full = factories.ContainerFullFactory() + url = reverse('container_blueprint', kwargs={'container_id': cont_full.id}) + + with open(os.path.join(settings.TEST_FILE_BLUEPRINT_EXAMPLE), 'rb') as f: + data = {'file': f} + response = self.client.post(url, data) + + # check HTTP response + self.assertEqual( + response.status_code, status.HTTP_202_ACCEPTED, + msg='Recieved bad status: %d. Response was: %s' % (response.status_code, response.data) + ) + + # check DB state + try: + cont = Container.get(response.data['id']) + blueprint = Blueprint.get(response.data['blueprint']['cfy_id']) + except NotFound: + raise AssertionError('Container or Blueprint was not found in DB, but it should be there.') + + self.assertEqual(cont.blueprint, blueprint) + + # check filesystem + with open(blueprint.archive.path) as f: + data = f.read() + if not data: + raise AssertionError('Blueprint .tar.gz file could not be found/opened on filesystem')