From 42635f57c7af35c90f6327636f157aa807cde118 Mon Sep 17 00:00:00 2001 From: aleontiev Date: Thu, 15 Dec 2016 13:23:12 -0800 Subject: [PATCH 01/35] add Python DREST client --- dynamic_rest/client.py | 194 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 dynamic_rest/client.py diff --git a/dynamic_rest/client.py b/dynamic_rest/client.py new file mode 100644 index 00000000..3739b1b4 --- /dev/null +++ b/dynamic_rest/client.py @@ -0,0 +1,194 @@ +import json +import requests +import inflection + + +API_AUTH_ENDPOINT = '/accounts/login/' + + +class AuthenticationFailed(Exception): + pass + + +class APIClient(object): + + def __init__( + self, + host, + version=None, + session=None, + sessionid=None, + username=None, + password=None, + token=None, + authorization_type='JWT' + ): + self.host = host + self.version = version + self.session = session or requests.session() + self.session.headers.update({ + 'Content-Type': 'application/json', + 'Accept': 'application/json' + }) + self.sessionid = sessionid + self.username = username + self.password = password + self.token = token + self.authorization_type = authorization_type + + @property + def token(self): + return self._token + + @token.setter + def token(self, value): + self._token = value + self.authenticated = bool(value) + if value: + self.session.headers.update({ + 'Authorization': '%s %s' % ( + self.AUTHORIZATION_TYPE, + self._token + ) + }) + + @property + def sessionid(self): + return self._sessionid + + @sessionid.setter + def sessionid(self, value): + self._sessionid = value + self.authenticated = bool(value) + if value: + self.session.headers.update({ + 'Cookie': 'sessionid=%s' % value + }) + + def authenticate(self, raise_exception=True): + response = None + if not self.authenticated: + username = self.username + password = self.password + response = requests.post( + self.build_url(API_AUTH_ENDPOINT), + data={ + 'login': username, + 'password': password + }, + allow_redirects=False + ) + if raise_exception: + response.raise_for_status() + + self.sessionid = response.cookies.get('sessionid') + + if raise_exception and not self.authenticated: + raise AuthenticationFailed( + response.text if response else 'Unknown error' + ) + return self.authenticated + + def build_url(self, url, prefix=None): + if not url.startswith('/'): + url = '/%s' % url + + if prefix: + if not prefix.startswith('/'): + prefix = '/%s' % prefix + + url = '%s%s' % (prefix, url) + return 'https://%s%s' % (self.host, url) + + def save( + self, + resource, + data, + **kwargs + ): + if 'id' in data: + update = True + pk = data['id'] + else: + update = False + pk = None + + data = json.dumps(data) + url = self.get_resource_url(resource, pk) + try: + verb = 'put' if update else 'post' + response = getattr(self, verb)(url, data=data) + return response + except requests.exceptions.HTTPError as e: + return e.response + + def get_resource_body(self, response, resource, many=False): + body = json.loads(response.text) + key = resource if many else inflection.singularize(resource) + return body[key] + + def get_resource_url(self, resource, pk=None): + return '%s/%s' % ( + resource, + '%s/' % pk if pk else '' + ) + + def find( + self, + resource, + *args, + **kwargs + ): + pk = args[0] if len(args) else None + many = False if pk else True + url = self.get_resource_url(resource, pk) + filters = kwargs.get('filters') + include = kwargs.get('include') + params = [] + if filters: + for key, value in filters.items(): + params.append('filter{%s}=%s' % (key, value)) + if include: + for key in include: + params.append('include[]=%s' % key) + if params: + url = '%s?%s' % (url, '&'.join(params)) + response = self.get(url) + return self.get_resource_body(response, resource, many=many) + + def destroy( + self, + resource, + pk + ): + url = self.get_resource_url(resource, pk) + try: + return self.delete(url) + except requests.exceptions.HTTPError as e: + return e.response + + def post(self, url, params=None, data=None): + return self.request('POST', url, params, data) + + def get(self, url, params=None, data=None): + return self.request('GET', url, params, data) + + def put(self, url, params=None, data=None): + return self.request('PUT', url, params, data) + + def delete(self, url, params=None, data=None): + return self.request('DELETE', url, params, data) + + def request(self, method, url, params=None, data=None): + self.authenticate() + response = self.session.request( + method, + self.build_url(url, prefix=self.version), + params=params, + data=data + ) + if response.status_code == 401: + raise AuthenticationFailed() + + response.raise_for_status() + return response From f43b5233196ff4797fc9be8d8176b34f05ddd340 Mon Sep 17 00:00:00 2001 From: aleontiev Date: Thu, 15 Dec 2016 13:24:53 -0800 Subject: [PATCH 02/35] add scheme --- dynamic_rest/client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dynamic_rest/client.py b/dynamic_rest/client.py index 3739b1b4..c3d919de 100644 --- a/dynamic_rest/client.py +++ b/dynamic_rest/client.py @@ -21,7 +21,8 @@ def __init__( username=None, password=None, token=None, - authorization_type='JWT' + authorization_type='JWT', + scheme='https' ): self.host = host self.version = version @@ -34,6 +35,7 @@ def __init__( self.username = username self.password = password self.token = token + self.scheme = scheme self.authorization_type = authorization_type @property @@ -98,7 +100,7 @@ def build_url(self, url, prefix=None): prefix = '/%s' % prefix url = '%s%s' % (prefix, url) - return 'https://%s%s' % (self.host, url) + return '%s://%s%s' % (self.scheme, self.host, url) def save( self, From f4ed823c3cf638ada5f6f89b047a35d009166832 Mon Sep 17 00:00:00 2001 From: aleontiev Date: Thu, 15 Dec 2016 13:44:36 -0800 Subject: [PATCH 03/35] fix typo --- dynamic_rest/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dynamic_rest/client.py b/dynamic_rest/client.py index c3d919de..a2323f51 100644 --- a/dynamic_rest/client.py +++ b/dynamic_rest/client.py @@ -49,7 +49,7 @@ def token(self, value): if value: self.session.headers.update({ 'Authorization': '%s %s' % ( - self.AUTHORIZATION_TYPE, + self.authorization_type, self._token ) }) From 7c08c4fc8c0800550ab34bcb0dcca3fe5ac81d97 Mon Sep 17 00:00:00 2001 From: aleontiev Date: Thu, 15 Dec 2016 13:47:57 -0800 Subject: [PATCH 04/35] fix order of init --- dynamic_rest/client.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dynamic_rest/client.py b/dynamic_rest/client.py index a2323f51..216a45bc 100644 --- a/dynamic_rest/client.py +++ b/dynamic_rest/client.py @@ -24,6 +24,7 @@ def __init__( authorization_type='JWT', scheme='https' ): + self.authorization_type = authorization_type self.host = host self.version = version self.session = session or requests.session() @@ -31,12 +32,12 @@ def __init__( 'Content-Type': 'application/json', 'Accept': 'application/json' }) - self.sessionid = sessionid self.username = username self.password = password - self.token = token self.scheme = scheme - self.authorization_type = authorization_type + + self.sessionid = sessionid + self.token = token @property def token(self): From 492bcb0e6730f615c80b02ce6171c253e50ed12b Mon Sep 17 00:00:00 2001 From: Bradley Gunn Date: Wed, 11 Jan 2017 14:17:07 -0500 Subject: [PATCH 05/35] Proxy renamed field for downstream dependencies --- dynamic_rest/filters.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dynamic_rest/filters.py b/dynamic_rest/filters.py index f5c26c77..7e4eda51 100644 --- a/dynamic_rest/filters.py +++ b/dynamic_rest/filters.py @@ -194,6 +194,13 @@ def filter_queryset(self, request, queryset, view): self.DEBUG = settings.DEBUG return self._build_queryset(queryset=queryset) + """ + This function was renamed and broke downstream dependencies that haven't + been updated to use the new naming convention. + """ + def _extract_filters(self, **kwargs): + return self._get_requested_filters(**kwargs) + def _get_requested_filters(self, **kwargs): """ Convert 'filters' query params into a dict that can be passed From 928ef922bdfb335796218e10d97b0e2b8b5f6b07 Mon Sep 17 00:00:00 2001 From: aleontiev Date: Thu, 12 Jan 2017 16:25:36 -0800 Subject: [PATCH 06/35] add support for forced embedding/sideloading --- dynamic_rest/fields/fields.py | 8 ++++-- dynamic_rest/processors.py | 4 ++- dynamic_rest/serializers.py | 53 ++++++++++++++++++++-------------- dynamic_rest/viewsets.py | 26 +++++++++++------ tests/test_api.py | 51 +++++++++++++++++++++++++++++++++ tests/test_generic.py | 2 +- tests/test_serializers.py | 54 ++++++++++++++++++++++++----------- tests/viewsets.py | 3 +- 8 files changed, 150 insertions(+), 51 deletions(-) diff --git a/dynamic_rest/fields/fields.py b/dynamic_rest/fields/fields.py index 01213a47..6966154d 100644 --- a/dynamic_rest/fields/fields.py +++ b/dynamic_rest/fields/fields.py @@ -82,6 +82,7 @@ def __init__( many=False, queryset=None, embed=False, + sideloading=None, **kwargs ): """ @@ -97,7 +98,8 @@ def __init__( self._serializer_class = serializer_class self.bound = False self.queryset = queryset - self.embed = embed + self.sideloading = sideloading + self.embed = embed if sideloading is None else not sideloading if '.' in kwargs.get('source', ''): raise Exception('Nested relationships are not supported') if 'link' in kwargs: @@ -219,8 +221,8 @@ def _inherit_parent_kwargs(self, kwargs): # If 'embed' then make sure we fetch the full object. kwargs['request_fields'] = {} - if hasattr(self.parent, 'sideload'): - kwargs['sideload'] = self.parent.sideload + if hasattr(self.parent, 'sideloading'): + kwargs['sideloading'] = self.parent.sideloading return kwargs diff --git a/dynamic_rest/processors.py b/dynamic_rest/processors.py index e37209c8..8e48afaa 100644 --- a/dynamic_rest/processors.py +++ b/dynamic_rest/processors.py @@ -77,7 +77,9 @@ def process(self, obj, parent=None, parent_key=None, depth=0): 1 ) - if not dynamic or getattr(obj, 'embed', False): + if ( + not dynamic or getattr(obj, 'embed', False) + ): return name = obj.serializer.get_plural_name() diff --git a/dynamic_rest/serializers.py b/dynamic_rest/serializers.py index 1df9647c..4843a239 100644 --- a/dynamic_rest/serializers.py +++ b/dynamic_rest/serializers.py @@ -8,7 +8,7 @@ from django.utils.functional import cached_property from rest_framework import exceptions, fields, serializers from rest_framework.fields import SkipField -from rest_framework.utils.serializer_helpers import ReturnDict, ReturnList +from rest_framework.utils.serializer_helpers import ReturnDict from dynamic_rest.bases import DynamicSerializerBase from dynamic_rest.conf import settings @@ -63,22 +63,25 @@ def id_only(self): """Get the child's rendering mode.""" return self.child.id_only() + @property + def data_no_envelope(self): + """For compatability with DRF's .data""" + data = self.data + return data.values()[0] + @property def data(self): """Get the data, after performing post-processing if necessary.""" - if not hasattr(self, '_sideloaded_data'): + if not hasattr(self, '_processed_data'): data = super(DynamicListSerializer, self).data - if self.child.sideload: - self._sideloaded_data = ReturnDict( - SideloadingProcessor( - self, - data - ).data, - serializer=self - ) - else: - self._sideloaded_data = ReturnList(data, serializer=self) - return self._sideloaded_data + self._processed_data = ReturnDict( + SideloadingProcessor( + self, + data + ).data, + serializer=self + ) + return self._processed_data def update(self, queryset, validated_data): lookup_attr = getattr(self.child.Meta, 'update_lookup_field', 'id') @@ -160,7 +163,7 @@ def __init__( include_fields=None, exclude_fields=None, request_fields=None, - sideload=False, + sideloading=None, dynamic=True, embed=False, **kwargs @@ -175,8 +178,10 @@ def __init__( exclude_fields: List of field names to exclude. request_fields: map of field names that supports inclusions, exclusions, and nested sideloads. - sideload: If False, do not perform any sideloading at this level. embed: If True, force the current representation to be embedded. + sideloading: if False, force embedding for all descendants + If True, force sideloading for all descendants + If None (default), respect individual embed parameters dynamic: If False, ignore deferred rules and revert to standard DRF `.fields` behavior. """ @@ -201,10 +206,10 @@ def __init__( kwargs['data'] = data super(WithDynamicSerializerMixin, self).__init__(**kwargs) - self.sideload = sideload + self.sideloading = sideloading self.dynamic = dynamic self.request_fields = request_fields or {} - self.embed = embed + self.embed = embed if sideloading is None else not sideloading self._dynamic_init(only_fields, include_fields, exclude_fields) self.enable_optimization = settings.ENABLE_SERIALIZER_OPTIMIZATIONS @@ -575,18 +580,24 @@ def id_only(self): """ return self.dynamic and self.request_fields is True + @property + def data_no_envelope(self): + """For compatability with DRF's .data""" + data = self.data + return data.values()[0] + @property def data(self): - if not hasattr(self, '_sideloaded_data'): + if not hasattr(self, '_processed_data'): data = super(WithDynamicSerializerMixin, self).data - self._sideloaded_data = ReturnDict( + self._processed_data = ReturnDict( SideloadingProcessor( self, data - ).data if self.sideload else data, + ).data, serializer=self ) - return self._sideloaded_data + return self._processed_data class WithDynamicModelSerializerMixin(WithDynamicSerializerMixin): diff --git a/dynamic_rest/viewsets.py b/dynamic_rest/viewsets.py index f5ed06ea..38a89548 100644 --- a/dynamic_rest/viewsets.py +++ b/dynamic_rest/viewsets.py @@ -63,6 +63,7 @@ class WithDynamicViewSetMixin(object): meta: Extra data that is added to the response by the DynamicRenderer. """ + SIDELOADING = 'sideloading' INCLUDE = 'include[]' EXCLUDE = 'exclude[]' FILTER = 'filter{}' @@ -74,8 +75,7 @@ class WithDynamicViewSetMixin(object): pagination_class = DynamicPageNumberPagination metadata_class = DynamicMetadata renderer_classes = (JSONRenderer, DynamicBrowsableAPIRenderer) - features = (INCLUDE, EXCLUDE, FILTER, PAGE, PER_PAGE, SORT) - sideload = True + features = (INCLUDE, EXCLUDE, FILTER, PAGE, PER_PAGE, SORT, SIDELOADING) meta = None filter_backends = (DynamicFilterBackend, DynamicSortingFilter) @@ -190,7 +190,7 @@ def get_queryset(self, queryset=None): return getattr(self, 'queryset', serializer.Meta.model.objects.all()) def get_request_fields(self): - """Parses the `include[]` and `exclude[]` features. + """Parses the INCLUDE and EXCLUDE features. Extracts the dynamic field features from the request parameters into a field map that can be passed to a serializer. @@ -202,8 +202,8 @@ def get_request_fields(self): if hasattr(self, '_request_fields'): return self._request_fields - include_fields = self.get_request_feature('include[]') - exclude_fields = self.get_request_feature('exclude[]') + include_fields = self.get_request_feature(self.INCLUDE) + exclude_fields = self.get_request_feature(self.EXCLUDE) request_fields = {} for fields, include in( (include_fields, True), @@ -232,6 +232,16 @@ def get_request_fields(self): self._request_fields = request_fields return request_fields + def get_request_sideloading(self): + sideloading = self.get_request_feature(self.SIDELOADING) + if sideloading: + sideloading = sideloading.lower() + if sideloading == 'true': + sideloading = True + else: + sideloading = False + return sideloading + def is_update(self): if ( self.request and @@ -253,8 +263,8 @@ def is_delete(self): def get_serializer(self, *args, **kwargs): if 'request_fields' not in kwargs: kwargs['request_fields'] = self.get_request_fields() - if 'sideload' not in kwargs: - kwargs['sideload'] = self.sideload + if 'sideloading' not in kwargs: + kwargs['sideloading'] = self.get_request_sideloading() if self.is_update(): kwargs['include_fields'] = '*' return super( @@ -312,7 +322,7 @@ def list_related(self, request, pk=None, field_name=None): # Filter for parent object, include related field. self.request.query_params.add('filter{pk}', pk) - self.request.query_params.add('include[]', field_prefix) + self.request.query_params.add(self.INCLUDE, field_prefix) # Get serializer and field. serializer = self.get_serializer() diff --git a/tests/test_api.py b/tests/test_api.py index c7e99153..0fe39682 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -130,6 +130,44 @@ def test_get_with_exclude(self): }] }, json.loads(response.content.decode('utf-8'))) + def test_get_with_nested_has_one_sideloading_disabled(self): + with self.assertNumQueries(2): + response = self.client.get( + '/users/?include[]=location.&sideloading=false' + ) + self.assertEquals(200, response.status_code) + self.assertEquals({ + 'users': [{ + 'id': 1, + 'location': { + 'id': 1, + 'name': '0' + }, + 'name': '0' + }, { + 'id': 2, + 'location': { + 'id': 1, + 'name': '0' + }, + 'name': '1' + }, { + 'id': 3, + 'location': { + 'id': 2, + 'name': '1' + }, + 'name': '2' + }, { + 'id': 4, + 'location': { + 'id': 3, + 'name': '2' + }, + 'name': '3' + }] + }, json.loads(response.content.decode('utf-8'))) + def test_get_with_nested_has_one(self): with self.assertNumQueries(2): response = self.client.get('/users/?include[]=location.') @@ -1031,6 +1069,19 @@ def test_get_embedded(self): self.assertTrue(isinstance(groups[0], dict)) self.assertTrue(isinstance(location, dict)) + def test_get_embedded_force_sideloading(self): + with self.assertNumQueries(3): + url = '/v1/user_locations/1/?sideloading=true' + response = self.client.get(url) + + self.assertEqual(200, response.status_code) + content = json.loads(response.content.decode('utf-8')) + groups = content['user_location']['groups'] + location = content['user_location']['location'] + self.assertEqual(content['locations'][0]['name'], '0') + self.assertFalse(isinstance(groups[0], dict)) + self.assertFalse(isinstance(location, dict)) + class TestLinks(APITestCase): diff --git a/tests/test_generic.py b/tests/test_generic.py index 4155a740..eac47a75 100644 --- a/tests/test_generic.py +++ b/tests/test_generic.py @@ -171,7 +171,7 @@ class Meta: 'favorite_pet' ).first() - data = FooUserSerializer(user).data + data = FooUserSerializer(user).data['user'] self.assertIsNotNone(data) self.assertTrue('favorite_pet' in data) self.assertTrue(isinstance(data['favorite_pet'], dict)) diff --git a/tests/test_serializers.py b/tests/test_serializers.py index ea7d35b1..028acfe9 100644 --- a/tests/test_serializers.py +++ b/tests/test_serializers.py @@ -40,7 +40,7 @@ def test_data(self): serializer = UserSerializer( self.fixture.users, many=True, - sideload=True) + ) self.assertEqual(serializer.data, { 'users': [ OrderedDict( @@ -54,6 +54,24 @@ def test_data(self): ] }) + def test_data_no_envelope(self): + serializer = UserSerializer( + self.fixture.users, + many=True, + ) + self.assertEqual( + serializer.data_no_envelope, [ + OrderedDict( + [('id', 1), ('name', '0'), ('location', 1)]), + OrderedDict( + [('id', 2), ('name', '1'), ('location', 1)]), + OrderedDict( + [('id', 3), ('name', '2'), ('location', 2)]), + OrderedDict( + [('id', 4), ('name', '3'), ('location', 3)]) + ] + ) + def test_data_with_included_field(self): request_fields = { 'last_name': True @@ -62,7 +80,7 @@ def test_data_with_included_field(self): self.fixture.users, many=True, request_fields=request_fields, - sideload=True) + ) self.assertEqual(serializer.data, { 'users': [ OrderedDict( @@ -88,7 +106,7 @@ def test_data_with_excluded_field(self): self.fixture.users, many=True, request_fields=request_fields, - sideload=True) + ) self.assertEqual(serializer.data, { 'users': [ OrderedDict( @@ -110,7 +128,7 @@ def test_data_with_included_has_one(self): self.fixture.users, many=True, request_fields=request_fields, - sideload=True) + ) self.assertEqual(serializer.data, { 'locations': [{ 'id': 1, @@ -144,7 +162,7 @@ def test_data_with_included_has_one(self): serializer = UserSerializer( self.fixture.users[0], request_fields=request_fields, - sideload=True) + ) self.assertEqual(serializer.data, { 'locations': [{ 'id': 1, @@ -215,7 +233,7 @@ def test_data_with_included_has_many(self): self.fixture.users, many=True, request_fields=request_fields, - sideload=True) + ) self.assertEqual(serializer.data, expected) request_fields = { @@ -272,7 +290,7 @@ def test_data_with_included_has_many(self): self.fixture.groups, many=True, request_fields=request_fields, - sideload=True) + ) self.assertEqual(serializer.data, expected) def test_data_with_nested_include(self): @@ -286,7 +304,7 @@ def test_data_with_nested_include(self): self.fixture.users, many=True, request_fields=request_fields, - sideload=True) + ) expected = { 'users': [ { @@ -355,7 +373,7 @@ def test_data_with_nested_exclude(self): self.fixture.users, many=True, request_fields=request_fields, - sideload=True) + ) self.assertEqual(serializer.data, { 'groups': [{ 'id': 1 @@ -506,21 +524,21 @@ def test_data(self): data['location'] = location data['groups'] = self.fixture.groups instance = EphemeralObject(data) - data = LocationGroupSerializer(instance).data + data = LocationGroupSerializer(instance).data['locationgroup'] self.assertEqual( data, {'id': 1, 'groups': [1, 2], 'location': 1} ) def test_data_count_field(self): eo = EphemeralObject({'pk': 1, 'values': [1, 1, 2]}) - data = CountsSerializer(eo).data + data = CountsSerializer(eo).data['counts'] self.assertEqual(data['count'], 3) self.assertEqual(data['unique_count'], 2) def test_data_count_field_returns_none_if_null_values(self): eo = EphemeralObject({'pk': 1, 'values': None}) - data = CountsSerializer(eo).data + data = CountsSerializer(eo).data['counts'] self.assertEqual(data['count'], None) self.assertEqual(data['unique_count'], None) @@ -557,7 +575,8 @@ def setUp(self): def test_data_with_embed(self): data = UserLocationSerializer( - self.fixture.users[0], sideload=True).data + self.fixture.users[0] + ).data self.assertEqual(data['user_location']['location']['name'], '0') self.assertEqual( ['0', '1'], @@ -585,7 +604,8 @@ class Meta: 'id': True, 'name': True, 'location': True - }).data + } + ).data['user_deferred_location'] self.assertTrue('location' in data) self.assertEqual(data['location']['name'], '0') @@ -614,7 +634,8 @@ class Meta: 'id': True, 'name': True, 'groups': True - }).data + } + ).data['user_deferred_location'] self.assertTrue('groups' in data) @override_settings( @@ -632,7 +653,8 @@ class Meta: groups = DynamicRelationField('GroupSerializer', many=True) data = UserDeferredLocationSerializer( - self.fixture.users[0]).data + self.fixture.users[0] + ).data['user_deferred_location'] self.assertTrue('groups' in data) diff --git a/tests/viewsets.py b/tests/viewsets.py index b921b391..be4c7f2b 100644 --- a/tests/viewsets.py +++ b/tests/viewsets.py @@ -29,7 +29,8 @@ class UserViewSet(DynamicModelViewSet): features = ( DynamicModelViewSet.INCLUDE, DynamicModelViewSet.EXCLUDE, - DynamicModelViewSet.FILTER, DynamicModelViewSet.SORT + DynamicModelViewSet.FILTER, DynamicModelViewSet.SORT, + DynamicModelViewSet.SIDELOADING ) model = User serializer_class = UserSerializer From 9281b9330c8fe3fe25af637f98c9937c690497b7 Mon Sep 17 00:00:00 2001 From: aleontiev Date: Fri, 13 Jan 2017 15:43:00 -0800 Subject: [PATCH 07/35] working queryset/manager --- dynamic_rest/client.py | 197 ------------------------------ dynamic_rest/client/__init__.py | 0 dynamic_rest/client/client.py | 135 ++++++++++++++++++++ dynamic_rest/client/exceptions.py | 2 + dynamic_rest/client/manager.py | 14 +++ dynamic_rest/client/recordset.py | 148 ++++++++++++++++++++++ dynamic_rest/client/resource.py | 7 ++ dynamic_rest/serializers.py | 6 - dynamic_rest/viewsets.py | 1 - install_requires.txt | 1 + tests/test_serializers.py | 18 --- 11 files changed, 307 insertions(+), 222 deletions(-) delete mode 100644 dynamic_rest/client.py create mode 100644 dynamic_rest/client/__init__.py create mode 100644 dynamic_rest/client/client.py create mode 100644 dynamic_rest/client/exceptions.py create mode 100644 dynamic_rest/client/manager.py create mode 100644 dynamic_rest/client/recordset.py create mode 100644 dynamic_rest/client/resource.py diff --git a/dynamic_rest/client.py b/dynamic_rest/client.py deleted file mode 100644 index 216a45bc..00000000 --- a/dynamic_rest/client.py +++ /dev/null @@ -1,197 +0,0 @@ -import json -import requests -import inflection - - -API_AUTH_ENDPOINT = '/accounts/login/' - - -class AuthenticationFailed(Exception): - pass - - -class APIClient(object): - - def __init__( - self, - host, - version=None, - session=None, - sessionid=None, - username=None, - password=None, - token=None, - authorization_type='JWT', - scheme='https' - ): - self.authorization_type = authorization_type - self.host = host - self.version = version - self.session = session or requests.session() - self.session.headers.update({ - 'Content-Type': 'application/json', - 'Accept': 'application/json' - }) - self.username = username - self.password = password - self.scheme = scheme - - self.sessionid = sessionid - self.token = token - - @property - def token(self): - return self._token - - @token.setter - def token(self, value): - self._token = value - self.authenticated = bool(value) - if value: - self.session.headers.update({ - 'Authorization': '%s %s' % ( - self.authorization_type, - self._token - ) - }) - - @property - def sessionid(self): - return self._sessionid - - @sessionid.setter - def sessionid(self, value): - self._sessionid = value - self.authenticated = bool(value) - if value: - self.session.headers.update({ - 'Cookie': 'sessionid=%s' % value - }) - - def authenticate(self, raise_exception=True): - response = None - if not self.authenticated: - username = self.username - password = self.password - response = requests.post( - self.build_url(API_AUTH_ENDPOINT), - data={ - 'login': username, - 'password': password - }, - allow_redirects=False - ) - if raise_exception: - response.raise_for_status() - - self.sessionid = response.cookies.get('sessionid') - - if raise_exception and not self.authenticated: - raise AuthenticationFailed( - response.text if response else 'Unknown error' - ) - return self.authenticated - - def build_url(self, url, prefix=None): - if not url.startswith('/'): - url = '/%s' % url - - if prefix: - if not prefix.startswith('/'): - prefix = '/%s' % prefix - - url = '%s%s' % (prefix, url) - return '%s://%s%s' % (self.scheme, self.host, url) - - def save( - self, - resource, - data, - **kwargs - ): - if 'id' in data: - update = True - pk = data['id'] - else: - update = False - pk = None - - data = json.dumps(data) - url = self.get_resource_url(resource, pk) - try: - verb = 'put' if update else 'post' - response = getattr(self, verb)(url, data=data) - return response - except requests.exceptions.HTTPError as e: - return e.response - - def get_resource_body(self, response, resource, many=False): - body = json.loads(response.text) - key = resource if many else inflection.singularize(resource) - return body[key] - - def get_resource_url(self, resource, pk=None): - return '%s/%s' % ( - resource, - '%s/' % pk if pk else '' - ) - - def find( - self, - resource, - *args, - **kwargs - ): - pk = args[0] if len(args) else None - many = False if pk else True - url = self.get_resource_url(resource, pk) - filters = kwargs.get('filters') - include = kwargs.get('include') - params = [] - if filters: - for key, value in filters.items(): - params.append('filter{%s}=%s' % (key, value)) - if include: - for key in include: - params.append('include[]=%s' % key) - if params: - url = '%s?%s' % (url, '&'.join(params)) - response = self.get(url) - return self.get_resource_body(response, resource, many=many) - - def destroy( - self, - resource, - pk - ): - url = self.get_resource_url(resource, pk) - try: - return self.delete(url) - except requests.exceptions.HTTPError as e: - return e.response - - def post(self, url, params=None, data=None): - return self.request('POST', url, params, data) - - def get(self, url, params=None, data=None): - return self.request('GET', url, params, data) - - def put(self, url, params=None, data=None): - return self.request('PUT', url, params, data) - - def delete(self, url, params=None, data=None): - return self.request('DELETE', url, params, data) - - def request(self, method, url, params=None, data=None): - self.authenticate() - response = self.session.request( - method, - self.build_url(url, prefix=self.version), - params=params, - data=data - ) - if response.status_code == 401: - raise AuthenticationFailed() - - response.raise_for_status() - return response diff --git a/dynamic_rest/client/__init__.py b/dynamic_rest/client/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dynamic_rest/client/client.py b/dynamic_rest/client/client.py new file mode 100644 index 00000000..13c32d70 --- /dev/null +++ b/dynamic_rest/client/client.py @@ -0,0 +1,135 @@ +import json +import requests +import inflection +from .exceptions import AuthenticationFailed +from .resource import APIResource + +API_AUTH_ENDPOINT = '/accounts/login/' + + +class APIClient(object): + + def __init__( + self, + host, + version=None, + session=None, + sessionid=None, + username=None, + password=None, + token=None, + authorization_type='JWT', + scheme='https' + ): + self._authorization_type = authorization_type + self._host = host + self._version = version + self._session = session or requests.session() + self._session.headers.update({ + 'Content-Type': 'application/json', + 'Accept': 'application/json' + }) + self._username = username + self._password = password + self._scheme = scheme + self._authenticated = False + self._resources = {} + + if token: + self.token = token + if sessionid: + self.sessionid = sessionid + + @property + def token(self): + return self._token + + @token.setter + def token(self, value): + self._token = value + self._authenticated = bool(value) + if value: + self._session.headers.update({ + 'Authorization': '%s %s' % ( + self._authorization_type, + self._token + ) + }) + + @property + def sessionid(self): + return self._sessionid + + @sessionid.setter + def sessionid(self, value): + self._sessionid = value + self._authenticated = bool(value) + if value: + self._session.headers.update({ + 'Cookie': 'sessionid=%s' % value + }) + + def post(self, url, params=None, data=None): + return self._request('POST', url, params, data) + + def get(self, url, params=None, data=None): + return self._request('GET', url, params, data) + + def put(self, url, params=None, data=None): + return self._request('PUT', url, params, data) + + def delete(self, url, params=None, data=None): + return self._request('DELETE', url, params, data) + + def __getattr__(self, key): + key = key.lower() + return self._resources.get(key, APIResource(self, key)) + + def _authenticate(self, raise_exception=True): + response = None + if not self._authenticated: + username = self._username + password = self._password + response = requests.post( + self._build_url(API_AUTH_ENDPOINT), + data={ + 'login': username, + 'password': password + }, + allow_redirects=False + ) + if raise_exception: + response.raise_for_status() + + self.sessionid = response.cookies.get('sessionid') + + if raise_exception and not self._authenticated: + raise AuthenticationFailed( + response.text if response else 'Unknown error' + ) + return self._authenticated + + def _build_url(self, url, prefix=None): + if not url.startswith('/'): + url = '/%s' % url + + if prefix: + if not prefix.startswith('/'): + prefix = '/%s' % prefix + + url = '%s%s' % (prefix, url) + return '%s://%s%s' % (self._scheme, self._host, url) + + def _request(self, method, url, params=None, data=None): + self._authenticate() + response = self._session.request( + method, + self._build_url(url, prefix=self._version), + params=params, + data=data + ) + if response.status_code == 401: + raise AuthenticationFailed() + + response.raise_for_status() + return json.loads(response.content) diff --git a/dynamic_rest/client/exceptions.py b/dynamic_rest/client/exceptions.py new file mode 100644 index 00000000..1840bc25 --- /dev/null +++ b/dynamic_rest/client/exceptions.py @@ -0,0 +1,2 @@ +class AuthenticationFailed(Exception): + pass diff --git a/dynamic_rest/client/manager.py b/dynamic_rest/client/manager.py new file mode 100644 index 00000000..2b90db37 --- /dev/null +++ b/dynamic_rest/client/manager.py @@ -0,0 +1,14 @@ +from .recordset import APIRecordSet + +class APIResourceManager(object): + def __init__(self, resource): + self.resource = resource + + def get(self, pk): + return APIRecordSet(self).get(pk) + + def filter(self, **kwargs): + return APIRecordSet(self, filters=kwargs) + + def all(self): + return APIRecordSet(self) diff --git a/dynamic_rest/client/recordset.py b/dynamic_rest/client/recordset.py new file mode 100644 index 00000000..5e337f91 --- /dev/null +++ b/dynamic_rest/client/recordset.py @@ -0,0 +1,148 @@ +from copy import copy + + +def extract(content): + keys = [k for k in content.keys() if k != 'meta'] + return content[keys[0]] + + +def build_params( + filters, + includes, + excludes, + orders, + extras +): + params = {} + for key, value in filters.items(): + filter_key = 'filter{%s}' % key.replace('__', '.') + params[filter_key] = value + + params['include[]'] = includes + params['exclude[]'] = excludes + params['sort[]'] = orders + + for key, value in extras.items(): + params[key] = value + return params + + +class APIRecordSet(object): + + def __init__( + self, + manager=None, + pk=None, + filters=None, + orders=None, + includes=None, + excludes=None, + extras=None + ): + self.manager = manager + self.filters = filters or {} + self.includes = includes or [] + self.excludes = excludes or [] + self.orders = orders or [] + self.extras = extras or {} + # force-disable sideloading for easier loading + self.extras['sideloading'] = 'false' + self.pk = pk + self._reset() + + def get_params(self): + return build_params( + self.filters, + self.includes, + self.excludes, + self.orders, + self.extras + ) + + def _reset(self): + # current page of data + self._data = None + # iteration index on current page + self._index = None + # page number + self._page = None + # total number of pages + self._pages = None + + def _copy(self, **kwargs): + data = self.__dict__ + new_data = { + k: copy(v) + for k, v in data.items() if not k.startswith('_') + } + for key, value in kwargs.items(): + new_value = data.get(key) + if isinstance(new_value, dict): + if value != new_value: + new_value = copy(new_value) + new_value.update(value) + elif ( + isinstance(new_value, (list, tuple)) and + not isinstance(new_value, basestring) + ): + if value != new_value: + new_value = list(set(new_value + value)) + new_data[key] = new_value + return APIRecordSet(**new_data) + + def get(self, pk): + resource = self.manager.resource + client = resource.client + data = client.get( + '%s/%s' % (resource.name, pk), + params=self.get_params() + ) + return extract(data) + + def filter(self, **kwargs): + return self._copy(filters=kwargs) + + def include(self, *args): + return self._copy(includes=args) + + def exclude(self, *args): + return self._copy(excludes=args) + + def extra(self, **kwargs): + return self._copy(extras=kwargs) + + def order_by(self, *args): + return self._copy(orders=args) + + def _get_page(self, params): + if self._page is None: + self._page = 1 + else: + self._page += 1 + + resource = self.manager.resource + client = resource.client + + params['page'] = self._page + data = client.get(resource.name, params=params) + meta = data.get('meta', {}) + pages = meta.get('total_pages', 1) + + self._data = extract(data) + self._pages = pages + self._index = 0 + + def __iter__(self): + params = self.get_params() + self._get_page(params) + while True: + if self._index == len(self._data): + # end of page + if self._page == self._pages: + # end of results + self._reset() + raise StopIteration() + self._get_page(params) + + yield self._data[self._index] + self._index += 1 diff --git a/dynamic_rest/client/resource.py b/dynamic_rest/client/resource.py new file mode 100644 index 00000000..428d5248 --- /dev/null +++ b/dynamic_rest/client/resource.py @@ -0,0 +1,7 @@ +from .manager import APIResourceManager + +class APIResource(object): + def __init__(self, client, name): + self.name = name + self.client = client + self.objects = APIResourceManager(self) diff --git a/dynamic_rest/serializers.py b/dynamic_rest/serializers.py index 4843a239..77c25f7a 100644 --- a/dynamic_rest/serializers.py +++ b/dynamic_rest/serializers.py @@ -63,12 +63,6 @@ def id_only(self): """Get the child's rendering mode.""" return self.child.id_only() - @property - def data_no_envelope(self): - """For compatability with DRF's .data""" - data = self.data - return data.values()[0] - @property def data(self): """Get the data, after performing post-processing if necessary.""" diff --git a/dynamic_rest/viewsets.py b/dynamic_rest/viewsets.py index 38a89548..57ee3a84 100644 --- a/dynamic_rest/viewsets.py +++ b/dynamic_rest/viewsets.py @@ -59,7 +59,6 @@ class WithDynamicViewSetMixin(object): Attributes: features: A list of features supported by the viewset. - sideload: Whether or not to enable sideloading in the DynamicRenderer. meta: Extra data that is added to the response by the DynamicRenderer. """ diff --git a/install_requires.txt b/install_requires.txt index 20e14baa..06889931 100644 --- a/install_requires.txt +++ b/install_requires.txt @@ -1,3 +1,4 @@ Django>=1.7,<=1.10 djangorestframework>=3.1.0,<=3.4.0 inflection==0.3.1 +requests diff --git a/tests/test_serializers.py b/tests/test_serializers.py index 028acfe9..6fc8f39e 100644 --- a/tests/test_serializers.py +++ b/tests/test_serializers.py @@ -54,24 +54,6 @@ def test_data(self): ] }) - def test_data_no_envelope(self): - serializer = UserSerializer( - self.fixture.users, - many=True, - ) - self.assertEqual( - serializer.data_no_envelope, [ - OrderedDict( - [('id', 1), ('name', '0'), ('location', 1)]), - OrderedDict( - [('id', 2), ('name', '1'), ('location', 1)]), - OrderedDict( - [('id', 3), ('name', '2'), ('location', 2)]), - OrderedDict( - [('id', 4), ('name', '3'), ('location', 3)]) - ] - ) - def test_data_with_included_field(self): request_fields = { 'last_name': True From 018800a1f18f79c1d572dd3f28c33e38b31b6f6b Mon Sep 17 00:00:00 2001 From: aleontiev Date: Fri, 13 Jan 2017 15:47:26 -0800 Subject: [PATCH 08/35] fix include/exclude --- dynamic_rest/client/recordset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dynamic_rest/client/recordset.py b/dynamic_rest/client/recordset.py index 5e337f91..16e70cc8 100644 --- a/dynamic_rest/client/recordset.py +++ b/dynamic_rest/client/recordset.py @@ -86,7 +86,7 @@ def _copy(self, **kwargs): not isinstance(new_value, basestring) ): if value != new_value: - new_value = list(set(new_value + value)) + new_value = list(set(new_value + list(value))) new_data[key] = new_value return APIRecordSet(**new_data) From 1f9744ed55706a24f9091551caeda4ce1c1c370f Mon Sep 17 00:00:00 2001 From: aleontiev Date: Fri, 13 Jan 2017 16:17:55 -0800 Subject: [PATCH 09/35] fix lint errs --- dynamic_rest/client/client.py | 1 - dynamic_rest/client/manager.py | 2 ++ dynamic_rest/client/resource.py | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/dynamic_rest/client/client.py b/dynamic_rest/client/client.py index 13c32d70..80cda37b 100644 --- a/dynamic_rest/client/client.py +++ b/dynamic_rest/client/client.py @@ -1,6 +1,5 @@ import json import requests -import inflection from .exceptions import AuthenticationFailed from .resource import APIResource diff --git a/dynamic_rest/client/manager.py b/dynamic_rest/client/manager.py index 2b90db37..42b6692a 100644 --- a/dynamic_rest/client/manager.py +++ b/dynamic_rest/client/manager.py @@ -1,6 +1,8 @@ from .recordset import APIRecordSet + class APIResourceManager(object): + def __init__(self, resource): self.resource = resource diff --git a/dynamic_rest/client/resource.py b/dynamic_rest/client/resource.py index 428d5248..6956f44f 100644 --- a/dynamic_rest/client/resource.py +++ b/dynamic_rest/client/resource.py @@ -1,6 +1,8 @@ from .manager import APIResourceManager + class APIResource(object): + def __init__(self, client, name): self.name = name self.client = client From 0e8d0dc377580175f6ec75eee16cf1172011283b Mon Sep 17 00:00:00 2001 From: aleontiev Date: Tue, 17 Jan 2017 18:30:24 -0800 Subject: [PATCH 10/35] wip: queries --- dynamic_rest/client/__init__.py | 1 + dynamic_rest/client/client.py | 138 +++++++-------- dynamic_rest/client/exceptions.py | 8 + dynamic_rest/client/manager.py | 16 -- .../client/{recordset.py => query.py} | 164 ++++++++++-------- dynamic_rest/client/record.py | 45 +++++ dynamic_rest/client/resource.py | 82 ++++++++- dynamic_rest/client/utils.py | 3 + dynamic_rest/conf.py | 3 + dynamic_rest/fields/fields.py | 7 +- dynamic_rest/processors.py | 21 ++- dynamic_rest/serializers.py | 2 + dynamic_rest/viewsets.py | 13 ++ tests/serializers.py | 3 +- tests/test_client.py | 108 ++++++++++++ tests/viewsets.py | 5 +- 16 files changed, 439 insertions(+), 180 deletions(-) delete mode 100644 dynamic_rest/client/manager.py rename dynamic_rest/client/{recordset.py => query.py} (52%) create mode 100644 dynamic_rest/client/record.py create mode 100644 dynamic_rest/client/utils.py create mode 100644 tests/test_client.py diff --git a/dynamic_rest/client/__init__.py b/dynamic_rest/client/__init__.py index e69de29b..4b26e969 100644 --- a/dynamic_rest/client/__init__.py +++ b/dynamic_rest/client/__init__.py @@ -0,0 +1 @@ +from .client import DRESTClient # noqa diff --git a/dynamic_rest/client/client.py b/dynamic_rest/client/client.py index 80cda37b..6362b2b4 100644 --- a/dynamic_rest/client/client.py +++ b/dynamic_rest/client/client.py @@ -1,107 +1,86 @@ import json import requests -from .exceptions import AuthenticationFailed -from .resource import APIResource +from .exceptions import AuthenticationFailed, BadRequest +from .resource import DRESTResource -API_AUTH_ENDPOINT = '/accounts/login/' +AUTH_ENDPOINT = '/accounts/login/' -class APIClient(object): +class DRESTClient(object): def __init__( self, host, version=None, - session=None, - sessionid=None, - username=None, - password=None, - token=None, - authorization_type='JWT', - scheme='https' + client=None, + scheme='https', + authentication=None ): - self._authorization_type = authorization_type self._host = host self._version = version - self._session = session or requests.session() - self._session.headers.update({ + self._client = client or requests.session() + self._client.headers.update({ 'Content-Type': 'application/json', 'Accept': 'application/json' }) - self._username = username - self._password = password - self._scheme = scheme - self._authenticated = False self._resources = {} + self._scheme = scheme + self._authenticated = True + self._authentication = authentication + + if authentication: + self._authenticated = False + token = authentication.get('token') + sessionid = authentication.get('sessionid') + if token: + self._use_token(token) + if sessionid: + self._use_sessionid(sessionid) + + def __repr__(self): + return '%s%s' % ( + self._host, + '/%s/' % self._version if self._version else '' + ) - if token: - self.token = token - if sessionid: - self.sessionid = sessionid - - @property - def token(self): - return self._token - - @token.setter - def token(self, value): + def _use_token(self, value): self._token = value self._authenticated = bool(value) - if value: - self._session.headers.update({ - 'Authorization': '%s %s' % ( - self._authorization_type, - self._token - ) - }) - - @property - def sessionid(self): - return self._sessionid - - @sessionid.setter - def sessionid(self, value): + self._client.headers.update({ + 'Authorization': self._token if value else '' + }) + + def _use_sessionid(self, value): self._sessionid = value self._authenticated = bool(value) - if value: - self._session.headers.update({ - 'Cookie': 'sessionid=%s' % value - }) - - def post(self, url, params=None, data=None): - return self._request('POST', url, params, data) - - def get(self, url, params=None, data=None): - return self._request('GET', url, params, data) - - def put(self, url, params=None, data=None): - return self._request('PUT', url, params, data) - - def delete(self, url, params=None, data=None): - return self._request('DELETE', url, params, data) + self._client.headers.update({ + 'Cookie': 'sessionid=%s' % value if value else '' + }) def __getattr__(self, key): key = key.lower() - return self._resources.get(key, APIResource(self, key)) + return self._resources.get(key, DRESTResource(self, key)) + + def _login(self, raise_exception=True): + username = self._username + password = self._password + response = requests.post( + self._build_url(AUTH_ENDPOINT), + data={ + 'login': username, + 'password': password + }, + allow_redirects=False + ) + if raise_exception: + response.raise_for_status() + + self._use_sessionid(response.cookies.get('sessionid')) def _authenticate(self, raise_exception=True): response = None if not self._authenticated: - username = self._username - password = self._password - response = requests.post( - self._build_url(API_AUTH_ENDPOINT), - data={ - 'login': username, - 'password': password - }, - allow_redirects=False - ) - if raise_exception: - response.raise_for_status() - - self.sessionid = response.cookies.get('sessionid') - + self._login(self._username, self._password, raise_exception) if raise_exception and not self._authenticated: raise AuthenticationFailed( response.text if response else 'Unknown error' @@ -119,16 +98,19 @@ def _build_url(self, url, prefix=None): url = '%s%s' % (prefix, url) return '%s://%s%s' % (self._scheme, self._host, url) - def _request(self, method, url, params=None, data=None): + def request(self, method, url, params=None, data=None): self._authenticate() - response = self._session.request( + response = self._client.request( method, self._build_url(url, prefix=self._version), params=params, data=data ) + if response.status_code == 401: raise AuthenticationFailed() - response.raise_for_status() + if response.status_code >= 400: + raise BadRequest() + return json.loads(response.content) diff --git a/dynamic_rest/client/exceptions.py b/dynamic_rest/client/exceptions.py index 1840bc25..76bfe499 100644 --- a/dynamic_rest/client/exceptions.py +++ b/dynamic_rest/client/exceptions.py @@ -1,2 +1,10 @@ +class DoesNotExist(Exception): + pass + + class AuthenticationFailed(Exception): pass + + +class BadRequest(Exception): + pass diff --git a/dynamic_rest/client/manager.py b/dynamic_rest/client/manager.py deleted file mode 100644 index 42b6692a..00000000 --- a/dynamic_rest/client/manager.py +++ /dev/null @@ -1,16 +0,0 @@ -from .recordset import APIRecordSet - - -class APIResourceManager(object): - - def __init__(self, resource): - self.resource = resource - - def get(self, pk): - return APIRecordSet(self).get(pk) - - def filter(self, **kwargs): - return APIRecordSet(self, filters=kwargs) - - def all(self): - return APIRecordSet(self) diff --git a/dynamic_rest/client/recordset.py b/dynamic_rest/client/query.py similarity index 52% rename from dynamic_rest/client/recordset.py rename to dynamic_rest/client/query.py index 16e70cc8..c093c9bd 100644 --- a/dynamic_rest/client/recordset.py +++ b/dynamic_rest/client/query.py @@ -1,63 +1,105 @@ from copy import copy +from .utils import unpack -def extract(content): - keys = [k for k in content.keys() if k != 'meta'] - return content[keys[0]] - - -def build_params( - filters, - includes, - excludes, - orders, - extras -): - params = {} - for key, value in filters.items(): - filter_key = 'filter{%s}' % key.replace('__', '.') - params[filter_key] = value - - params['include[]'] = includes - params['exclude[]'] = excludes - params['sort[]'] = orders - - for key, value in extras.items(): - params[key] = value - return params - - -class APIRecordSet(object): +class DRESTQuery(object): def __init__( self, - manager=None, - pk=None, + resource=None, filters=None, orders=None, includes=None, excludes=None, extras=None ): - self.manager = manager + self.resource = resource self.filters = filters or {} self.includes = includes or [] self.excludes = excludes or [] self.orders = orders or [] self.extras = extras or {} - # force-disable sideloading for easier loading + # disable sideloading for easy loading self.extras['sideloading'] = 'false' - self.pk = pk + # enable debug for inline types + self.extras['debug'] = 'true' self._reset() - def get_params(self): - return build_params( - self.filters, - self.includes, - self.excludes, - self.orders, - self.extras - ) + def __repr__(self): + return 'Query: %s' % self.resource.name + + def all(self): + return self._copy() + + def list(self): + return list(self) + + def first(self): + l = self.list() + return l[0] if l else None + + def last(self): + l = self.list() + return l[-1] if l else None + + def map(self, field='id'): + return dict(( + (getattr(k, field), k) for k in self.list() + )) + + def get(self, id): + """Returns a single record by ID. + + Arguments: + id: a resource ID + """ + resource = self.resource + response = resource.request('get', id=id, params=self._get_params()) + return self._load(response) + + def filter(self, **kwargs): + return self._copy(filters=kwargs) + + def exclude(self, **kwargs): + filters = dict(('-' + k, v) for k, v in kwargs.items()) + return self._copy(filters=filters) + + def including(self, *args): + return self._copy(includes=args) + + def excluding(self, *args): + return self._copy(excludes=args) + + def extra(self, **kwargs): + return self._copy(extras=kwargs) + + def order_by(self, *args): + return self._copy(orders=args) + + def _get_params(self): + filters = self.filters + includes = self.includes + excludes = self.excludes + orders = self.orders + extras = self.extras + + params = {} + for key, value in filters.items(): + filter_key = 'filter{%s}' % key.replace('__', '.') + params[filter_key] = value + + if includes: + params['include[]'] = includes + + if excludes: + params['exclude[]'] = excludes + + if orders: + params['sort[]'] = orders + + for key, value in extras.items(): + params[key] = value + return params def _reset(self): # current page of data @@ -73,7 +115,8 @@ def _copy(self, **kwargs): data = self.__dict__ new_data = { k: copy(v) - for k, v in data.items() if not k.startswith('_') + for k, v in data.items() + if not k.startswith('_') } for key, value in kwargs.items(): new_value = data.get(key) @@ -88,31 +131,7 @@ def _copy(self, **kwargs): if value != new_value: new_value = list(set(new_value + list(value))) new_data[key] = new_value - return APIRecordSet(**new_data) - - def get(self, pk): - resource = self.manager.resource - client = resource.client - data = client.get( - '%s/%s' % (resource.name, pk), - params=self.get_params() - ) - return extract(data) - - def filter(self, **kwargs): - return self._copy(filters=kwargs) - - def include(self, *args): - return self._copy(includes=args) - - def exclude(self, *args): - return self._copy(excludes=args) - - def extra(self, **kwargs): - return self._copy(extras=kwargs) - - def order_by(self, *args): - return self._copy(orders=args) + return DRESTQuery(**new_data) def _get_page(self, params): if self._page is None: @@ -120,20 +139,21 @@ def _get_page(self, params): else: self._page += 1 - resource = self.manager.resource - client = resource.client - + resource = self.resource params['page'] = self._page - data = client.get(resource.name, params=params) + data = resource.request('get', params=params) meta = data.get('meta', {}) pages = meta.get('total_pages', 1) - self._data = extract(data) + self._data = self._load(data) self._pages = pages self._index = 0 + def _load(self, data): + return self.resource.load(unpack(data)) + def __iter__(self): - params = self.get_params() + params = self._get_params() self._get_page(params) while True: if self._index == len(self._data): diff --git a/dynamic_rest/client/record.py b/dynamic_rest/client/record.py new file mode 100644 index 00000000..207b99c6 --- /dev/null +++ b/dynamic_rest/client/record.py @@ -0,0 +1,45 @@ +from .utils import unpack +from .exceptions import DoesNotExist + + +class DRESTRecord(object): + FIELD_BLACKLIST = {'links'} + + def __init__(self, resource, **data): + self._resource = resource + self._load(data) + + def _get_data(self): + return { + k: v for k, v in self.__dict__.items() + if not k.startswith('_') and k not in self.FIELD_BLACKLIST + } + + def _load(self, data): + for key, value in data.items(): + if key.startswith('_'): + data.pop(key) + setattr(self, key, value) + + self.id = data.get('id', data.get('_id', None)) + + def __repr__(self): + return '%s.%s' % (self._resource.name, self.id if self.id else '') + + def save(self): + id = self.id + response = self._resource.request( + 'patch' if id else 'post', + id=self.id, + data=self._get_data() + ) + self._load(unpack(response)) + return response + + def reload(self): + id = self.id + if id: + response = self._resource.request('get', id=id) + self._load(unpack(response)) + else: + raise DoesNotExist() diff --git a/dynamic_rest/client/resource.py b/dynamic_rest/client/resource.py index 6956f44f..795562ae 100644 --- a/dynamic_rest/client/resource.py +++ b/dynamic_rest/client/resource.py @@ -1,9 +1,81 @@ -from .manager import APIResourceManager +from .query import DRESTQuery +from .record import DRESTRecord -class APIResource(object): +class DRESTResource(object): + """Represents a single resource in a DREST API. + + Arguments: + client: a DRESTClient + name: a resource's name + """ def __init__(self, client, name): - self.name = name - self.client = client - self.objects = APIResourceManager(self) + self.name = name.lower() + self._client = client + + def __repr__(self): + return self.name + + def request(self, method, id=None, params=None, data=None): + """Perform a request against this resource. + + Arguments: + method: HTTP method (get, put, post, delete, options) + id: resource ID. by default, assume no ID + params: HTTP params + data: HTTP data + """ + return self._client.request( + method, + self._get_url(id), + params=params, + data=data + ) + + def load(self, data): + """Loads data to an internal representation. + + Arguments: + data: array or object + Returns: + Array of DRESTRecord or single DRESTRecord. + """ + if isinstance(data, dict): + name = data.get('_type') + pk = data.get('_id') or data.get('id') + if name and pk: + # load from dict + data['id'] = pk + if name == self.name: + for key, value in data.items(): + _value = self.load(value) + if value != _value: + data[key] = _value + return DRESTRecord(resource=self, **data) + else: + resource = getattr(self._client, name) + return resource.load(data) + else: + # plain dict + return data + elif isinstance(data, list) and not isinstance(data, basestring): + for i, value in enumerate(data): + # load from list + _value = self.load(value) + if value != _value: + data[i] = _value + return data + else: + return data + + def create(self, **kwargs): + record = DRESTRecord(resource=self, **kwargs) + record.save() + return record + + def __getattr__(self, value): + return getattr(DRESTQuery(self), value) + + def _get_url(self, id=None): + return '%s%s' % (self.name, '/%s' % id if id else '') diff --git a/dynamic_rest/client/utils.py b/dynamic_rest/client/utils.py new file mode 100644 index 00000000..95a84cf6 --- /dev/null +++ b/dynamic_rest/client/utils.py @@ -0,0 +1,3 @@ +def unpack(content): + keys = [k for k in content.keys() if k != 'meta'] + return content[keys[0]] diff --git a/dynamic_rest/conf.py b/dynamic_rest/conf.py index 7cf0fbb0..2441ef8f 100644 --- a/dynamic_rest/conf.py +++ b/dynamic_rest/conf.py @@ -5,6 +5,9 @@ # DEBUG: enable/disable internal debugging 'DEBUG': False, + # AUTH_ENDPOINT: authentication endpoint (used by DREST client) + 'AUTH_ENDPOINT': '/accounts/login/', + # ENABLE_BROWSABLE_API: enable/disable the browsable API. # It can be useful to disable it in production. 'ENABLE_BROWSABLE_API': True, diff --git a/dynamic_rest/fields/fields.py b/dynamic_rest/fields/fields.py index 6966154d..a1e64e2b 100644 --- a/dynamic_rest/fields/fields.py +++ b/dynamic_rest/fields/fields.py @@ -83,6 +83,7 @@ def __init__( queryset=None, embed=False, sideloading=None, + debug=False, **kwargs ): """ @@ -99,6 +100,7 @@ def __init__( self.bound = False self.queryset = queryset self.sideloading = sideloading + self.debug = debug self.embed = embed if sideloading is None else not sideloading if '.' in kwargs.get('source', ''): raise Exception('Nested relationships are not supported') @@ -224,6 +226,9 @@ def _inherit_parent_kwargs(self, kwargs): if hasattr(self.parent, 'sideloading'): kwargs['sideloading'] = self.parent.sideloading + if hasattr(self.parent, 'debug'): + kwargs['debug'] = self.parent.debug + return kwargs def get_serializer(self, *args, **kwargs): @@ -288,7 +293,7 @@ def to_representation(self, instance): return serializer.to_representation(related) except Exception as e: # Provide more context to help debug these cases - if settings.DEBUG: + if serializer.debug: import traceback traceback.print_exc() raise Exception( diff --git a/dynamic_rest/processors.py b/dynamic_rest/processors.py index 8e48afaa..a7ce781f 100644 --- a/dynamic_rest/processors.py +++ b/dynamic_rest/processors.py @@ -31,6 +31,7 @@ def __init__(self, serializer, data): serializer = serializer.child self.data = {} self.seen = defaultdict(set) + self.debug = serializer.debug self.plural_name = serializer.get_plural_name() self.name = serializer.get_name() @@ -77,13 +78,23 @@ def process(self, obj, parent=None, parent_key=None, depth=0): 1 ) - if ( - not dynamic or getattr(obj, 'embed', False) - ): + serializer = obj.serializer + name = serializer.get_plural_name() + instance = getattr(obj, 'instance', serializer.instance) + if not instance: return - name = obj.serializer.get_plural_name() - pk = obj.pk_value or obj.instance.pk + pk = getattr(obj, 'pk_value', instance.pk) or instance.pk + + if self.debug: + obj['_id'] = pk + obj['_type'] = name + + if not dynamic: + return + + if getattr(obj, 'embed', False): + return # For polymorphic relations, `pk` can be a dict, so use the # string representation (dict isn't hashable). diff --git a/dynamic_rest/serializers.py b/dynamic_rest/serializers.py index 77c25f7a..339677b3 100644 --- a/dynamic_rest/serializers.py +++ b/dynamic_rest/serializers.py @@ -158,6 +158,7 @@ def __init__( exclude_fields=None, request_fields=None, sideloading=None, + debug=False, dynamic=True, embed=False, **kwargs @@ -201,6 +202,7 @@ def __init__( super(WithDynamicSerializerMixin, self).__init__(**kwargs) self.sideloading = sideloading + self.debug = debug self.dynamic = dynamic self.request_fields = request_fields or {} self.embed = embed if sideloading is None else not sideloading diff --git a/dynamic_rest/viewsets.py b/dynamic_rest/viewsets.py index 57ee3a84..676b26ce 100644 --- a/dynamic_rest/viewsets.py +++ b/dynamic_rest/viewsets.py @@ -62,6 +62,7 @@ class WithDynamicViewSetMixin(object): meta: Extra data that is added to the response by the DynamicRenderer. """ + DEBUG = 'debug' SIDELOADING = 'sideloading' INCLUDE = 'include[]' EXCLUDE = 'exclude[]' @@ -231,6 +232,16 @@ def get_request_fields(self): self._request_fields = request_fields return request_fields + def get_request_debug(self): + debug = self.get_request_feature(self.DEBUG) + if debug: + debug = debug .lower() + if debug == 'true': + debug = True + else: + debug = False + return debug + def get_request_sideloading(self): sideloading = self.get_request_feature(self.SIDELOADING) if sideloading: @@ -264,6 +275,8 @@ def get_serializer(self, *args, **kwargs): kwargs['request_fields'] = self.get_request_fields() if 'sideloading' not in kwargs: kwargs['sideloading'] = self.get_request_sideloading() + if 'debug' not in kwargs: + kwargs['debug'] = self.get_request_debug() if self.is_update(): kwargs['include_fields'] = '*' return super( diff --git a/tests/serializers.py b/tests/serializers.py index 3e46e424..e4fc2fac 100644 --- a/tests/serializers.py +++ b/tests/serializers.py @@ -190,7 +190,8 @@ class Meta: favorite_pet = DynamicGenericRelationField(required=False) def get_number_of_cats(self, user): - return len(user.location.cat_set.all()) + location = user.location + return len(location.cat_set.all()) if location else 0 class ProfileSerializer(DynamicModelSerializer): diff --git a/tests/test_client.py b/tests/test_client.py new file mode 100644 index 00000000..47e5cfbc --- /dev/null +++ b/tests/test_client.py @@ -0,0 +1,108 @@ +from rest_framework.test import APITestCase, APIClient +from dynamic_rest.client import DRESTClient +from tests.setup import create_fixture +import urllib + + +class IntegrationTestClient(object): + """requests.session compatiability adapter for DRESTClient.""" + def __init__(self, client): + self._client = client or APIClient() + self.headers = {} + self.status_code = 200 + + def request(self, method, url, params=None, data=None): + def make_params(params): + list_params = [] + for key, value in params.items(): + if isinstance( + value, basestring + ) or not isinstance(value, list): + value = [value] + for v in value: + list_params.append((key, v)) + return urllib.urlencode(list_params) + + url = '%s%s' % ( + url, + ('?%s' % make_params(params)) if params else '' + ) + url = url.replace('https://test', '') + response = getattr(self._client, method)(url, data=data) + return response + + +class ClientTestCase(APITestCase): + + def setUp(self): + self.fixture = create_fixture() + self.drest = DRESTClient( + 'test', + client=IntegrationTestClient(self.client) + ) + + def test_get_all(self): + users = list(self.drest.Users.all()) + self.assertEquals(len(users), len(self.fixture.users)) + self.assertEquals( + {user.name for user in users}, + {user.name for user in self.fixture.users} + ) + + def test_get_filter(self): + users = self.drest.Users.filter(id=self.fixture.users[0].id).list() + self.assertEquals(1, len(users)) + self.assertEquals(users[0].name, self.fixture.users[0].name) + + def test_get_one(self): + fixed_user = self.fixture.users[0] + pk = fixed_user.pk + user = self.drest.Users.get(pk) + self.assertEquals(user.name, fixed_user.name) + + def test_get_include(self): + users = self.drest.Users.including('location.*').list() + self.assertEquals(users[0].location.name, '0') + + def test_get_map(self): + users = self.drest.Users.map() + id = self.fixture.users[0].pk + self.assertEquals(users[id].id, id) + + def test_get_exclude(self): + user = self.fixture.users[0] + users = self.drest.Users.exclude(name=user.name).map() + self.assertTrue(user.pk not in users) + + def test_get_including_related_save(self): + users = self.drest.Users.including('location.*').list() + user = users[0] + location = user.location + _location = self.drest.Locations.map()[location.id] + self.assertTrue( + location.name in {l.name for l in self.fixture.locations} + ) + _location.name = 'foo' + _location.save() + location.reload() + self.assertTrue(_location.name, location.name) + self.assertTrue(location.name, 'foo') + + def test_get_excluding(self): + users = self.drest.Users.excluding('*').map() + pk = self.fixture.users[0].pk + self.assertEquals(users[pk].id, pk) + + def test_update(self): + user = self.drest.Users.first() + user.name = 'foo' + user.save() + user = self.drest.Users.first() + self.assertTrue(user.name, 'foo') + + def test_create(self): + user = self.drest.Users.create( + name='foo', + last_name='bar' + ) + self.assertIsNotNone(user.id) diff --git a/tests/viewsets.py b/tests/viewsets.py index be4c7f2b..03f2cc1a 100644 --- a/tests/viewsets.py +++ b/tests/viewsets.py @@ -30,7 +30,7 @@ class UserViewSet(DynamicModelViewSet): features = ( DynamicModelViewSet.INCLUDE, DynamicModelViewSet.EXCLUDE, DynamicModelViewSet.FILTER, DynamicModelViewSet.SORT, - DynamicModelViewSet.SIDELOADING + DynamicModelViewSet.SIDELOADING, DynamicModelViewSet.DEBUG ) model = User serializer_class = UserSerializer @@ -93,7 +93,8 @@ class GroupViewSet(DynamicModelViewSet): class LocationViewSet(DynamicModelViewSet): features = ( DynamicModelViewSet.INCLUDE, DynamicModelViewSet.EXCLUDE, - DynamicModelViewSet.FILTER, DynamicModelViewSet.SORT + DynamicModelViewSet.FILTER, DynamicModelViewSet.SORT, + DynamicModelViewSet.DEBUG, DynamicModelViewSet.SIDELOADING ) model = Location serializer_class = LocationSerializer From dd2ce850240aef67ef374773e2114d9cb1f08311 Mon Sep 17 00:00:00 2001 From: aleontiev Date: Tue, 17 Jan 2017 18:38:06 -0800 Subject: [PATCH 11/35] test --- dynamic_rest/client/client.py | 5 ++++- tests/test_client.py | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/dynamic_rest/client/client.py b/dynamic_rest/client/client.py index 6362b2b4..cf432b3a 100644 --- a/dynamic_rest/client/client.py +++ b/dynamic_rest/client/client.py @@ -1,6 +1,6 @@ import json import requests -from .exceptions import AuthenticationFailed, BadRequest +from .exceptions import AuthenticationFailed, BadRequest, DoesNotExist from .resource import DRESTResource AUTH_ENDPOINT = '/accounts/login/' @@ -110,6 +110,9 @@ def request(self, method, url, params=None, data=None): if response.status_code == 401: raise AuthenticationFailed() + if response.status_code == 404: + raise DoesNotExist() + if response.status_code >= 400: raise BadRequest() diff --git a/tests/test_client.py b/tests/test_client.py index 47e5cfbc..459d3384 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,5 +1,8 @@ from rest_framework.test import APITestCase, APIClient from dynamic_rest.client import DRESTClient +from dynamic_rest.client.exceptions import ( + BadRequest, DoesNotExist +) from tests.setup import create_fixture import urllib @@ -106,3 +109,19 @@ def test_create(self): last_name='bar' ) self.assertIsNotNone(user.id) + + def test_invalid_resource(self): + with self.assertRaises(DoesNotExist): + self.drest.Foo.create(name='foo') + with self.assertRaises(DoesNotExist): + self.drest.Foo.filter(name='foo').list() + + def test_save_invalid_data(self): + user = self.drest.Users.first() + user.name = '' + with self.assertRaises(BadRequest): + user.save() + + def test_get_invalid_data(self): + with self.assertRaises(DoesNotExist): + self.drest.Users.get('does-not-exist') From 801799bba5739c166d716bd2bae7ed7abd217b86 Mon Sep 17 00:00:00 2001 From: aleontiev Date: Tue, 17 Jan 2017 18:42:12 -0800 Subject: [PATCH 12/35] fix tests / use settings --- dynamic_rest/client/client.py | 5 ++--- tests/test_api.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/dynamic_rest/client/client.py b/dynamic_rest/client/client.py index cf432b3a..1ab24af9 100644 --- a/dynamic_rest/client/client.py +++ b/dynamic_rest/client/client.py @@ -2,8 +2,7 @@ import requests from .exceptions import AuthenticationFailed, BadRequest, DoesNotExist from .resource import DRESTResource - -AUTH_ENDPOINT = '/accounts/login/' +from dynamic_rest.conf import settings class DRESTClient(object): @@ -65,7 +64,7 @@ def _login(self, raise_exception=True): username = self._username password = self._password response = requests.post( - self._build_url(AUTH_ENDPOINT), + self._build_url(settings.AUTH_ENDPOINT), data={ 'login': username, 'password': password diff --git a/tests/test_api.py b/tests/test_api.py index 0fe39682..621e82a5 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -876,7 +876,6 @@ def test_options(self): actual = json.loads(response.content.decode('utf-8')) expected = { 'description': '', - 'features': ['include[]', 'exclude[]', 'filter{}', 'sort[]'], 'name': 'Location List', 'parses': [ 'application/json', @@ -969,6 +968,7 @@ def test_options(self): for field in ['cats', 'friendly_cats', 'bad_cats', 'users']: del actual['properties'][field]['nullable'] del expected['properties'][field]['nullable'] + actual.pop('features') self.assertEquals( json.loads(json.dumps(expected)), json.loads(json.dumps(actual)) From eb410a32464a2b9655f45e3801c7386388c4cd62 Mon Sep 17 00:00:00 2001 From: aleontiev Date: Tue, 17 Jan 2017 18:52:46 -0800 Subject: [PATCH 13/35] change debug-tagging --- dynamic_rest/processors.py | 16 ++++------------ dynamic_rest/serializers.py | 4 ++++ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/dynamic_rest/processors.py b/dynamic_rest/processors.py index a7ce781f..2ce9e894 100644 --- a/dynamic_rest/processors.py +++ b/dynamic_rest/processors.py @@ -31,7 +31,6 @@ def __init__(self, serializer, data): serializer = serializer.child self.data = {} self.seen = defaultdict(set) - self.debug = serializer.debug self.plural_name = serializer.get_plural_name() self.name = serializer.get_name() @@ -78,24 +77,17 @@ def process(self, obj, parent=None, parent_key=None, depth=0): 1 ) + if not dynamic or getattr(obj, 'embed', False): + return + serializer = obj.serializer name = serializer.get_plural_name() instance = getattr(obj, 'instance', serializer.instance) + if not instance: return pk = getattr(obj, 'pk_value', instance.pk) or instance.pk - - if self.debug: - obj['_id'] = pk - obj['_type'] = name - - if not dynamic: - return - - if getattr(obj, 'embed', False): - return - # For polymorphic relations, `pk` can be a dict, so use the # string representation (dict isn't hashable). pk_key = repr(pk) diff --git a/dynamic_rest/serializers.py b/dynamic_rest/serializers.py index 339677b3..8437a3f5 100644 --- a/dynamic_rest/serializers.py +++ b/dynamic_rest/serializers.py @@ -527,6 +527,10 @@ def to_representation(self, instance): self, representation, instance ) + if self.debug: + representation['_id'] = instance.pk + representation['_type'] = self.get_plural_name() + # tag the representation with the serializer and instance return tag_dict( representation, From bb4600951c65abd4bb9ba462c1ea8ef0bd5034f5 Mon Sep 17 00:00:00 2001 From: aleontiev Date: Tue, 17 Jan 2017 19:07:12 -0800 Subject: [PATCH 14/35] wrap _meta --- dynamic_rest/client/record.py | 2 +- dynamic_rest/client/resource.py | 5 +++-- dynamic_rest/serializers.py | 6 ++++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/dynamic_rest/client/record.py b/dynamic_rest/client/record.py index 207b99c6..ff16a2c6 100644 --- a/dynamic_rest/client/record.py +++ b/dynamic_rest/client/record.py @@ -21,7 +21,7 @@ def _load(self, data): data.pop(key) setattr(self, key, value) - self.id = data.get('id', data.get('_id', None)) + self.id = data.get('_meta', {}).get('id', data.get('id', None)) def __repr__(self): return '%s.%s' % (self._resource.name, self.id if self.id else '') diff --git a/dynamic_rest/client/resource.py b/dynamic_rest/client/resource.py index 795562ae..634d1e3a 100644 --- a/dynamic_rest/client/resource.py +++ b/dynamic_rest/client/resource.py @@ -42,8 +42,9 @@ def load(self, data): Array of DRESTRecord or single DRESTRecord. """ if isinstance(data, dict): - name = data.get('_type') - pk = data.get('_id') or data.get('id') + meta = data.get('_meta', {}) + name = meta.get('type') + pk = meta.get('id') if name and pk: # load from dict data['id'] = pk diff --git a/dynamic_rest/serializers.py b/dynamic_rest/serializers.py index 8437a3f5..aee7b291 100644 --- a/dynamic_rest/serializers.py +++ b/dynamic_rest/serializers.py @@ -528,8 +528,10 @@ def to_representation(self, instance): ) if self.debug: - representation['_id'] = instance.pk - representation['_type'] = self.get_plural_name() + representation['_meta'] = { + 'id': instance.pk, + 'type': self.get_plural_name() + } # tag the representation with the serializer and instance return tag_dict( From e1715089ad8f7c815ba335368763f6d91eccaa31 Mon Sep 17 00:00:00 2001 From: aleontiev Date: Tue, 17 Jan 2017 19:17:05 -0800 Subject: [PATCH 15/35] only send changed fields --- dynamic_rest/client/record.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/dynamic_rest/client/record.py b/dynamic_rest/client/record.py index ff16a2c6..9725a407 100644 --- a/dynamic_rest/client/record.py +++ b/dynamic_rest/client/record.py @@ -3,24 +3,36 @@ class DRESTRecord(object): + # TODO: use metadata to figure out which fields to ignore FIELD_BLACKLIST = {'links'} def __init__(self, resource, **data): self._resource = resource self._load(data) - def _get_data(self): + @property + def _data(self): return { k: v for k, v in self.__dict__.items() if not k.startswith('_') and k not in self.FIELD_BLACKLIST } + @property + def _diff(self): + data = self._data + for key, value in data.items(): + if key in self._clean and value == self._clean[key]: + data.pop(key) + + return data + def _load(self, data): for key, value in data.items(): if key.startswith('_'): data.pop(key) setattr(self, key, value) + self._clean = self._data self.id = data.get('_meta', {}).get('id', data.get('id', None)) def __repr__(self): @@ -28,13 +40,14 @@ def __repr__(self): def save(self): id = self.id - response = self._resource.request( - 'patch' if id else 'post', - id=self.id, - data=self._get_data() - ) - self._load(unpack(response)) - return response + data = self._diff if id else self._data + if data: + response = self._resource.request( + 'patch' if id else 'post', + id=id, + data=data + ) + self._load(unpack(response)) def reload(self): id = self.id From c257669e7ca3912da035d38d4903ba0474339859 Mon Sep 17 00:00:00 2001 From: aleontiev Date: Tue, 17 Jan 2017 19:33:39 -0800 Subject: [PATCH 16/35] test extra, equality, remove blacklist --- dynamic_rest/client/record.py | 22 ++++++++++++++++------ tests/test_client.py | 10 ++++++++++ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/dynamic_rest/client/record.py b/dynamic_rest/client/record.py index 9725a407..7c6b8bd8 100644 --- a/dynamic_rest/client/record.py +++ b/dynamic_rest/client/record.py @@ -3,18 +3,18 @@ class DRESTRecord(object): - # TODO: use metadata to figure out which fields to ignore - FIELD_BLACKLIST = {'links'} - def __init__(self, resource, **data): self._resource = resource self._load(data) + def __eq__(self, other): + return self._data == other._data + @property def _data(self): return { k: v for k, v in self.__dict__.items() - if not k.startswith('_') and k not in self.FIELD_BLACKLIST + if not k.startswith('_') } @property @@ -23,7 +23,6 @@ def _diff(self): for key, value in data.items(): if key in self._clean and value == self._clean[key]: data.pop(key) - return data def _load(self, data): @@ -38,9 +37,20 @@ def _load(self, data): def __repr__(self): return '%s.%s' % (self._resource.name, self.id if self.id else '') + def _serialize(self, data): + for key, values in data.items(): + if isinstance(values, list): + if len(values) > 0: + for i, value in enumerate(values): + if isinstance(value, DRESTRecord): + values[i] = value.id + elif isinstance(values, DRESTRecord): + data[key] = values.id + return data + def save(self): id = self.id - data = self._diff if id else self._data + data = self._serialize(self._diff if id else self._data) if data: response = self._resource.request( 'patch' if id else 'post', diff --git a/tests/test_client.py b/tests/test_client.py index 459d3384..100cce0a 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -125,3 +125,13 @@ def test_save_invalid_data(self): def test_get_invalid_data(self): with self.assertRaises(DoesNotExist): self.drest.Users.get('does-not-exist') + + def test_extra_pagination(self): + users = list(self.drest.Users.all().extra(per_page=1)) + users2 = list(self.drest.Users.all()) + self.assertEquals(users, users2) + + name = users[0].name + users_named = list(self.drest.Users.extra(name=name)) + self.assertTrue(len(users_named), 1) + self.assertTrue(users_named[0].name, name) From 30cf31384213e1459c941daf4b8b41b3f6f6344c Mon Sep 17 00:00:00 2001 From: aleontiev Date: Tue, 17 Jan 2017 19:36:54 -0800 Subject: [PATCH 17/35] test for saving a deferred instance --- tests/test_client.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_client.py b/tests/test_client.py index 100cce0a..945b402a 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -45,7 +45,7 @@ def setUp(self): ) def test_get_all(self): - users = list(self.drest.Users.all()) + users = self.drest.Users.all().list() self.assertEquals(len(users), len(self.fixture.users)) self.assertEquals( {user.name for user in users}, @@ -135,3 +135,11 @@ def test_extra_pagination(self): users_named = list(self.drest.Users.extra(name=name)) self.assertTrue(len(users_named), 1) self.assertTrue(users_named[0].name, name) + + def test_save_deferred(self): + user = self.drest.Users.excluding('*').list()[0] + user.name = 'foo' + user.save() + + user2 = self.drest.Users.first() + self.assertEquals(user2.name, user.name) From d90dbdff14b5d5919008c15b4f6771ad58402f98 Mon Sep 17 00:00:00 2001 From: aleontiev Date: Tue, 17 Jan 2017 19:49:34 -0800 Subject: [PATCH 18/35] add docs --- dynamic_rest/client/client.py | 60 +++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/dynamic_rest/client/client.py b/dynamic_rest/client/client.py index 1ab24af9..4b64dae0 100644 --- a/dynamic_rest/client/client.py +++ b/dynamic_rest/client/client.py @@ -6,7 +6,67 @@ class DRESTClient(object): + """DREST Python client. + Exposes a DREST API to Python using a Django-esque interface. + Resources are available on the client through access-by-name. + + Arguments: + host: hostname to a DREST API + version: version (defaults to no version), + client: HTTP client (defaults to requests.session), + scheme: defaults to https + authentication: provides credentials + + Examples: + + Getting a client: + + client = DRESTClient('my.api.io', authentication={'token': 'secret'}) + + Working wiht a resource: + + User = client.users + + Getting a single resource: + + User.get('123') + + Getting all resources (auto-paging): + + User.all() + + Getting filtered resources: + + User.filter(name__icontains='john') + other_users = client.users.exclude(name__icontains='john') + + Including / excluding fields: + + users = User.all() + .excluding('birthday') + .including('events.*') + .get('123') + + Mapping by field: + + users_by_id = User.map() + users_by_name = User.map('name') + + Ordering results: + + users = User.order_by('-name') + + Updating records: + + user = User.first() + user.name = 'john' + user.save() + + Creating resources: + + user = User.create(name='john') + """ def __init__( self, host, From 042f3a036622b39fd60597aa9696837654d205bf Mon Sep 17 00:00:00 2001 From: aleontiev Date: Tue, 17 Jan 2017 19:51:42 -0800 Subject: [PATCH 19/35] .. --- dynamic_rest/client/client.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/dynamic_rest/client/client.py b/dynamic_rest/client/client.py index 4b64dae0..b8dd1760 100644 --- a/dynamic_rest/client/client.py +++ b/dynamic_rest/client/client.py @@ -24,48 +24,48 @@ class DRESTClient(object): client = DRESTClient('my.api.io', authentication={'token': 'secret'}) - Working wiht a resource: + Working with a resource: - User = client.users + Users = client.Users Getting a single resource: - User.get('123') + Users.get('123') Getting all resources (auto-paging): - User.all() + Users.all() Getting filtered resources: - User.filter(name__icontains='john') + Users.filter(name__icontains='john') other_users = client.users.exclude(name__icontains='john') Including / excluding fields: - users = User.all() + users = Users.all() .excluding('birthday') .including('events.*') .get('123') Mapping by field: - users_by_id = User.map() - users_by_name = User.map('name') + users_by_id = Users.map() + users_by_name = Users.map('name') Ordering results: - users = User.order_by('-name') + users = Users.order_by('-name') Updating records: - user = User.first() + user = Users.first() user.name = 'john' user.save() Creating resources: - user = User.create(name='john') + user = Users.create(name='john') """ def __init__( self, From cbd996431709def57c702df8c3b07c06ae97bb9e Mon Sep 17 00:00:00 2001 From: aleontiev Date: Tue, 17 Jan 2017 19:53:59 -0800 Subject: [PATCH 20/35] .. --- dynamic_rest/serializers.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/dynamic_rest/serializers.py b/dynamic_rest/serializers.py index aee7b291..c3b851d2 100644 --- a/dynamic_rest/serializers.py +++ b/dynamic_rest/serializers.py @@ -582,12 +582,6 @@ def id_only(self): """ return self.dynamic and self.request_fields is True - @property - def data_no_envelope(self): - """For compatability with DRF's .data""" - data = self.data - return data.values()[0] - @property def data(self): if not hasattr(self, '_processed_data'): From 85c2c9c9cef7c514051151c08247b6b459074756 Mon Sep 17 00:00:00 2001 From: aleontiev Date: Tue, 17 Jan 2017 19:54:54 -0800 Subject: [PATCH 21/35] .. --- dynamic_rest/client/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dynamic_rest/client/client.py b/dynamic_rest/client/client.py index b8dd1760..aa4f839d 100644 --- a/dynamic_rest/client/client.py +++ b/dynamic_rest/client/client.py @@ -13,8 +13,8 @@ class DRESTClient(object): Arguments: host: hostname to a DREST API - version: version (defaults to no version), - client: HTTP client (defaults to requests.session), + version: version (defaults to no version) + client: HTTP client (defaults to requests.session) scheme: defaults to https authentication: provides credentials From 2eaf306ecfc6aa709912f5adb26b8ef4f08e4a12 Mon Sep 17 00:00:00 2001 From: aleontiev Date: Tue, 17 Jan 2017 19:55:42 -0800 Subject: [PATCH 22/35] rename to MockSession --- tests/test_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_client.py b/tests/test_client.py index 945b402a..61270686 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -7,7 +7,7 @@ import urllib -class IntegrationTestClient(object): +class MockSession(object): """requests.session compatiability adapter for DRESTClient.""" def __init__(self, client): self._client = client or APIClient() @@ -41,7 +41,7 @@ def setUp(self): self.fixture = create_fixture() self.drest = DRESTClient( 'test', - client=IntegrationTestClient(self.client) + client=MockSession(self.client) ) def test_get_all(self): From ea356fd532acdecdc235efdce47d0eb4ed78e032 Mon Sep 17 00:00:00 2001 From: aleontiev Date: Tue, 17 Jan 2017 20:12:13 -0800 Subject: [PATCH 23/35] .. --- dynamic_rest/client/client.py | 29 +++++++++++++++++++---------- dynamic_rest/conf.py | 8 +++++++- tests/test_client.py | 7 +------ 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/dynamic_rest/client/client.py b/dynamic_rest/client/client.py index aa4f839d..4e6dd0b4 100644 --- a/dynamic_rest/client/client.py +++ b/dynamic_rest/client/client.py @@ -16,7 +16,14 @@ class DRESTClient(object): version: version (defaults to no version) client: HTTP client (defaults to requests.session) scheme: defaults to https - authentication: provides credentials + authentication: if unset, authentication is disabled. + If set, provides credentials: { + usename: login username, + password: login password, + token: authorization token, + cookie: session cookie + } + Either username/password, token, or cookie should be provided. Examples: @@ -90,11 +97,11 @@ def __init__( if authentication: self._authenticated = False token = authentication.get('token') - sessionid = authentication.get('sessionid') + cookie = authentication.get('cookie') if token: self._use_token(token) - if sessionid: - self._use_sessionid(sessionid) + if cookie: + self._use_cookie(cookie) def __repr__(self): return '%s%s' % ( @@ -106,14 +113,16 @@ def _use_token(self, value): self._token = value self._authenticated = bool(value) self._client.headers.update({ - 'Authorization': self._token if value else '' + 'Authorization': '%s %s' % ( + settings.AUTH_TYPE, self._token if value else '' + ) }) - def _use_sessionid(self, value): - self._sessionid = value + def _use_cookie(self, value): + self._cookie = value self._authenticated = bool(value) self._client.headers.update({ - 'Cookie': 'sessionid=%s' % value if value else '' + 'Cookie': '%s=%s' % (settings.AUTH_COOKIE_NAME, value) }) def __getattr__(self, key): @@ -124,7 +133,7 @@ def _login(self, raise_exception=True): username = self._username password = self._password response = requests.post( - self._build_url(settings.AUTH_ENDPOINT), + self._build_url(settings.AUTH_LOGIN_ENDPOINT), data={ 'login': username, 'password': password @@ -134,7 +143,7 @@ def _login(self, raise_exception=True): if raise_exception: response.raise_for_status() - self._use_sessionid(response.cookies.get('sessionid')) + self._use_cookie(response.cookies.get(settings.AUTH_COOKIE_NAME)) def _authenticate(self, raise_exception=True): response = None diff --git a/dynamic_rest/conf.py b/dynamic_rest/conf.py index 2441ef8f..e07c2814 100644 --- a/dynamic_rest/conf.py +++ b/dynamic_rest/conf.py @@ -6,7 +6,13 @@ 'DEBUG': False, # AUTH_ENDPOINT: authentication endpoint (used by DREST client) - 'AUTH_ENDPOINT': '/accounts/login/', + 'AUTH_LOGIN_ENDPOINT': '/accounts/login/', + + # AUTH_COOKIE_NAME: sessionid cookie + 'AUTH_COOKIE_NAME': 'sessionid', + + # AUTH_TYPE: authentication type + 'AUTH_TYPE': 'JWT', # ENABLE_BROWSABLE_API: enable/disable the browsable API. # It can be useful to disable it in production. diff --git a/tests/test_client.py b/tests/test_client.py index 61270686..a65f4f97 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -12,7 +12,6 @@ class MockSession(object): def __init__(self, client): self._client = client or APIClient() self.headers = {} - self.status_code = 200 def request(self, method, url, params=None, data=None): def make_params(params): @@ -30,7 +29,6 @@ def make_params(params): url, ('?%s' % make_params(params)) if params else '' ) - url = url.replace('https://test', '') response = getattr(self._client, method)(url, data=data) return response @@ -39,10 +37,7 @@ class ClientTestCase(APITestCase): def setUp(self): self.fixture = create_fixture() - self.drest = DRESTClient( - 'test', - client=MockSession(self.client) - ) + self.drest = DRESTClient('test', client=MockSession(self.client)) def test_get_all(self): users = self.drest.Users.all().list() From 3ae3adf0d5d7a3f19750fe68fd92fdf49e6a2656 Mon Sep 17 00:00:00 2001 From: aleontiev Date: Fri, 20 Jan 2017 10:26:58 -0800 Subject: [PATCH 24/35] revert sideloading processor change requiring instance to be defined (ephemeral) --- dynamic_rest/processors.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dynamic_rest/processors.py b/dynamic_rest/processors.py index 2ce9e894..ef803181 100644 --- a/dynamic_rest/processors.py +++ b/dynamic_rest/processors.py @@ -83,11 +83,9 @@ def process(self, obj, parent=None, parent_key=None, depth=0): serializer = obj.serializer name = serializer.get_plural_name() instance = getattr(obj, 'instance', serializer.instance) + instance_pk = instance.pk if instance else None + pk = getattr(obj, 'pk_value', instance_pk) or instance_pk - if not instance: - return - - pk = getattr(obj, 'pk_value', instance.pk) or instance.pk # For polymorphic relations, `pk` can be a dict, so use the # string representation (dict isn't hashable). pk_key = repr(pk) From e28eee0220bbf4f8708fc376678cc3d87e4df1e5 Mon Sep 17 00:00:00 2001 From: aleontiev Date: Tue, 24 Jan 2017 16:00:50 -0800 Subject: [PATCH 25/35] .. --- dynamic_rest/client/query.py | 2 +- dynamic_rest/client/resource.py | 2 +- tests/test_client.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dynamic_rest/client/query.py b/dynamic_rest/client/query.py index c093c9bd..90dbfb64 100644 --- a/dynamic_rest/client/query.py +++ b/dynamic_rest/client/query.py @@ -126,7 +126,7 @@ def _copy(self, **kwargs): new_value.update(value) elif ( isinstance(new_value, (list, tuple)) and - not isinstance(new_value, basestring) + not isinstance(new_value, str) ): if value != new_value: new_value = list(set(new_value + list(value))) diff --git a/dynamic_rest/client/resource.py b/dynamic_rest/client/resource.py index 634d1e3a..17600ca8 100644 --- a/dynamic_rest/client/resource.py +++ b/dynamic_rest/client/resource.py @@ -60,7 +60,7 @@ def load(self, data): else: # plain dict return data - elif isinstance(data, list) and not isinstance(data, basestring): + elif isinstance(data, list) and not isinstance(data, str): for i, value in enumerate(data): # load from list _value = self.load(value) diff --git a/tests/test_client.py b/tests/test_client.py index a65f4f97..08792087 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -18,7 +18,7 @@ def make_params(params): list_params = [] for key, value in params.items(): if isinstance( - value, basestring + value, str ) or not isinstance(value, list): value = [value] for v in value: From 7dc057adc0d816df371bf46d5162465dcb6e701b Mon Sep 17 00:00:00 2001 From: aleontiev Date: Tue, 24 Jan 2017 17:07:47 -0800 Subject: [PATCH 26/35] py3.5 compat --- dynamic_rest/client/client.py | 2 +- dynamic_rest/client/query.py | 3 ++- dynamic_rest/client/record.py | 21 +++++++++++---------- dynamic_rest/client/resource.py | 6 +++--- tests/test_client.py | 21 ++++++++++++++++----- 5 files changed, 33 insertions(+), 20 deletions(-) diff --git a/dynamic_rest/client/client.py b/dynamic_rest/client/client.py index 4e6dd0b4..e911eb18 100644 --- a/dynamic_rest/client/client.py +++ b/dynamic_rest/client/client.py @@ -184,4 +184,4 @@ def request(self, method, url, params=None, data=None): if response.status_code >= 400: raise BadRequest() - return json.loads(response.content) + return json.loads(response.content.decode('utf-8')) diff --git a/dynamic_rest/client/query.py b/dynamic_rest/client/query.py index 90dbfb64..3c4f4971 100644 --- a/dynamic_rest/client/query.py +++ b/dynamic_rest/client/query.py @@ -1,5 +1,6 @@ from copy import copy from .utils import unpack +from six import string_types class DRESTQuery(object): @@ -126,7 +127,7 @@ def _copy(self, **kwargs): new_value.update(value) elif ( isinstance(new_value, (list, tuple)) and - not isinstance(new_value, str) + not isinstance(new_value, string_types) ): if value != new_value: new_value = list(set(new_value + list(value))) diff --git a/dynamic_rest/client/record.py b/dynamic_rest/client/record.py index 7c6b8bd8..82d5efbd 100644 --- a/dynamic_rest/client/record.py +++ b/dynamic_rest/client/record.py @@ -8,7 +8,9 @@ def __init__(self, resource, **data): self._load(data) def __eq__(self, other): - return self._data == other._data + if hasattr(other, '_data'): + other = other._data + return self._data == other @property def _data(self): @@ -19,16 +21,14 @@ def _data(self): @property def _diff(self): - data = self._data - for key, value in data.items(): - if key in self._clean and value == self._clean[key]: - data.pop(key) - return data + diff = {} + for key, value in self._data.items(): + if key not in self._clean or value != self._clean[key]: + diff[key] = value + return diff def _load(self, data): for key, value in data.items(): - if key.startswith('_'): - data.pop(key) setattr(self, key, value) self._clean = self._data @@ -50,10 +50,11 @@ def _serialize(self, data): def save(self): id = self.id - data = self._serialize(self._diff if id else self._data) + new = not id + data = self._data if new else self._serialize(self._diff) if data: response = self._resource.request( - 'patch' if id else 'post', + 'post' if new else 'patch', id=id, data=data ) diff --git a/dynamic_rest/client/resource.py b/dynamic_rest/client/resource.py index 17600ca8..8d527c56 100644 --- a/dynamic_rest/client/resource.py +++ b/dynamic_rest/client/resource.py @@ -50,9 +50,9 @@ def load(self, data): data['id'] = pk if name == self.name: for key, value in data.items(): - _value = self.load(value) - if value != _value: - data[key] = _value + loaded = self.load(value) + if value != loaded: + data[key] = loaded return DRESTRecord(resource=self, **data) else: resource = getattr(self._client, name) diff --git a/tests/test_client.py b/tests/test_client.py index 08792087..f7349032 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,14 +1,22 @@ from rest_framework.test import APITestCase, APIClient from dynamic_rest.client import DRESTClient +from six import string_types from dynamic_rest.client.exceptions import ( BadRequest, DoesNotExist ) from tests.setup import create_fixture import urllib +try: + urlencode = urllib.urlencode +except: + # Py3 + urlencode = urllib.parse.urlencode class MockSession(object): + """requests.session compatiability adapter for DRESTClient.""" + def __init__(self, client): self._client = client or APIClient() self.headers = {} @@ -18,18 +26,20 @@ def make_params(params): list_params = [] for key, value in params.items(): if isinstance( - value, str + value, string_types ) or not isinstance(value, list): value = [value] for v in value: list_params.append((key, v)) - return urllib.urlencode(list_params) + return urlencode(list_params) url = '%s%s' % ( url, ('?%s' % make_params(params)) if params else '' ) response = getattr(self._client, method)(url, data=data) + content = response.content.decode('utf-8') + response.content = content return response @@ -132,9 +142,10 @@ def test_extra_pagination(self): self.assertTrue(users_named[0].name, name) def test_save_deferred(self): - user = self.drest.Users.excluding('*').list()[0] + user = self.drest.Users.excluding('*').first() user.name = 'foo' user.save() - user2 = self.drest.Users.first() - self.assertEquals(user2.name, user.name) + user2 = self.drest.Users.filter(name='foo').first() + self.assertIsNotNone(user2) + self.assertEquals(user2.id, user.id) From 344382ccbe9929996c7f3b5ea10dc5a51e8e87aa Mon Sep 17 00:00:00 2001 From: aleontiev Date: Tue, 24 Jan 2017 17:14:10 -0800 Subject: [PATCH 27/35] alias sort/order_by and add test --- dynamic_rest/client/client.py | 2 +- dynamic_rest/client/query.py | 5 ++++- tests/test_client.py | 7 +++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/dynamic_rest/client/client.py b/dynamic_rest/client/client.py index e911eb18..f6f16379 100644 --- a/dynamic_rest/client/client.py +++ b/dynamic_rest/client/client.py @@ -62,7 +62,7 @@ class DRESTClient(object): Ordering results: - users = Users.order_by('-name') + users = Users.sort('-name') Updating records: diff --git a/dynamic_rest/client/query.py b/dynamic_rest/client/query.py index 3c4f4971..62fae741 100644 --- a/dynamic_rest/client/query.py +++ b/dynamic_rest/client/query.py @@ -74,9 +74,12 @@ def excluding(self, *args): def extra(self, **kwargs): return self._copy(extras=kwargs) - def order_by(self, *args): + def sort(self, *args): return self._copy(orders=args) + def order_by(self, *args): + return self.sort(*args) + def _get_params(self): filters = self.filters includes = self.includes diff --git a/tests/test_client.py b/tests/test_client.py index f7349032..1af262e2 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -149,3 +149,10 @@ def test_save_deferred(self): user2 = self.drest.Users.filter(name='foo').first() self.assertIsNotNone(user2) self.assertEquals(user2.id, user.id) + + def test_sort(self): + users = self.drest.Users.sort('name').list() + self.assertEquals( + users, + list(sorted(users, key=lambda x: x.name)) + ) From 29dfbabc2214505cd22875f64e80eaf425b23583 Mon Sep 17 00:00:00 2001 From: aleontiev Date: Tue, 24 Jan 2017 17:18:12 -0800 Subject: [PATCH 28/35] better doc --- dynamic_rest/client/client.py | 38 +++++++++++++++++------------------ 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/dynamic_rest/client/client.py b/dynamic_rest/client/client.py index f6f16379..38347061 100644 --- a/dynamic_rest/client/client.py +++ b/dynamic_rest/client/client.py @@ -27,52 +27,50 @@ class DRESTClient(object): Examples: + Assume there is a DREST resource at my.api.io/v0/users. + Getting a client: - client = DRESTClient('my.api.io', authentication={'token': 'secret'}) + client = DRESTClient('my.api.io', version='v0', authentication={'token': 'secret'}) - Working with a resource: + Getting a single record of the Users resource - Users = client.Users + client.Users.get('123') - Getting a single resource: + Getting all records (automatic pagination): - Users.get('123') + client.Users.all() - Getting all resources (auto-paging): + Filtering records: - Users.all() + client.Users.filter(name__icontains='john') + other_users = client.Users.exclude(name__icontains='john') - Getting filtered resources: + Ordering records: - Users.filter(name__icontains='john') - other_users = client.users.exclude(name__icontains='john') + users = client.Users.sort('-name') Including / excluding fields: - users = Users.all() + users = client.Users.all() .excluding('birthday') .including('events.*') .get('123') Mapping by field: - users_by_id = Users.map() - users_by_name = Users.map('name') - - Ordering results: - - users = Users.sort('-name') + users_by_id = client.Users.map() + users_by_name = client.Users.map('name') Updating records: - user = Users.first() + user = client.Users.first() user.name = 'john' user.save() - Creating resources: + Creating records: - user = Users.create(name='john') + user = client.Users.create(name='john') """ def __init__( self, From a2fd714473e20ad9ee1f1d267549aa6191fdcc23 Mon Sep 17 00:00:00 2001 From: aleontiev Date: Tue, 24 Jan 2017 17:18:47 -0800 Subject: [PATCH 29/35] add todo about query getitem --- dynamic_rest/client/query.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dynamic_rest/client/query.py b/dynamic_rest/client/query.py index 62fae741..1e7d2048 100644 --- a/dynamic_rest/client/query.py +++ b/dynamic_rest/client/query.py @@ -157,6 +157,7 @@ def _load(self, data): return self.resource.load(unpack(data)) def __iter__(self): + # TODO: implement __getitem__ for random access params = self._get_params() self._get_page(params) while True: From 39ed3f6767d45ebaadb409724fbba3e4c4043ae1 Mon Sep 17 00:00:00 2001 From: aleontiev Date: Tue, 24 Jan 2017 17:31:22 -0800 Subject: [PATCH 30/35] optimize diff/data --- dynamic_rest/client/client.py | 9 +++- dynamic_rest/client/record.py | 82 +++++++++++++++++++---------------- 2 files changed, 52 insertions(+), 39 deletions(-) diff --git a/dynamic_rest/client/client.py b/dynamic_rest/client/client.py index 38347061..38d6c9b9 100644 --- a/dynamic_rest/client/client.py +++ b/dynamic_rest/client/client.py @@ -27,11 +27,16 @@ class DRESTClient(object): Examples: - Assume there is a DREST resource at my.api.io/v0/users. + Assume there is a DREST resource at "https://my.api.io/v0/users", + and that we can access this resource with an auth token "secret". Getting a client: - client = DRESTClient('my.api.io', version='v0', authentication={'token': 'secret'}) + client = DRESTClient( + 'my.api.io', + version='v0', + authentication={'token': 'secret'} + ) Getting a single record of the Users resource diff --git a/dynamic_rest/client/record.py b/dynamic_rest/client/record.py index 82d5efbd..efc855f5 100644 --- a/dynamic_rest/client/record.py +++ b/dynamic_rest/client/record.py @@ -3,40 +3,68 @@ class DRESTRecord(object): + def __init__(self, resource, **data): self._resource = resource self._load(data) + def save(self): + id = self.id + new = not id + data = ( + self._get_data() if new + else self._serialize(self._get_diff()) + ) + if data: + response = self._resource.request( + 'post' if new else 'patch', + id=id, + data=data + ) + self._load(unpack(response)) + + def reload(self): + id = self.id + if id: + response = self._resource.request('get', id=id) + self._load(unpack(response)) + else: + raise DoesNotExist() + def __eq__(self, other): - if hasattr(other, '_data'): - other = other._data - return self._data == other + if hasattr(other, '_get_data'): + other = other._get_data() + return self._get_data() == other + + def __repr__(self): + return '%s.%s' % (self._resource.name, self.id if self.id else '') + + def _get_data(self, fn=None): + if fn is None: + flt = lambda k, v: not k.startswith('_') + else: + flt = lambda k, v: fn(k, v) - @property - def _data(self): return { k: v for k, v in self.__dict__.items() - if not k.startswith('_') + if flt(k, v) } - @property - def _diff(self): - diff = {} - for key, value in self._data.items(): - if key not in self._clean or value != self._clean[key]: - diff[key] = value - return diff + def _get_diff(self): + return self._get_data( + lambda k, v: ( + not k.startswith('_') and + (k not in self._clean or v != self._clean[k]) + ) + ) def _load(self, data): for key, value in data.items(): setattr(self, key, value) - self._clean = self._data + self._clean = self._get_data() self.id = data.get('_meta', {}).get('id', data.get('id', None)) - def __repr__(self): - return '%s.%s' % (self._resource.name, self.id if self.id else '') - def _serialize(self, data): for key, values in data.items(): if isinstance(values, list): @@ -47,23 +75,3 @@ def _serialize(self, data): elif isinstance(values, DRESTRecord): data[key] = values.id return data - - def save(self): - id = self.id - new = not id - data = self._data if new else self._serialize(self._diff) - if data: - response = self._resource.request( - 'post' if new else 'patch', - id=id, - data=data - ) - self._load(unpack(response)) - - def reload(self): - id = self.id - if id: - response = self._resource.request('get', id=id) - self._load(unpack(response)) - else: - raise DoesNotExist() From 439601c1421db28c03eaba3b7bf2bff0ee941699 Mon Sep 17 00:00:00 2001 From: aleontiev Date: Tue, 24 Jan 2017 18:55:56 -0800 Subject: [PATCH 31/35] change to date of birth field --- tests/test_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_client.py b/tests/test_client.py index 1af262e2..0d537373 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -123,7 +123,7 @@ def test_invalid_resource(self): def test_save_invalid_data(self): user = self.drest.Users.first() - user.name = '' + user.date_of_birth = 'foo' with self.assertRaises(BadRequest): user.save() From eb2909e3ca611d7b155d34316ee7c6bc0b7246c1 Mon Sep 17 00:00:00 2001 From: aleontiev Date: Thu, 26 Jan 2017 13:18:37 -0800 Subject: [PATCH 32/35] .. --- dynamic_rest/filters.py | 13 +++---------- dynamic_rest/serializers.py | 6 ++++++ dynamic_rest/utils.py | 13 +++++++++++++ dynamic_rest/viewsets.py | 17 +++-------------- 4 files changed, 25 insertions(+), 24 deletions(-) create mode 100644 dynamic_rest/utils.py diff --git a/dynamic_rest/filters.py b/dynamic_rest/filters.py index 7e4eda51..439f8b2d 100644 --- a/dynamic_rest/filters.py +++ b/dynamic_rest/filters.py @@ -9,6 +9,7 @@ from rest_framework.fields import BooleanField, NullBooleanField from rest_framework.filters import BaseFilterBackend, OrderingFilter +from dynamic_rest.utils import is_truthy from dynamic_rest.conf import settings from dynamic_rest.datastructures import TreeMap from dynamic_rest.fields import DynamicRelationField @@ -147,8 +148,6 @@ class DynamicFilterBackend(BaseFilterBackend): Attributes: VALID_FILTER_OPERATORS: A list of filter operators. - FALSEY_STRINGS: A list of strings that are interpretted as - False by the isnull operator. """ VALID_FILTER_OPERATORS = ( @@ -176,12 +175,6 @@ class DynamicFilterBackend(BaseFilterBackend): None, ) - FALSEY_STRINGS = ( - '0', - 'false', - '', - ) - def filter_queryset(self, request, queryset, view): """Filter the queryset. @@ -255,7 +248,7 @@ def _get_requested_filters(self, **kwargs): operator == 'isnull' and isinstance(value, six.string_types) ): - value = value.lower() not in self.FALSEY_STRINGS + value = is_truthy(value) elif operator == 'eq': operator = None @@ -294,7 +287,7 @@ def rewrite_filters(filters, serializer): for k, node in six.iteritems(filters): filter_key, field = node.generate_query_key(serializer) if isinstance(field, (BooleanField, NullBooleanField)): - node.value = node.value.lower() not in self.FALSEY_STRINGS + node.value = is_truthy(node.value) out[filter_key] = node.value return out diff --git a/dynamic_rest/serializers.py b/dynamic_rest/serializers.py index c3b851d2..c5d423ee 100644 --- a/dynamic_rest/serializers.py +++ b/dynamic_rest/serializers.py @@ -199,6 +199,12 @@ def __init__( kwargs['instance'] = instance kwargs['data'] = data + + # "sideload" argument is now deprecated + # pop it instead of breaking for backwards compatibility + # TODO: remove in DREST 2 + kwargs.pop('sideload', None) + super(WithDynamicSerializerMixin, self).__init__(**kwargs) self.sideloading = sideloading diff --git a/dynamic_rest/utils.py b/dynamic_rest/utils.py new file mode 100644 index 00000000..1119f6a5 --- /dev/null +++ b/dynamic_rest/utils.py @@ -0,0 +1,13 @@ +from six import string_types + +FALSEY_STRINGS = ( + '0', + 'false', + '', +) + + +def is_truthy(x): + if isinstance(x, string_types): + return x.lower() not in FALSEY_STRINGS + return bool(x) diff --git a/dynamic_rest/viewsets.py b/dynamic_rest/viewsets.py index 676b26ce..02ad90ac 100644 --- a/dynamic_rest/viewsets.py +++ b/dynamic_rest/viewsets.py @@ -13,6 +13,7 @@ from dynamic_rest.pagination import DynamicPageNumberPagination from dynamic_rest.processors import SideloadingProcessor from dynamic_rest.renderers import DynamicBrowsableAPIRenderer +from dynamic_rest.utils import is_truthy UPDATE_REQUEST_METHODS = ('PUT', 'PATCH', 'POST') DELETE_REQUEST_METHOD = 'DELETE' @@ -234,23 +235,11 @@ def get_request_fields(self): def get_request_debug(self): debug = self.get_request_feature(self.DEBUG) - if debug: - debug = debug .lower() - if debug == 'true': - debug = True - else: - debug = False - return debug + return is_truthy(debug) if debug is not None else None def get_request_sideloading(self): sideloading = self.get_request_feature(self.SIDELOADING) - if sideloading: - sideloading = sideloading.lower() - if sideloading == 'true': - sideloading = True - else: - sideloading = False - return sideloading + return is_truthy(sideloading) if sideloading is not None else None def is_update(self): if ( From 86794f889f876e69b6c77764a6dd49fd00aa23eb Mon Sep 17 00:00:00 2001 From: aleontiev Date: Mon, 30 Jan 2017 14:37:42 -0800 Subject: [PATCH 33/35] bump version, serializer: add "envelope" parameter and support "sideload" parameter for BWC --- dynamic_rest/client/query.py | 2 +- dynamic_rest/client/record.py | 2 +- dynamic_rest/client/utils.py | 3 -- dynamic_rest/constants.py | 2 +- dynamic_rest/fields/fields.py | 3 ++ dynamic_rest/serializers.py | 41 ++++++++++++++----------- dynamic_rest/utils.py | 6 ++++ dynamic_rest/viewsets.py | 36 ++++++++++++++-------- tests/test_generic.py | 2 +- tests/test_serializers.py | 56 ++++++++++++++++++++++++++++------- 10 files changed, 104 insertions(+), 49 deletions(-) delete mode 100644 dynamic_rest/client/utils.py diff --git a/dynamic_rest/client/query.py b/dynamic_rest/client/query.py index 1e7d2048..ddb8abf4 100644 --- a/dynamic_rest/client/query.py +++ b/dynamic_rest/client/query.py @@ -1,5 +1,5 @@ from copy import copy -from .utils import unpack +from dynamic_rest.utils import unpack from six import string_types diff --git a/dynamic_rest/client/record.py b/dynamic_rest/client/record.py index efc855f5..1dc036ef 100644 --- a/dynamic_rest/client/record.py +++ b/dynamic_rest/client/record.py @@ -1,4 +1,4 @@ -from .utils import unpack +from dynamic_rest.utils import unpack from .exceptions import DoesNotExist diff --git a/dynamic_rest/client/utils.py b/dynamic_rest/client/utils.py deleted file mode 100644 index 95a84cf6..00000000 --- a/dynamic_rest/client/utils.py +++ /dev/null @@ -1,3 +0,0 @@ -def unpack(content): - keys = [k for k in content.keys() if k != 'meta'] - return content[keys[0]] diff --git a/dynamic_rest/constants.py b/dynamic_rest/constants.py index 7de6c17b..91f53bdc 100644 --- a/dynamic_rest/constants.py +++ b/dynamic_rest/constants.py @@ -5,4 +5,4 @@ REPO_NAME = "dynamic-rest" PROJECT_NAME = "Dynamic REST" ORG_NAME = "AltSchool" -VERSION = "1.5.4" +VERSION = "1.6.0" diff --git a/dynamic_rest/fields/fields.py b/dynamic_rest/fields/fields.py index a1e64e2b..edd8066f 100644 --- a/dynamic_rest/fields/fields.py +++ b/dynamic_rest/fields/fields.py @@ -93,6 +93,9 @@ def __init__( many: Boolean, if relation is to-many. queryset: Default queryset to apply when filtering for related objects. + sideloading: if True, force sideloading all the way down. + if False, force embedding all the way down. + This overrides the "embed" option if set. embed: If True, always embed related object(s). Will not sideload, and will include the full object unless specifically excluded. """ diff --git a/dynamic_rest/serializers.py b/dynamic_rest/serializers.py index c5d423ee..21e8c907 100644 --- a/dynamic_rest/serializers.py +++ b/dynamic_rest/serializers.py @@ -8,8 +8,9 @@ from django.utils.functional import cached_property from rest_framework import exceptions, fields, serializers from rest_framework.fields import SkipField -from rest_framework.utils.serializer_helpers import ReturnDict +from rest_framework.utils.serializer_helpers import ReturnDict, ReturnList +from dynamic_rest.utils import unpack from dynamic_rest.bases import DynamicSerializerBase from dynamic_rest.conf import settings from dynamic_rest.fields import DynamicRelationField @@ -68,11 +69,12 @@ def data(self): """Get the data, after performing post-processing if necessary.""" if not hasattr(self, '_processed_data'): data = super(DynamicListSerializer, self).data + data = SideloadingProcessor(self, data).data self._processed_data = ReturnDict( - SideloadingProcessor( - self, - data - ).data, + data, + serializer=self + ) if self.child.envelope else ReturnList( + unpack(data), serializer=self ) return self._processed_data @@ -88,7 +90,7 @@ def update(self, queryset, validated_data): lookup_keys = lookup_objects.keys() if not all((bool(_) and not inspect.isclass(_) for _ in lookup_keys)): - raise exceptions.ValidationError('Invalid lookup key value') + raise exceptions.ValidationError('Invalid lookup key value.') # Since this method is given a queryset which can have many # model instances, first find all objects to update @@ -99,7 +101,7 @@ def update(self, queryset, validated_data): if len(lookup_keys) != objects_to_update.count(): raise exceptions.ValidationError( - 'Could not find all objects to update: {} != {}' + 'Could not find all objects to update: {} != {}.' .format(len(lookup_keys), objects_to_update.count()) ) @@ -161,6 +163,7 @@ def __init__( debug=False, dynamic=True, embed=False, + envelope=False, **kwargs ): """ @@ -179,6 +182,8 @@ def __init__( If None (default), respect individual embed parameters dynamic: If False, ignore deferred rules and revert to standard DRF `.fields` behavior. + envelope: if True, wrap `.data` in envelope. + if False, do not use an envelope. """ name = self.get_name() if data is not fields.empty and name in data and len(data) == 1: @@ -200,13 +205,14 @@ def __init__( kwargs['instance'] = instance kwargs['data'] = data - # "sideload" argument is now deprecated - # pop it instead of breaking for backwards compatibility - # TODO: remove in DREST 2 - kwargs.pop('sideload', None) + # "sideload" argument is pending deprecation as of 1.6 + if kwargs.pop('sideload', False): + # if "sideload=True" is passed, turn on the envelope + envelope = True super(WithDynamicSerializerMixin, self).__init__(**kwargs) + self.envelope = envelope self.sideloading = sideloading self.debug = debug self.dynamic = dynamic @@ -396,8 +402,9 @@ def get_fields(self): for name, include in six.iteritems(request_fields): if name not in serializer_fields: raise exceptions.ParseError( - "'%s' is not a valid field name for '%s'" % - (name, self.get_name())) + '"%s" is not a valid field name for "%s".' % + (name, self.get_name()) + ) if include is not False and name in deferred: deferred.remove(name) elif include is False: @@ -592,11 +599,9 @@ def id_only(self): def data(self): if not hasattr(self, '_processed_data'): data = super(WithDynamicSerializerMixin, self).data + data = SideloadingProcessor(self, data).data self._processed_data = ReturnDict( - SideloadingProcessor( - self, - data - ).data, + data if self.envelope else unpack(data), serializer=self ) return self._processed_data @@ -652,7 +657,7 @@ class EphemeralObject(object): def __init__(self, values_dict): if 'pk' not in values_dict: - raise Exception("'pk' key is required") + raise Exception('"pk" key is required') self.__dict__.update(values_dict) diff --git a/dynamic_rest/utils.py b/dynamic_rest/utils.py index 1119f6a5..62fdce32 100644 --- a/dynamic_rest/utils.py +++ b/dynamic_rest/utils.py @@ -11,3 +11,9 @@ def is_truthy(x): if isinstance(x, string_types): return x.lower() not in FALSEY_STRINGS return bool(x) + + +def unpack(content): + keys = [k for k in content.keys() if k != 'meta'] + unpacked = content[keys[0]] + return unpacked diff --git a/dynamic_rest/viewsets.py b/dynamic_rest/viewsets.py index 02ad90ac..3a85f097 100644 --- a/dynamic_rest/viewsets.py +++ b/dynamic_rest/viewsets.py @@ -172,7 +172,7 @@ def _extract_object_params(self, name): # malformed argument like: # filter{foo=bar raise exceptions.ParseError( - "'%s' is not a well-formed filter key" % name + '"%s" is not a well-formed filter key.' % name ) else: continue @@ -227,8 +227,9 @@ def get_request_fields(self): elif not last: # empty segment must be the last segment raise exceptions.ParseError( - "'%s' is not a valid field" % - field) + '"%s" is not a valid field.' % + field + ) self._request_fields = request_fields return request_fields @@ -266,22 +267,30 @@ def get_serializer(self, *args, **kwargs): kwargs['sideloading'] = self.get_request_sideloading() if 'debug' not in kwargs: kwargs['debug'] = self.get_request_debug() + if 'envelope' not in kwargs: + kwargs['envelope'] = True if self.is_update(): kwargs['include_fields'] = '*' return super( - WithDynamicViewSetMixin, self).get_serializer( - *args, **kwargs) + WithDynamicViewSetMixin, self + ).get_serializer( + *args, **kwargs + ) def paginate_queryset(self, *args, **kwargs): if self.PAGE in self.features: # make sure pagination is enabled - if self.PER_PAGE not in self.features and \ - self.PER_PAGE in self.request.query_params: + if ( + self.PER_PAGE not in self.features and + self.PER_PAGE in self.request.query_params + ): # remove per_page if it is disabled self.request.query_params[self.PER_PAGE] = None return super( - WithDynamicViewSetMixin, self).paginate_queryset( - *args, **kwargs) + WithDynamicViewSetMixin, self + ).paginate_queryset( + *args, **kwargs + ) return None def _prefix_inex_params(self, request, feature, prefix): @@ -312,7 +321,7 @@ def list_related(self, request, pk=None, field_name=None): # can have unintended consequences when applied asynchronously. if self.get_request_feature(self.FILTER): raise ValidationError( - "Filtering is not enabled on relation endpoints." + 'Filtering is not enabled on relation endpoints.' ) # Prefix include/exclude filters with field_name so it's scoped to @@ -329,7 +338,7 @@ def list_related(self, request, pk=None, field_name=None): serializer = self.get_serializer() field = serializer.fields.get(field_name) if field is None: - raise ValidationError("Unknown field: %s" % field_name) + raise ValidationError('Unknown field: "%s".' % field_name) # Query for root object, with related field prefetched queryset = self.get_queryset() @@ -341,7 +350,7 @@ def list_related(self, request, pk=None, field_name=None): # Serialize the related data. Use the field's serializer to ensure # it's configured identically to the sideload case. - serializer = field.serializer + serializer = field.get_serializer(envelope=True) try: # TODO(ryo): Probably should use field.get_attribute() but that # seems to break a bunch of things. Investigate later. @@ -535,4 +544,5 @@ def destroy(self, request, *args, **kwargs): # assume that it is a poorly formatted bulk request return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) return super(DynamicModelViewSet, self).destroy( - request, *args, **kwargs) + request, *args, **kwargs + ) diff --git a/tests/test_generic.py b/tests/test_generic.py index eac47a75..b77d2d7d 100644 --- a/tests/test_generic.py +++ b/tests/test_generic.py @@ -171,7 +171,7 @@ class Meta: 'favorite_pet' ).first() - data = FooUserSerializer(user).data['user'] + data = FooUserSerializer(user, envelope=True).data['user'] self.assertIsNotNone(data) self.assertTrue('favorite_pet' in data) self.assertTrue(isinstance(data['favorite_pet'], dict)) diff --git a/tests/test_serializers.py b/tests/test_serializers.py index 6fc8f39e..90cdbcf1 100644 --- a/tests/test_serializers.py +++ b/tests/test_serializers.py @@ -36,11 +36,28 @@ def setUp(self): self.fixture = create_fixture() self.maxDiff = None - def test_data(self): + def test_data_without_envelope(self): serializer = UserSerializer( self.fixture.users, many=True, ) + self.assertEqual(serializer.data, [ + OrderedDict( + [('id', 1), ('name', '0'), ('location', 1)]), + OrderedDict( + [('id', 2), ('name', '1'), ('location', 1)]), + OrderedDict( + [('id', 3), ('name', '2'), ('location', 2)]), + OrderedDict( + [('id', 4), ('name', '3'), ('location', 3)]) + ]) + + def test_data_with_envelope(self): + serializer = UserSerializer( + self.fixture.users, + many=True, + envelope=True + ) self.assertEqual(serializer.data, { 'users': [ OrderedDict( @@ -61,6 +78,7 @@ def test_data_with_included_field(self): serializer = UserSerializer( self.fixture.users, many=True, + sideload=True, # pending deprecation 1.6 request_fields=request_fields, ) self.assertEqual(serializer.data, { @@ -87,6 +105,7 @@ def test_data_with_excluded_field(self): serializer = UserSerializer( self.fixture.users, many=True, + envelope=True, request_fields=request_fields, ) self.assertEqual(serializer.data, { @@ -109,6 +128,7 @@ def test_data_with_included_has_one(self): serializer = UserSerializer( self.fixture.users, many=True, + envelope=True, request_fields=request_fields, ) self.assertEqual(serializer.data, { @@ -143,6 +163,7 @@ def test_data_with_included_has_one(self): serializer = UserSerializer( self.fixture.users[0], + envelope=True, request_fields=request_fields, ) self.assertEqual(serializer.data, { @@ -214,6 +235,7 @@ def test_data_with_included_has_many(self): serializer = UserSerializer( self.fixture.users, many=True, + envelope=True, request_fields=request_fields, ) self.assertEqual(serializer.data, expected) @@ -271,6 +293,7 @@ def test_data_with_included_has_many(self): serializer = GroupSerializer( self.fixture.groups, many=True, + envelope=True, request_fields=request_fields, ) self.assertEqual(serializer.data, expected) @@ -285,6 +308,7 @@ def test_data_with_nested_include(self): serializer = UserSerializer( self.fixture.users, many=True, + envelope=True, request_fields=request_fields, ) expected = { @@ -354,6 +378,7 @@ def test_data_with_nested_exclude(self): serializer = UserSerializer( self.fixture.users, many=True, + envelope=True, request_fields=request_fields, ) self.assertEqual(serializer.data, { @@ -506,21 +531,23 @@ def test_data(self): data['location'] = location data['groups'] = self.fixture.groups instance = EphemeralObject(data) - data = LocationGroupSerializer(instance).data['locationgroup'] + data = LocationGroupSerializer( + instance, envelope=True + ).data['locationgroup'] self.assertEqual( data, {'id': 1, 'groups': [1, 2], 'location': 1} ) def test_data_count_field(self): eo = EphemeralObject({'pk': 1, 'values': [1, 1, 2]}) - data = CountsSerializer(eo).data['counts'] + data = CountsSerializer(eo, envelope=True).data['counts'] self.assertEqual(data['count'], 3) self.assertEqual(data['unique_count'], 2) def test_data_count_field_returns_none_if_null_values(self): eo = EphemeralObject({'pk': 1, 'values': None}) - data = CountsSerializer(eo).data['counts'] + data = CountsSerializer(eo, envelope=True).data['counts'] self.assertEqual(data['count'], None) self.assertEqual(data['unique_count'], None) @@ -528,7 +555,7 @@ def test_data_count_field_returns_none_if_null_values(self): def test_data_count_raises_exception_if_wrong_type(self): eo = EphemeralObject({'pk': 1, 'values': {}}) with self.assertRaises(TypeError): - CountsSerializer(eo).data + CountsSerializer(eo, envelope=True).data def test_to_representation_if_id_only(self): ''' Test EphemeralSerializer.to_representation() in id_only mode ''' @@ -557,7 +584,8 @@ def setUp(self): def test_data_with_embed(self): data = UserLocationSerializer( - self.fixture.users[0] + self.fixture.users[0], + envelope=True ).data self.assertEqual(data['user_location']['location']['name'], '0') self.assertEqual( @@ -573,10 +601,13 @@ class Meta: model = User name = 'user_deferred_location' location = DynamicRelationField( - LocationSerializer, embed=True, deferred=True) + LocationSerializer, embed=True, deferred=True + ) data = UserDeferredLocationSerializer( - self.fixture.users[0]).data + self.fixture.users[0], + envelope=True + ).data self.assertFalse('location' in data) # Now include deferred embedded field @@ -586,7 +617,8 @@ class Meta: 'id': True, 'name': True, 'location': True - } + }, + envelope=True ).data['user_deferred_location'] self.assertTrue('location' in data) self.assertEqual(data['location']['name'], '0') @@ -616,7 +648,8 @@ class Meta: 'id': True, 'name': True, 'groups': True - } + }, + envelope=True ).data['user_deferred_location'] self.assertTrue('groups' in data) @@ -635,7 +668,8 @@ class Meta: groups = DynamicRelationField('GroupSerializer', many=True) data = UserDeferredLocationSerializer( - self.fixture.users[0] + self.fixture.users[0], + envelope=True ).data['user_deferred_location'] self.assertTrue('groups' in data) From b7de56b9890a73b0a8a7664e7c9e88c2450def07 Mon Sep 17 00:00:00 2001 From: aleontiev Date: Mon, 30 Jan 2017 15:49:11 -0800 Subject: [PATCH 34/35] disable sideloading if envelope is explicitly disabled --- dynamic_rest/processors.py | 4 +-- dynamic_rest/serializers.py | 51 ++++++++++++++++++++++--------------- dynamic_rest/utils.py | 4 +++ tests/test_api.py | 2 +- 4 files changed, 37 insertions(+), 24 deletions(-) diff --git a/dynamic_rest/processors.py b/dynamic_rest/processors.py index ef803181..5acf6262 100644 --- a/dynamic_rest/processors.py +++ b/dynamic_rest/processors.py @@ -17,8 +17,6 @@ class SideloadingProcessor(object): typically smaller than their nested equivalent. """ - prefix = settings.ADDITIONAL_PRIMARY_RESOURCE_PREFIX - def __init__(self, serializer, data): """Initializes and runs the processor. @@ -107,7 +105,7 @@ def process(self, obj, parent=None, parent_key=None, depth=0): # if the primary resource is embedded, add it to a prefixed key if name == self.plural_name: name = '%s%s' % ( - self.prefix, + settings.ADDITIONAL_PRIMARY_RESOURCE_PREFIX, name ) diff --git a/dynamic_rest/serializers.py b/dynamic_rest/serializers.py index 21e8c907..8511224e 100644 --- a/dynamic_rest/serializers.py +++ b/dynamic_rest/serializers.py @@ -10,7 +10,6 @@ from rest_framework.fields import SkipField from rest_framework.utils.serializer_helpers import ReturnDict, ReturnList -from dynamic_rest.utils import unpack from dynamic_rest.bases import DynamicSerializerBase from dynamic_rest.conf import settings from dynamic_rest.fields import DynamicRelationField @@ -69,12 +68,11 @@ def data(self): """Get the data, after performing post-processing if necessary.""" if not hasattr(self, '_processed_data'): data = super(DynamicListSerializer, self).data - data = SideloadingProcessor(self, data).data self._processed_data = ReturnDict( - data, + SideloadingProcessor(self, data).data, serializer=self ) if self.child.envelope else ReturnList( - unpack(data), + data, serializer=self ) return self._processed_data @@ -163,27 +161,29 @@ def __init__( debug=False, dynamic=True, embed=False, - envelope=False, + envelope=None, **kwargs ): """ Custom initializer that builds `request_fields`. Arguments: - instance: Instance to be managed by the serializer. + instance: Initial instance, used by updates. + data: Initial data, used by updates / creates. only_fields: List of field names to render. include_fields: List of field names to include. exclude_fields: List of field names to exclude. - request_fields: map of field names that supports - inclusions, exclusions, and nested sideloads. - embed: If True, force the current representation to be embedded. - sideloading: if False, force embedding for all descendants - If True, force sideloading for all descendants - If None (default), respect individual embed parameters - dynamic: If False, ignore deferred rules and - revert to standard DRF `.fields` behavior. - envelope: if True, wrap `.data` in envelope. - if False, do not use an envelope. + request_fields: Map of field names that supports + nested inclusions / exclusions. + embed: If True, embed the current representation. + If False, sideload the current representation. + sideloading: If True, force sideloading for all descendents. + If False, force embedding for all descendents. + If None (default), respect descendents' embed parameters. + dynamic: If False, disable inclusion / exclusion features. + envelope: If True, wrap `.data` in an envelope. + If False, do not use an envelope and disable sideloading. + If None, do not use an envelope. """ name = self.get_name() if data is not fields.empty and name in data and len(data) == 1: @@ -213,11 +213,20 @@ def __init__( super(WithDynamicSerializerMixin, self).__init__(**kwargs) self.envelope = envelope - self.sideloading = sideloading self.debug = debug self.dynamic = dynamic self.request_fields = request_fields or {} - self.embed = embed if sideloading is None else not sideloading + + # sideloading modifies top-level response keys, + # so it requires an envelope + if envelope is False: + sideloading = False + + # `embed` is overriden by `sideloading` + embed = embed if sideloading is None else not sideloading + + self.sideloading = sideloading + self.embed = embed self._dynamic_init(only_fields, include_fields, exclude_fields) self.enable_optimization = settings.ENABLE_SERIALIZER_OPTIMIZATIONS @@ -599,9 +608,11 @@ def id_only(self): def data(self): if not hasattr(self, '_processed_data'): data = super(WithDynamicSerializerMixin, self).data - data = SideloadingProcessor(self, data).data + data = SideloadingProcessor( + self, data + ).data if self.envelope else data self._processed_data = ReturnDict( - data if self.envelope else unpack(data), + data, serializer=self ) return self._processed_data diff --git a/dynamic_rest/utils.py b/dynamic_rest/utils.py index 62fdce32..674e17d6 100644 --- a/dynamic_rest/utils.py +++ b/dynamic_rest/utils.py @@ -14,6 +14,10 @@ def is_truthy(x): def unpack(content): + if not content: + # empty values pass through + return content + keys = [k for k in content.keys() if k != 'meta'] unpacked = content[keys[0]] return unpacked diff --git a/tests/test_api.py b/tests/test_api.py index 621e82a5..976bcb09 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -317,7 +317,7 @@ def test_get_one_with_include(self): response = self.client.get('/users/1/?include[]=groups.') self.assertEquals(200, response.status_code) data = json.loads(response.content.decode('utf-8')) - self.assertEquals(len(data['groups']), 2) + self.assertEquals(len(data.get('groups', [])), 2) def test_get_with_filter(self): with self.assertNumQueries(1): From c3dc7840d7e39e78ad40862f4110b78e43931956 Mon Sep 17 00:00:00 2001 From: aleontiev Date: Mon, 30 Jan 2017 15:57:19 -0800 Subject: [PATCH 35/35] remove envelope -> sideloading dependency --- dynamic_rest/serializers.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/dynamic_rest/serializers.py b/dynamic_rest/serializers.py index 8511224e..089de2d8 100644 --- a/dynamic_rest/serializers.py +++ b/dynamic_rest/serializers.py @@ -161,7 +161,7 @@ def __init__( debug=False, dynamic=True, embed=False, - envelope=None, + envelope=False, **kwargs ): """ @@ -182,8 +182,7 @@ def __init__( If None (default), respect descendents' embed parameters. dynamic: If False, disable inclusion / exclusion features. envelope: If True, wrap `.data` in an envelope. - If False, do not use an envelope and disable sideloading. - If None, do not use an envelope. + If False, do not use an envelope. """ name = self.get_name() if data is not fields.empty and name in data and len(data) == 1: @@ -213,19 +212,13 @@ def __init__( super(WithDynamicSerializerMixin, self).__init__(**kwargs) self.envelope = envelope + self.sideloading = sideloading self.debug = debug self.dynamic = dynamic self.request_fields = request_fields or {} - # sideloading modifies top-level response keys, - # so it requires an envelope - if envelope is False: - sideloading = False - # `embed` is overriden by `sideloading` embed = embed if sideloading is None else not sideloading - - self.sideloading = sideloading self.embed = embed self._dynamic_init(only_fields, include_fields, exclude_fields)