diff --git a/autopush/db.py b/autopush/db.py index 19a66bee..3f010c13 100644 --- a/autopush/db.py +++ b/autopush/db.py @@ -46,12 +46,8 @@ Factory, Attribute) -from boto.dynamodb2.exceptions import ( - ItemNotFound, -) import boto3 import botocore -from boto.dynamodb2.table import Table # noqa from boto3.resources.base import ServiceResource # noqa from boto3.dynamodb.conditions import Key from boto3.exceptions import Boto3Error @@ -77,9 +73,8 @@ import autopush.metrics from autopush import constants -from autopush.exceptions import AutopushException +from autopush.exceptions import AutopushException, ItemNotFound from autopush.metrics import IMetrics # noqa -from autopush.types import ItemLike # noqa from autopush.utils import ( generate_hash, normalize_id, @@ -95,7 +90,7 @@ # Typing T = TypeVar('T') # noqa -TableFunc = Callable[[str, int, int, ServiceResource], Table] +TableFunc = Callable[[str, int, int, ServiceResource], Any] key_hash = "" TRACK_DB_CALLS = False @@ -149,7 +144,7 @@ def create_rotating_message_table( write_throughput=5, # type: int boto_resource=None # type: DynamoDBResource ): - # type: (...) -> Table # noqa + # type: (...) -> Any # noqa """Create a new message table for webpush style message storage""" tablename = make_rotating_tablename(prefix, delta, date) @@ -227,7 +222,7 @@ def get_rotating_message_tablename( def create_router_table(tablename="router", read_throughput=5, write_throughput=5, boto_resource=None): - # type: (str, int, int, DynamoDBResource) -> Table + # type: (str, int, int, DynamoDBResource) -> Any """Create a new router table The last_connect index is a value used to determine the last month a user @@ -332,7 +327,7 @@ def _expiry(ttl): def get_router_table(tablename="router", read_throughput=5, write_throughput=5, boto_resource=None): - # type: (str, int, int, DynamoDBResource) -> Table + # type: (str, int, int, DynamoDBResource) -> Any """Get the main router table object Creates the table if it doesn't already exist, otherwise returns the @@ -401,7 +396,7 @@ def wrapper(self, *args, **kwargs): def has_connected_this_month(item): - # type: (ItemLike) -> bool + # type: (Dict[str, Any]) -> bool """Whether or not a router item has connected this month""" last_connect = item.get("last_connect") if not last_connect: @@ -790,7 +785,7 @@ def get_uaid(self, uaid): @track_provisioned def register_user(self, data): - # type: (ItemLike) -> Tuple[bool, Dict[str, Any]] + # type: (Dict[str, Any]) -> Tuple[bool, Dict[str, Any]] """Register this user If a record exists with a newer ``connected_at``, then the user will diff --git a/autopush/exceptions.py b/autopush/exceptions.py index 53c9094e..e02ba462 100644 --- a/autopush/exceptions.py +++ b/autopush/exceptions.py @@ -64,3 +64,7 @@ class LogCheckError(Exception): class InvalidConfig(Exception): """Error in initialization of AutopushConfig""" + + +class ItemNotFound(Exception): + """Signals missing DynamoDB Item data""" diff --git a/autopush/router/webpush.py b/autopush/router/webpush.py index 93602140..b196c25b 100644 --- a/autopush/router/webpush.py +++ b/autopush/router/webpush.py @@ -10,8 +10,7 @@ from StringIO import StringIO from typing import Any # noqa -from boto.dynamodb2.exceptions import ItemNotFound -from boto.exception import JSONResponseError +from botocore.exceptions import ClientError from twisted.internet.threads import deferToThread from twisted.web.client import FileBodyProducer from twisted.internet.defer import ( @@ -27,7 +26,7 @@ from twisted.logger import Logger from twisted.web._newclient import ResponseFailed -from autopush.exceptions import RouterException +from autopush.exceptions import ItemNotFound, RouterException from autopush.metrics import make_tags from autopush.protocol import IgnoreBody from autopush.router.interface import RouterResponse @@ -101,7 +100,7 @@ def route_notification(self, notification, uaid_data): # - Error (db error): Done, return 503 try: yield self._save_notification(uaid_data, notification) - except JSONResponseError: + except ClientError: raise RouterException("Error saving to database", status_code=503, response_body="Retry Request", @@ -118,7 +117,7 @@ def route_notification(self, notification, uaid_data): # - Error (no client) : Done, return 404 try: uaid_data = yield deferToThread(router.get_uaid, uaid) - except JSONResponseError: + except ClientError: returnValue(self.stored_response(notification)) except ItemNotFound: self.metrics.increment("updates.client.deleted") @@ -240,4 +239,4 @@ def _save_notification(self, uaid_data, notification): ############################################################# def _eat_db_err(self, fail): """errBack for ignoring provisioned throughput errors""" - fail.trap(JSONResponseError) + fail.trap(ClientError) diff --git a/autopush/tests/__init__.py b/autopush/tests/__init__.py index b090f187..df072c2d 100644 --- a/autopush/tests/__init__.py +++ b/autopush/tests/__init__.py @@ -3,7 +3,6 @@ import signal import subprocess -import boto import psutil from twisted.internet import reactor @@ -19,7 +18,7 @@ def setUp(): - for name in ('boto', 'boto3', 'botocore'): + for name in ('boto3', 'botocore'): logging.getLogger(name).setLevel(logging.CRITICAL) global ddb_process, boto_resource cmd = " ".join([ @@ -54,11 +53,6 @@ def tearDown(): os.kill(p.pid, signal.SIGTERM) ddb_process.wait() - # Clear out the boto config that was loaded so the rest of the tests run - # fine - for section in boto.config.sections(): # pragma: nocover - boto.config.remove_section(section) - _multiprocess_shared_ = True diff --git a/autopush/tests/test_db.py b/autopush/tests/test_db.py index bd472515..6dc65cc6 100644 --- a/autopush/tests/test_db.py +++ b/autopush/tests/test_db.py @@ -4,9 +4,6 @@ from datetime import datetime, timedelta from autopush.websocket import ms_time -from boto.dynamodb2.exceptions import ( - ItemNotFound, -) from botocore.exceptions import ClientError from mock import Mock, patch import pytest @@ -27,7 +24,7 @@ DatabaseManager, DynamoDBResource ) -from autopush.exceptions import AutopushException +from autopush.exceptions import AutopushException, ItemNotFound from autopush.metrics import SinkMetrics from autopush.utils import WebPushNotification diff --git a/autopush/tests/test_integration.py b/autopush/tests/test_integration.py index 32c17bd3..374bbf78 100644 --- a/autopush/tests/test_integration.py +++ b/autopush/tests/test_integration.py @@ -39,6 +39,7 @@ has_connected_this_month, Message, ) +from autopush.exceptions import ItemNotFound from autopush.logging import begin_or_register from autopush.main import ConnectionApplication, EndpointApplication from autopush.utils import base64url_encode, normalize_id @@ -454,7 +455,6 @@ def test_webpush_data_delivery_to_connected_client(self): @inlineCallbacks def test_webpush_data_delivery_to_connected_client_uaid_fail(self): - from boto.dynamodb2.exceptions import ItemNotFound client = yield self.quick_register() self.conn.db.router.get_uaid = Mock(side_effect=ItemNotFound) assert client.channels diff --git a/autopush/tests/test_router.py b/autopush/tests/test_router.py index 33b54ee1..dc90a991 100644 --- a/autopush/tests/test_router.py +++ b/autopush/tests/test_router.py @@ -9,6 +9,7 @@ import ssl import pytest +from botocore.exceptions import ClientError from mock import Mock, PropertyMock, patch from twisted.trial import unittest from twisted.internet.error import ConnectionRefusedError @@ -24,7 +25,7 @@ from autopush.db import ( Message ) -from autopush.exceptions import RouterException +from autopush.exceptions import ItemNotFound, RouterException from autopush.metrics import SinkMetrics from autopush.router import ( APNSRouter, @@ -1116,10 +1117,12 @@ def test_amend(self): assert resp == expected def test_route_to_busy_node_save_throws_db_error(self): - from boto.dynamodb2.exceptions import JSONResponseError def throw(): - raise JSONResponseError(500, "Whoops") + raise ClientError( + {'Error': {'Code': 'InternalServerError'}}, + 'mock_store_message' + ) self.agent_mock.request.return_value = response_mock = Mock() response_mock.code = 202 @@ -1139,10 +1142,12 @@ def verify_deliver(fail): return d def test_route_lookup_uaid_fails(self): - from boto.dynamodb2.exceptions import JSONResponseError def throw(): - raise JSONResponseError(500, "Whoops") + raise ClientError( + {'Error': {'Code': 'InternalServerError'}}, + 'mock_get_uaid' + ) self.message_mock.store_message.return_value = True self.db.message_table = Mock(return_value=self.message_mock) @@ -1161,7 +1166,6 @@ def verify_deliver(status): return d def test_route_lookup_uaid_not_found(self): - from boto.dynamodb2.exceptions import ItemNotFound def throw(): raise ItemNotFound() @@ -1198,7 +1202,6 @@ def verify_deliver(status): return d def test_route_and_clear_failure(self): - from boto.dynamodb2.exceptions import JSONResponseError self.agent_mock.request = Mock(side_effect=ConnectionRefusedError) self.message_mock.store_message.return_value = True self.message_mock.all_channels.return_value = (True, [dummy_chid]) @@ -1208,7 +1211,10 @@ def test_route_and_clear_failure(self): self.router_mock.get_uaid.return_value = router_data def throw(): - raise JSONResponseError(500, "Whoops") + raise ClientError( + {'Error': {'Code': 'InternalServerError'}}, + 'mock_clear_node' + ) self.router_mock.clear_node.side_effect = MockAssist([throw]) self.router.message_id = uuid.uuid4().hex diff --git a/autopush/tests/test_web_validation.py b/autopush/tests/test_web_validation.py index 6122eed4..82a9e00b 100644 --- a/autopush/tests/test_web_validation.py +++ b/autopush/tests/test_web_validation.py @@ -4,9 +4,6 @@ from hashlib import sha256 import ecdsa -from boto.dynamodb2.exceptions import ( - ItemNotFound, -) from cryptography.fernet import InvalidToken from cryptography.exceptions import InvalidSignature from jose import jws @@ -17,7 +14,11 @@ from twisted.trial import unittest from autopush.metrics import SinkMetrics -from autopush.exceptions import InvalidRequest, InvalidTokenException +from autopush.exceptions import ( + InvalidRequest, + InvalidTokenException, + ItemNotFound +) from autopush.tests.support import test_db import autopush.utils as utils diff --git a/autopush/tests/test_webpush_server.py b/autopush/tests/test_webpush_server.py index 367b7dfe..543022ea 100644 --- a/autopush/tests/test_webpush_server.py +++ b/autopush/tests/test_webpush_server.py @@ -6,7 +6,6 @@ import attr import factory -from boto.dynamodb2.exceptions import ItemNotFound from mock import Mock from twisted.logger import globalLogPublisher import pytest @@ -19,6 +18,7 @@ ) from autopush.metrics import SinkMetrics from autopush.config import AutopushConfig +from autopush.exceptions import ItemNotFound from autopush.logging import begin_or_register from autopush.tests.support import TestingLogObserver from autopush.utils import WebPushNotification, ns_time diff --git a/autopush/types.py b/autopush/types.py index b6bfc01b..e0d63854 100644 --- a/autopush/types.py +++ b/autopush/types.py @@ -1,14 +1,7 @@ """Common types""" -from boto.dynamodb2.items import Item -from typing import ( - Any, - Dict, - Union -) +from typing import Any, Dict # no mypy reucrsive types yet: # https://github.com/python/mypy/issues/731 JSONDict = Dict[str, Any] - -ItemLike = Union[Item, Dict[str, Any]] diff --git a/autopush/utils.py b/autopush/utils.py index 4e5ca936..21f13a12 100644 --- a/autopush/utils.py +++ b/autopush/utils.py @@ -26,7 +26,6 @@ from autopush.exceptions import (InvalidTokenException, VapidAuthException) from autopush.jwt import repad, VerifyJWT -from autopush.types import ItemLike # noqa from autopush.web.base import AUTH_SCHEMES @@ -469,7 +468,7 @@ def expired(self, at_time=None): @classmethod def from_message_table(cls, uaid, item): - # type: (uuid.UUID, ItemLike) -> WebPushNotification + # type: (uuid.UUID, Dict[str, Any]) -> WebPushNotification """Create a WebPushNotification from a message table item""" key_info = cls.parse_sort_key(item["chidmessageid"]) if key_info["api_ver"] in ["01", "02"]: diff --git a/autopush/web/health.py b/autopush/web/health.py index f0142904..0accff96 100644 --- a/autopush/web/health.py +++ b/autopush/web/health.py @@ -1,7 +1,7 @@ """Health Check HTTP Handler""" import cyclone.web -from boto.dynamodb2.exceptions import InternalServerError +from botocore.exceptions import ClientError from twisted.internet.defer import DeferredList from twisted.internet.threads import deferToThread @@ -61,7 +61,9 @@ def _check_error(self, failure, name): cause = self._health_checks[name] = {"status": "NOT OK"} if failure.check(MissingTableException): cause["error"] = failure.value.message - elif failure.check(InternalServerError): # pragma nocover + elif (failure.check(ClientError) and + failure.value.response["Error"]["Code"] == + "InternalServerError"): # pragma nocover cause["error"] = "Server error" else: cause["error"] = "Internal error" # pragma nocover diff --git a/autopush/web/registration.py b/autopush/web/registration.py index 235a6469..18145c1e 100644 --- a/autopush/web/registration.py +++ b/autopush/web/registration.py @@ -10,7 +10,6 @@ ) import simplejson as json -from boto.dynamodb2.exceptions import ItemNotFound from cryptography.hazmat.primitives import constant_time from marshmallow import ( Schema, @@ -25,7 +24,7 @@ from twisted.internet.threads import deferToThread from autopush.db import generate_last_connect, hasher -from autopush.exceptions import InvalidRequest, RouterException +from autopush.exceptions import InvalidRequest, ItemNotFound, RouterException from autopush.types import JSONDict # noqa from autopush.utils import generate_hash, ms_time from autopush.web.base import ( diff --git a/autopush/web/webpush.py b/autopush/web/webpush.py index 954198b8..f211aef0 100644 --- a/autopush/web/webpush.py +++ b/autopush/web/webpush.py @@ -1,7 +1,6 @@ import re import time -from boto.dynamodb2.exceptions import ItemNotFound from cryptography.fernet import InvalidToken from cryptography.exceptions import InvalidSignature from marshmallow import ( @@ -32,6 +31,7 @@ from autopush.exceptions import ( InvalidRequest, InvalidTokenException, + ItemNotFound, VapidAuthException, ) from autopush.types import JSONDict # noqa diff --git a/autopush/webpush_server.py b/autopush/webpush_server.py index d9c8a178..dcfd8949 100644 --- a/autopush/webpush_server.py +++ b/autopush/webpush_server.py @@ -9,7 +9,6 @@ attrs, attrib, ) -from boto.dynamodb2.exceptions import ItemNotFound from botocore.exceptions import ClientError from typing import ( # noqa Dict, @@ -29,6 +28,7 @@ ) from autopush.config import AutopushConfig # noqa +from autopush.exceptions import ItemNotFound from autopush.metrics import IMetrics # noqa from autopush.web.webpush import MAX_TTL from autopush.types import JSONDict # noqa diff --git a/autopush/websocket.py b/autopush/websocket.py index 5ab845c7..28dc2b5c 100644 --- a/autopush/websocket.py +++ b/autopush/websocket.py @@ -48,9 +48,6 @@ WebSocketServerProtocol ) from autobahn.websocket.protocol import ConnectionRequest # noqa -from boto.dynamodb2.exceptions import ( - ItemNotFound -) from botocore.exceptions import ClientError from botocore.vendored.requests.packages import urllib3 from twisted.internet import reactor @@ -91,7 +88,7 @@ generate_last_connect, ) from autopush.db import DatabaseManager, Message # noqa -from autopush.exceptions import MessageOverloadException +from autopush.exceptions import MessageOverloadException, ItemNotFound from autopush.noseplugin import track_object from autopush.protocol import IgnoreBody from autopush.metrics import IMetrics, make_tags # noqa diff --git a/requirements.in b/requirements.in index da2d8bed..8f3fd433 100644 --- a/requirements.in +++ b/requirements.in @@ -1,7 +1,6 @@ apns attrs autobahn[twisted] -boto boto3 cffi click diff --git a/requirements.txt b/requirements.txt index 9776f9aa..e08c6623 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,6 @@ attrs==17.4.0 autobahn[twisted]==17.10.1 automat==0.6.0 # via twisted boto3==1.5.23 -boto==2.48.0 botocore==1.8.37 # via boto3, s3transfer certifi==2018.1.18 # via requests cffi==1.11.4