diff --git a/MANIFEST.in b/MANIFEST.in index b20cb88757ad..505c35f4c0ea 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,7 +6,7 @@ include moto/ec2/resources/instance_types.json include moto/ec2/resources/instance_type_offerings/*/*.json include moto/ec2/resources/amis.json include moto/cognitoidp/resources/*.json -include moto/dynamodb2/parsing/reserved_keywords.txt +include moto/dynamodb/parsing/reserved_keywords.txt include moto/ssm/resources/*.json include moto/support/resources/*.json recursive-include moto/moto_server * diff --git a/docs/docs/services/dynamodb.rst b/docs/docs/services/dynamodb.rst index 661646f11dc3..bcb1a19f678e 100644 --- a/docs/docs/services/dynamodb.rst +++ b/docs/docs/services/dynamodb.rst @@ -16,8 +16,8 @@ dynamodb .. sourcecode:: python - @mock_dynamodb2 - def test_dynamodb2_behaviour: + @mock_dynamodb + def test_dynamodb_behaviour: boto3.client("dynamodb") ... diff --git a/moto/__init__.py b/moto/__init__.py index 461afa6feedf..7dc81da0c816 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -53,7 +53,7 @@ def f(*args, **kwargs): mock_budgets = lazy_load(".budgets", "mock_budgets") mock_cloudformation = lazy_load(".cloudformation", "mock_cloudformation") mock_cloudfront = lazy_load(".cloudfront", "mock_cloudfront") -mock_cloudtrail = lazy_load(".cloudtrail", "mock_cloudtrail", boto3_name="cloudtrail") +mock_cloudtrail = lazy_load(".cloudtrail", "mock_cloudtrail") mock_cloudwatch = lazy_load(".cloudwatch", "mock_cloudwatch") mock_codecommit = lazy_load(".codecommit", "mock_codecommit") mock_codepipeline = lazy_load(".codepipeline", "mock_codepipeline") @@ -67,9 +67,11 @@ def f(*args, **kwargs): mock_datasync = lazy_load(".datasync", "mock_datasync") mock_dax = lazy_load(".dax", "mock_dax") mock_dms = lazy_load(".dms", "mock_dms") -mock_ds = lazy_load(".ds", "mock_ds", boto3_name="ds") -mock_dynamodb = lazy_load(".dynamodb", "mock_dynamodb", warn_repurpose=True) -mock_dynamodb2 = lazy_load(".dynamodb2", "mock_dynamodb2", backend="dynamodb_backends2") +mock_ds = lazy_load(".ds", "mock_ds") +mock_dynamodb = lazy_load(".dynamodb", "mock_dynamodb") +mock_dynamodb2 = lazy_load( + ".dynamodb", "mock_dynamodb", use_instead=("mock_dynamodb2", "mock_dynamodb") +) mock_dynamodbstreams = lazy_load(".dynamodbstreams", "mock_dynamodbstreams") mock_elasticbeanstalk = lazy_load( ".elasticbeanstalk", "mock_elasticbeanstalk", backend="eb_backends" diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index d304844ba033..b5f3548ae69b 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -45,7 +45,7 @@ split_layer_arn, ) from moto.sqs import sqs_backends -from moto.dynamodb2 import dynamodb_backends2 +from moto.dynamodb import dynamodb_backends from moto.dynamodbstreams import dynamodbstreams_backends from moto.core import ACCOUNT_ID from moto.utilities.docker_utilities import DockerModel, parse_image_ref @@ -1259,7 +1259,7 @@ def create_event_source_mapping(self, spec): esm = EventSourceMapping(spec) self._event_source_mappings[esm.uuid] = esm table_name = stream["TableName"] - table = dynamodb_backends2[self.region_name].get_table(table_name) + table = dynamodb_backends[self.region_name].get_table(table_name) table.lambda_event_source_mappings[esm.function_arn] = esm return esm raise RESTError("ResourceNotFoundException", "Invalid EventSourceArn") diff --git a/moto/backends.py b/moto/backends.py index 67af7f480883..d44232a840e4 100644 --- a/moto/backends.py +++ b/moto/backends.py @@ -10,6 +10,7 @@ ] decorator_functions = [getattr(moto, f) for f in decorators] BACKENDS = {f.boto3_name: (f.name, f.backend) for f in decorator_functions} +BACKENDS["dynamodb_v20111205"] = ("dynamodb_v20111205", "dynamodb_backends") BACKENDS["moto_api"] = ("core", "moto_api_backends") BACKENDS["instance_metadata"] = ("instance_metadata", "instance_metadata_backends") BACKENDS["s3bucket_path"] = ("s3", "s3_backends") diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index bfed33bfe2d9..91d9ad3fbb9d 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -21,7 +21,7 @@ from moto.cloudformation.custom_model import CustomModel from moto.cloudwatch import models # noqa # pylint: disable=all from moto.datapipeline import models # noqa # pylint: disable=all -from moto.dynamodb2 import models # noqa # pylint: disable=all +from moto.dynamodb import models # noqa # pylint: disable=all from moto.ec2 import models as ec2_models from moto.ecr import models # noqa # pylint: disable=all from moto.ecs import models # noqa # pylint: disable=all diff --git a/moto/dynamodb/__init__.py b/moto/dynamodb/__init__.py index b040e49ddd85..0f01dbaf2d6a 100644 --- a/moto/dynamodb/__init__.py +++ b/moto/dynamodb/__init__.py @@ -1,4 +1,5 @@ -from .models import dynamodb_backend +from moto.dynamodb.models import dynamodb_backends +from ..core.models import base_decorator -dynamodb_backends = {"global": dynamodb_backend} -mock_dynamodb = dynamodb_backend.decorator +dynamodb_backend = dynamodb_backends["us-east-1"] +mock_dynamodb = base_decorator(dynamodb_backends) diff --git a/moto/dynamodb/comparisons.py b/moto/dynamodb/comparisons.py index f31b9d5c32d7..ce7dc5a95a15 100644 --- a/moto/dynamodb/comparisons.py +++ b/moto/dynamodb/comparisons.py @@ -1,13 +1,120 @@ +import re +from collections import deque +from collections import namedtuple + +from moto.dynamodb.exceptions import ConditionAttributeIsReservedKeyword +from moto.dynamodb.parsing.reserved_keywords import ReservedKeywords + + +def get_filter_expression(expr, names, values): + """ + Parse a filter expression into an Op. + + Examples + expr = 'Id > 5 AND attribute_exists(test) AND Id BETWEEN 5 AND 6 OR length < 6 AND contains(test, 1) AND 5 IN (4,5, 6) OR (Id < 5 AND 5 > Id)' + expr = 'Id > 5 AND Subs < 7' + """ + parser = ConditionExpressionParser(expr, names, values) + return parser.parse() + + +def get_expected(expected): + """ + Parse a filter expression into an Op. + + Examples + expr = 'Id > 5 AND attribute_exists(test) AND Id BETWEEN 5 AND 6 OR length < 6 AND contains(test, 1) AND 5 IN (4,5, 6) OR (Id < 5 AND 5 > Id)' + expr = 'Id > 5 AND Subs < 7' + """ + ops = { + "EQ": OpEqual, + "NE": OpNotEqual, + "LE": OpLessThanOrEqual, + "LT": OpLessThan, + "GE": OpGreaterThanOrEqual, + "GT": OpGreaterThan, + "NOT_NULL": FuncAttrExists, + "NULL": FuncAttrNotExists, + "CONTAINS": FuncContains, + "NOT_CONTAINS": FuncNotContains, + "BEGINS_WITH": FuncBeginsWith, + "IN": FuncIn, + "BETWEEN": FuncBetween, + } + + # NOTE: Always uses ConditionalOperator=AND + conditions = [] + for key, cond in expected.items(): + path = AttributePath([key]) + if "Exists" in cond: + if cond["Exists"]: + conditions.append(FuncAttrExists(path)) + else: + conditions.append(FuncAttrNotExists(path)) + elif "Value" in cond: + conditions.append(OpEqual(path, AttributeValue(cond["Value"]))) + elif "ComparisonOperator" in cond: + operator_name = cond["ComparisonOperator"] + values = [AttributeValue(v) for v in cond.get("AttributeValueList", [])] + OpClass = ops[operator_name] + conditions.append(OpClass(path, *values)) + + # NOTE: Ignore ConditionalOperator + ConditionalOp = OpAnd + if conditions: + output = conditions[0] + for condition in conditions[1:]: + output = ConditionalOp(output, condition) + else: + return OpDefault(None, None) + + return output + + +class Op(object): + """ + Base class for a FilterExpression operator + """ + + OP = "" + + def __init__(self, lhs, rhs): + self.lhs = lhs + self.rhs = rhs + + def expr(self, item): + raise NotImplementedError("Expr not defined for {0}".format(type(self))) + + def __repr__(self): + return "({0} {1} {2})".format(self.lhs, self.OP, self.rhs) + + # TODO add tests for all of these + +EQ_FUNCTION = lambda item_value, test_value: item_value == test_value # noqa +NE_FUNCTION = lambda item_value, test_value: item_value != test_value # noqa +LE_FUNCTION = lambda item_value, test_value: item_value <= test_value # noqa +LT_FUNCTION = lambda item_value, test_value: item_value < test_value # noqa +GE_FUNCTION = lambda item_value, test_value: item_value >= test_value # noqa +GT_FUNCTION = lambda item_value, test_value: item_value > test_value # noqa + COMPARISON_FUNCS = { - "EQ": lambda item_value, test_value: item_value == test_value, - "NE": lambda item_value, test_value: item_value != test_value, - "LE": lambda item_value, test_value: item_value <= test_value, - "LT": lambda item_value, test_value: item_value < test_value, - "GE": lambda item_value, test_value: item_value >= test_value, - "GT": lambda item_value, test_value: item_value > test_value, - "NULL": lambda item_value: item_value is None, - "NOT_NULL": lambda item_value: item_value is not None, + "EQ": EQ_FUNCTION, + "=": EQ_FUNCTION, + "NE": NE_FUNCTION, + "!=": NE_FUNCTION, + "LE": LE_FUNCTION, + "<=": LE_FUNCTION, + "LT": LT_FUNCTION, + "<": LT_FUNCTION, + "GE": GE_FUNCTION, + ">=": GE_FUNCTION, + "GT": GT_FUNCTION, + ">": GT_FUNCTION, + # NULL means the value should not exist at all + "NULL": lambda item_value: False, + # NOT_NULL means the value merely has to exist, and values of None are valid + "NOT_NULL": lambda item_value: True, "CONTAINS": lambda item_value, test_value: test_value in item_value, "NOT_CONTAINS": lambda item_value, test_value: test_value not in item_value, "BEGINS_WITH": lambda item_value, test_value: item_value.startswith(test_value), @@ -20,3 +127,1096 @@ def get_comparison_func(range_comparison): return COMPARISON_FUNCS.get(range_comparison) + + +class RecursionStopIteration(StopIteration): + pass + + +class ConditionExpressionParser: + def __init__( + self, + condition_expression, + expression_attribute_names, + expression_attribute_values, + ): + self.condition_expression = condition_expression + self.expression_attribute_names = expression_attribute_names + self.expression_attribute_values = expression_attribute_values + + def parse(self): + """Returns a syntax tree for the expression. + + The tree, and all of the nodes in the tree are a tuple of + - kind: str + - children/value: + list of nodes for parent nodes + value for leaf nodes + + Raises ValueError if the condition expression is invalid + Raises KeyError if expression attribute names/values are invalid + + Here are the types of nodes that can be returned. + The types of child nodes are denoted with a colon (:). + An arbitrary number of children is denoted with ... + + Condition: + ('OR', [lhs : Condition, rhs : Condition]) + ('AND', [lhs: Condition, rhs: Condition]) + ('NOT', [argument: Condition]) + ('PARENTHESES', [argument: Condition]) + ('FUNCTION', [('LITERAL', function_name: str), argument: Operand, ...]) + ('BETWEEN', [query: Operand, low: Operand, high: Operand]) + ('IN', [query: Operand, possible_value: Operand, ...]) + ('COMPARISON', [lhs: Operand, ('LITERAL', comparator: str), rhs: Operand]) + + Operand: + ('EXPRESSION_ATTRIBUTE_VALUE', value: dict, e.g. {'S': 'foobar'}) + ('PATH', [('LITERAL', path_element: str), ...]) + NOTE: Expression attribute names will be expanded + ('FUNCTION', [('LITERAL', 'size'), argument: Operand]) + + Literal: + ('LITERAL', value: str) + + """ + if not self.condition_expression: + return OpDefault(None, None) + nodes = self._lex_condition_expression() + nodes = self._parse_paths(nodes) + # NOTE: The docs say that functions should be parsed after + # IN, BETWEEN, and comparisons like <=. + # However, these expressions are invalid as function arguments, + # so it is okay to parse functions first. This needs to be done + # to interpret size() correctly as an operand. + nodes = self._apply_functions(nodes) + nodes = self._apply_comparator(nodes) + nodes = self._apply_in(nodes) + nodes = self._apply_between(nodes) + nodes = self._apply_parens_and_booleans(nodes) + node = nodes[0] + op = self._make_op_condition(node) + return op + + class Kind: + """Enum defining types of nodes in the syntax tree.""" + + # Condition nodes + # --------------- + OR = "OR" + AND = "AND" + NOT = "NOT" + PARENTHESES = "PARENTHESES" + FUNCTION = "FUNCTION" + BETWEEN = "BETWEEN" + IN = "IN" + COMPARISON = "COMPARISON" + + # Operand nodes + # ------------- + EXPRESSION_ATTRIBUTE_VALUE = "EXPRESSION_ATTRIBUTE_VALUE" + PATH = "PATH" + + # Literal nodes + # -------------- + LITERAL = "LITERAL" + + class Nonterminal: + """Enum defining nonterminals for productions.""" + + CONDITION = "CONDITION" + OPERAND = "OPERAND" + COMPARATOR = "COMPARATOR" + FUNCTION_NAME = "FUNCTION_NAME" + IDENTIFIER = "IDENTIFIER" + AND = "AND" + OR = "OR" + NOT = "NOT" + BETWEEN = "BETWEEN" + IN = "IN" + COMMA = "COMMA" + LEFT_PAREN = "LEFT_PAREN" + RIGHT_PAREN = "RIGHT_PAREN" + WHITESPACE = "WHITESPACE" + + Node = namedtuple("Node", ["nonterminal", "kind", "text", "value", "children"]) + + @classmethod + def raise_exception_if_keyword(cls, attribute): + if attribute.upper() in ReservedKeywords.get_reserved_keywords(): + raise ConditionAttributeIsReservedKeyword(attribute) + + def _lex_condition_expression(self): + nodes = deque() + remaining_expression = self.condition_expression + while remaining_expression: + node, remaining_expression = self._lex_one_node(remaining_expression) + if node.nonterminal == self.Nonterminal.WHITESPACE: + continue + nodes.append(node) + return nodes + + def _lex_one_node(self, remaining_expression): + # TODO: Handle indexing like [1] + attribute_regex = r"(:|#)?[A-z0-9\-_]+" + patterns = [ + (self.Nonterminal.WHITESPACE, re.compile(r"^ +")), + ( + self.Nonterminal.COMPARATOR, + re.compile( + "^(" + # Put long expressions first for greedy matching + "<>|" + "<=|" + ">=|" + "=|" + "<|" + ">)" + ), + ), + ( + self.Nonterminal.OPERAND, + re.compile( + r"^{attribute_regex}(\.{attribute_regex}|\[[0-9]\])*".format( + attribute_regex=attribute_regex + ) + ), + ), + (self.Nonterminal.COMMA, re.compile(r"^,")), + (self.Nonterminal.LEFT_PAREN, re.compile(r"^\(")), + (self.Nonterminal.RIGHT_PAREN, re.compile(r"^\)")), + ] + + for nonterminal, pattern in patterns: + match = pattern.match(remaining_expression) + if match: + match_text = match.group() + break + else: # pragma: no cover + raise ValueError( + "Cannot parse condition starting at:{}".format(remaining_expression) + ) + + node = self.Node( + nonterminal=nonterminal, + kind=self.Kind.LITERAL, + text=match_text, + value=match_text, + children=[], + ) + + remaining_expression = remaining_expression[len(match_text) :] + + return node, remaining_expression + + def _parse_paths(self, nodes): + output = deque() + + while nodes: + node = nodes.popleft() + + if node.nonterminal == self.Nonterminal.OPERAND: + path = node.value.replace("[", ".[").split(".") + children = [self._parse_path_element(name) for name in path] + if len(children) == 1: + child = children[0] + if child.nonterminal != self.Nonterminal.IDENTIFIER: + output.append(child) + continue + else: + for child in children: + self._assert( + child.nonterminal == self.Nonterminal.IDENTIFIER, + "Cannot use {} in path".format(child.text), + [node], + ) + output.append( + self.Node( + nonterminal=self.Nonterminal.OPERAND, + kind=self.Kind.PATH, + text=node.text, + value=None, + children=children, + ) + ) + else: + output.append(node) + return output + + def _parse_path_element(self, name): + reserved = { + "and": self.Nonterminal.AND, + "or": self.Nonterminal.OR, + "in": self.Nonterminal.IN, + "between": self.Nonterminal.BETWEEN, + "not": self.Nonterminal.NOT, + } + + functions = { + "attribute_exists", + "attribute_not_exists", + "attribute_type", + "begins_with", + "contains", + "size", + } + + if name.lower() in reserved: + # e.g. AND + nonterminal = reserved[name.lower()] + return self.Node( + nonterminal=nonterminal, + kind=self.Kind.LITERAL, + text=name, + value=name, + children=[], + ) + elif name in functions: + # e.g. attribute_exists + return self.Node( + nonterminal=self.Nonterminal.FUNCTION_NAME, + kind=self.Kind.LITERAL, + text=name, + value=name, + children=[], + ) + elif name.startswith(":"): + # e.g. :value0 + return self.Node( + nonterminal=self.Nonterminal.OPERAND, + kind=self.Kind.EXPRESSION_ATTRIBUTE_VALUE, + text=name, + value=self._lookup_expression_attribute_value(name), + children=[], + ) + elif name.startswith("#"): + # e.g. #name0 + return self.Node( + nonterminal=self.Nonterminal.IDENTIFIER, + kind=self.Kind.LITERAL, + text=name, + value=self._lookup_expression_attribute_name(name), + children=[], + ) + elif name.startswith("["): + # e.g. [123] + if not name.endswith("]"): # pragma: no cover + raise ValueError("Bad path element {}".format(name)) + return self.Node( + nonterminal=self.Nonterminal.IDENTIFIER, + kind=self.Kind.LITERAL, + text=name, + value=int(name[1:-1]), + children=[], + ) + else: + # e.g. ItemId + self.raise_exception_if_keyword(name) + return self.Node( + nonterminal=self.Nonterminal.IDENTIFIER, + kind=self.Kind.LITERAL, + text=name, + value=name, + children=[], + ) + + def _lookup_expression_attribute_value(self, name): + return self.expression_attribute_values[name] + + def _lookup_expression_attribute_name(self, name): + return self.expression_attribute_names[name] + + # NOTE: The following constructions are ordered from high precedence to low precedence + # according to + # https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.OperatorsAndFunctions.html#Expressions.OperatorsAndFunctions.Precedence + # + # = <> < <= > >= + # IN + # BETWEEN + # attribute_exists attribute_not_exists begins_with contains + # Parentheses + # NOT + # AND + # OR + # + # The grammar is taken from + # https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.OperatorsAndFunctions.html#Expressions.OperatorsAndFunctions.Syntax + # + # condition-expression ::= + # operand comparator operand + # operand BETWEEN operand AND operand + # operand IN ( operand (',' operand (, ...) )) + # function + # condition AND condition + # condition OR condition + # NOT condition + # ( condition ) + # + # comparator ::= + # = + # <> + # < + # <= + # > + # >= + # + # function ::= + # attribute_exists (path) + # attribute_not_exists (path) + # attribute_type (path, type) + # begins_with (path, substr) + # contains (path, operand) + # size (path) + + def _matches(self, nodes, production): + """Check if the nodes start with the given production. + + Parameters + ---------- + nodes: list of Node + production: list of str + The name of a Nonterminal, or '*' for anything + + """ + if len(nodes) < len(production): + return False + for i in range(len(production)): + if production[i] == "*": + continue + expected = getattr(self.Nonterminal, production[i]) + if nodes[i].nonterminal != expected: + return False + return True + + def _apply_comparator(self, nodes): + """Apply condition := operand comparator operand.""" + output = deque() + + while nodes: + if self._matches(nodes, ["*", "COMPARATOR"]): + self._assert( + self._matches(nodes, ["OPERAND", "COMPARATOR", "OPERAND"]), + "Bad comparison", + list(nodes)[:3], + ) + lhs = nodes.popleft() + comparator = nodes.popleft() + rhs = nodes.popleft() + nodes.appendleft( + self.Node( + nonterminal=self.Nonterminal.CONDITION, + kind=self.Kind.COMPARISON, + text=" ".join([lhs.text, comparator.text, rhs.text]), + value=None, + children=[lhs, comparator, rhs], + ) + ) + else: + output.append(nodes.popleft()) + return output + + def _apply_in(self, nodes): + """Apply condition := operand IN ( operand , ... ).""" + output = deque() + while nodes: + if self._matches(nodes, ["*", "IN"]): + self._assert( + self._matches(nodes, ["OPERAND", "IN", "LEFT_PAREN"]), + "Bad IN expression", + list(nodes)[:3], + ) + lhs = nodes.popleft() + in_node = nodes.popleft() + left_paren = nodes.popleft() + all_children = [lhs, in_node, left_paren] + rhs = [] + while True: + if self._matches(nodes, ["OPERAND", "COMMA"]): + operand = nodes.popleft() + separator = nodes.popleft() + all_children += [operand, separator] + rhs.append(operand) + elif self._matches(nodes, ["OPERAND", "RIGHT_PAREN"]): + operand = nodes.popleft() + separator = nodes.popleft() + all_children += [operand, separator] + rhs.append(operand) + break # Close + else: + self._assert(False, "Bad IN expression starting at", nodes) + nodes.appendleft( + self.Node( + nonterminal=self.Nonterminal.CONDITION, + kind=self.Kind.IN, + text=" ".join([t.text for t in all_children]), + value=None, + children=[lhs] + rhs, + ) + ) + else: + output.append(nodes.popleft()) + return output + + def _apply_between(self, nodes): + """Apply condition := operand BETWEEN operand AND operand.""" + output = deque() + while nodes: + if self._matches(nodes, ["*", "BETWEEN"]): + self._assert( + self._matches( + nodes, ["OPERAND", "BETWEEN", "OPERAND", "AND", "OPERAND"] + ), + "Bad BETWEEN expression", + list(nodes)[:5], + ) + lhs = nodes.popleft() + between_node = nodes.popleft() + low = nodes.popleft() + and_node = nodes.popleft() + high = nodes.popleft() + all_children = [lhs, between_node, low, and_node, high] + nodes.appendleft( + self.Node( + nonterminal=self.Nonterminal.CONDITION, + kind=self.Kind.BETWEEN, + text=" ".join([t.text for t in all_children]), + value=None, + children=[lhs, low, high], + ) + ) + else: + output.append(nodes.popleft()) + return output + + def _apply_functions(self, nodes): + """Apply condition := function_name (operand , ...).""" + output = deque() + either_kind = {self.Kind.PATH, self.Kind.EXPRESSION_ATTRIBUTE_VALUE} + expected_argument_kind_map = { + "attribute_exists": [{self.Kind.PATH}], + "attribute_not_exists": [{self.Kind.PATH}], + "attribute_type": [either_kind, {self.Kind.EXPRESSION_ATTRIBUTE_VALUE}], + "begins_with": [either_kind, either_kind], + "contains": [either_kind, either_kind], + "size": [{self.Kind.PATH}], + } + while nodes: + if self._matches(nodes, ["FUNCTION_NAME"]): + self._assert( + self._matches( + nodes, ["FUNCTION_NAME", "LEFT_PAREN", "OPERAND", "*"] + ), + "Bad function expression at", + list(nodes)[:4], + ) + function_name = nodes.popleft() + left_paren = nodes.popleft() + all_children = [function_name, left_paren] + arguments = [] + while True: + if self._matches(nodes, ["OPERAND", "COMMA"]): + operand = nodes.popleft() + separator = nodes.popleft() + all_children += [operand, separator] + arguments.append(operand) + elif self._matches(nodes, ["OPERAND", "RIGHT_PAREN"]): + operand = nodes.popleft() + separator = nodes.popleft() + all_children += [operand, separator] + arguments.append(operand) + break # Close paren + else: + self._assert( + False, + "Bad function expression", + all_children + list(nodes)[:2], + ) + expected_kinds = expected_argument_kind_map[function_name.value] + self._assert( + len(arguments) == len(expected_kinds), + "Wrong number of arguments in", + all_children, + ) + for i in range(len(expected_kinds)): + self._assert( + arguments[i].kind in expected_kinds[i], + "Wrong type for argument %d in" % i, + all_children, + ) + if function_name.value == "size": + nonterminal = self.Nonterminal.OPERAND + else: + nonterminal = self.Nonterminal.CONDITION + nodes.appendleft( + self.Node( + nonterminal=nonterminal, + kind=self.Kind.FUNCTION, + text=" ".join([t.text for t in all_children]), + value=None, + children=[function_name] + arguments, + ) + ) + else: + output.append(nodes.popleft()) + return output + + def _apply_parens_and_booleans(self, nodes, left_paren=None): + """Apply condition := ( condition ) and booleans.""" + output = deque() + while nodes: + if self._matches(nodes, ["LEFT_PAREN"]): + parsed = self._apply_parens_and_booleans( + nodes, left_paren=nodes.popleft() + ) + self._assert(len(parsed) >= 1, "Failed to close parentheses at", nodes) + parens = parsed.popleft() + self._assert( + parens.kind == self.Kind.PARENTHESES, + "Failed to close parentheses at", + nodes, + ) + output.append(parens) + nodes = parsed + elif self._matches(nodes, ["RIGHT_PAREN"]): + self._assert(left_paren is not None, "Unmatched ) at", nodes) + close_paren = nodes.popleft() + children = self._apply_booleans(output) + all_children = [left_paren] + list(children) + [close_paren] + return deque( + [ + self.Node( + nonterminal=self.Nonterminal.CONDITION, + kind=self.Kind.PARENTHESES, + text=" ".join([t.text for t in all_children]), + value=None, + children=list(children), + ) + ] + + list(nodes) + ) + else: + output.append(nodes.popleft()) + + self._assert(left_paren is None, "Unmatched ( at", list(output)) + return self._apply_booleans(output) + + def _apply_booleans(self, nodes): + """Apply and, or, and not constructions.""" + nodes = self._apply_not(nodes) + nodes = self._apply_and(nodes) + nodes = self._apply_or(nodes) + # The expression should reduce to a single condition + self._assert(len(nodes) == 1, "Unexpected expression at", list(nodes)[1:]) + self._assert( + nodes[0].nonterminal == self.Nonterminal.CONDITION, + "Incomplete condition", + nodes, + ) + return nodes + + def _apply_not(self, nodes): + """Apply condition := NOT condition.""" + output = deque() + while nodes: + if self._matches(nodes, ["NOT"]): + self._assert( + self._matches(nodes, ["NOT", "CONDITION"]), + "Bad NOT expression", + list(nodes)[:2], + ) + not_node = nodes.popleft() + child = nodes.popleft() + nodes.appendleft( + self.Node( + nonterminal=self.Nonterminal.CONDITION, + kind=self.Kind.NOT, + text=" ".join([not_node.text, child.text]), + value=None, + children=[child], + ) + ) + else: + output.append(nodes.popleft()) + + return output + + def _apply_and(self, nodes): + """Apply condition := condition AND condition.""" + output = deque() + while nodes: + if self._matches(nodes, ["*", "AND"]): + self._assert( + self._matches(nodes, ["CONDITION", "AND", "CONDITION"]), + "Bad AND expression", + list(nodes)[:3], + ) + lhs = nodes.popleft() + and_node = nodes.popleft() + rhs = nodes.popleft() + all_children = [lhs, and_node, rhs] + nodes.appendleft( + self.Node( + nonterminal=self.Nonterminal.CONDITION, + kind=self.Kind.AND, + text=" ".join([t.text for t in all_children]), + value=None, + children=[lhs, rhs], + ) + ) + else: + output.append(nodes.popleft()) + + return output + + def _apply_or(self, nodes): + """Apply condition := condition OR condition.""" + output = deque() + while nodes: + if self._matches(nodes, ["*", "OR"]): + self._assert( + self._matches(nodes, ["CONDITION", "OR", "CONDITION"]), + "Bad OR expression", + list(nodes)[:3], + ) + lhs = nodes.popleft() + or_node = nodes.popleft() + rhs = nodes.popleft() + all_children = [lhs, or_node, rhs] + nodes.appendleft( + self.Node( + nonterminal=self.Nonterminal.CONDITION, + kind=self.Kind.OR, + text=" ".join([t.text for t in all_children]), + value=None, + children=[lhs, rhs], + ) + ) + else: + output.append(nodes.popleft()) + + return output + + def _make_operand(self, node): + if node.kind == self.Kind.PATH: + return AttributePath([child.value for child in node.children]) + elif node.kind == self.Kind.EXPRESSION_ATTRIBUTE_VALUE: + return AttributeValue(node.value) + elif node.kind == self.Kind.FUNCTION: + # size() + function_node = node.children[0] + arguments = node.children[1:] + function_name = function_node.value + arguments = [self._make_operand(arg) for arg in arguments] + return FUNC_CLASS[function_name](*arguments) + else: # pragma: no cover + raise ValueError("Unknown operand: %r" % node) + + def _make_op_condition(self, node): + if node.kind == self.Kind.OR: + lhs, rhs = node.children + return OpOr(self._make_op_condition(lhs), self._make_op_condition(rhs)) + elif node.kind == self.Kind.AND: + lhs, rhs = node.children + return OpAnd(self._make_op_condition(lhs), self._make_op_condition(rhs)) + elif node.kind == self.Kind.NOT: + (child,) = node.children + return OpNot(self._make_op_condition(child)) + elif node.kind == self.Kind.PARENTHESES: + (child,) = node.children + return self._make_op_condition(child) + elif node.kind == self.Kind.FUNCTION: + function_node = node.children[0] + arguments = node.children[1:] + function_name = function_node.value + arguments = [self._make_operand(arg) for arg in arguments] + return FUNC_CLASS[function_name](*arguments) + elif node.kind == self.Kind.BETWEEN: + query, low, high = node.children + return FuncBetween( + self._make_operand(query), + self._make_operand(low), + self._make_operand(high), + ) + elif node.kind == self.Kind.IN: + query = node.children[0] + possible_values = node.children[1:] + query = self._make_operand(query) + possible_values = [self._make_operand(v) for v in possible_values] + return FuncIn(query, *possible_values) + elif node.kind == self.Kind.COMPARISON: + lhs, comparator, rhs = node.children + return COMPARATOR_CLASS[comparator.value]( + self._make_operand(lhs), self._make_operand(rhs) + ) + else: # pragma: no cover + raise ValueError("Unknown expression node kind %r" % node.kind) + + def _assert(self, condition, message, nodes): + if not condition: + raise ValueError(message + " " + " ".join([t.text for t in nodes])) + + +class Operand(object): + def expr(self, item): + raise NotImplementedError + + def get_type(self, item): + raise NotImplementedError + + +class AttributePath(Operand): + def __init__(self, path): + """Initialize the AttributePath. + + Parameters + ---------- + path: list of int/str + + """ + assert len(path) >= 1 + self.path = path + + def _get_attr(self, item): + if item is None: + return None + + base = self.path[0] + if base not in item.attrs: + return None + attr = item.attrs[base] + + for name in self.path[1:]: + attr = attr.child_attr(name) + if attr is None: + return None + + return attr + + def expr(self, item): + attr = self._get_attr(item) + if attr is None: + return None + else: + return attr.cast_value + + def get_type(self, item): + attr = self._get_attr(item) + if attr is None: + return None + else: + return attr.type + + def __repr__(self): + return ".".join(self.path) + + +class AttributeValue(Operand): + def __init__(self, value): + """Initialize the AttributePath. + + Parameters + ---------- + value: dict + e.g. {'N': '1.234'} + + """ + self.type = list(value.keys())[0] + self.value = value[self.type] + + def expr(self, item): + # TODO: Reuse DynamoType code + if self.type == "N": + try: + return int(self.value) + except ValueError: + return float(self.value) + elif self.type in ["SS", "NS", "BS"]: + sub_type = self.type[0] + return set([AttributeValue({sub_type: v}).expr(item) for v in self.value]) + elif self.type == "L": + return [AttributeValue(v).expr(item) for v in self.value] + elif self.type == "M": + return dict( + [(k, AttributeValue(v).expr(item)) for k, v in self.value.items()] + ) + else: + return self.value + return self.value + + def get_type(self, item): + return self.type + + def __repr__(self): + return repr(self.value) + + +class OpDefault(Op): + OP = "NONE" + + def expr(self, item): + """If no condition is specified, always True.""" + return True + + +class OpNot(Op): + OP = "NOT" + + def __init__(self, lhs): + super().__init__(lhs, None) + + def expr(self, item): + lhs = self.lhs.expr(item) + return not lhs + + def __str__(self): + return "({0} {1})".format(self.OP, self.lhs) + + +class OpAnd(Op): + OP = "AND" + + def expr(self, item): + lhs = self.lhs.expr(item) + return lhs and self.rhs.expr(item) + + +class OpLessThan(Op): + OP = "<" + + def expr(self, item): + lhs = self.lhs.expr(item) + rhs = self.rhs.expr(item) + # In python3 None is not a valid comparator when using < or > so must be handled specially + if lhs is not None and rhs is not None: + return lhs < rhs + else: + return False + + +class OpGreaterThan(Op): + OP = ">" + + def expr(self, item): + lhs = self.lhs.expr(item) + rhs = self.rhs.expr(item) + # In python3 None is not a valid comparator when using < or > so must be handled specially + if lhs is not None and rhs is not None: + return lhs > rhs + else: + return False + + +class OpEqual(Op): + OP = "=" + + def expr(self, item): + lhs = self.lhs.expr(item) + rhs = self.rhs.expr(item) + return lhs == rhs + + +class OpNotEqual(Op): + OP = "<>" + + def expr(self, item): + lhs = self.lhs.expr(item) + rhs = self.rhs.expr(item) + return lhs != rhs + + +class OpLessThanOrEqual(Op): + OP = "<=" + + def expr(self, item): + lhs = self.lhs.expr(item) + rhs = self.rhs.expr(item) + # In python3 None is not a valid comparator when using < or > so must be handled specially + if lhs is not None and rhs is not None: + return lhs <= rhs + else: + return False + + +class OpGreaterThanOrEqual(Op): + OP = ">=" + + def expr(self, item): + lhs = self.lhs.expr(item) + rhs = self.rhs.expr(item) + # In python3 None is not a valid comparator when using < or > so must be handled specially + if lhs is not None and rhs is not None: + return lhs >= rhs + else: + return False + + +class OpOr(Op): + OP = "OR" + + def expr(self, item): + lhs = self.lhs.expr(item) + return lhs or self.rhs.expr(item) + + +class Func(object): + """ + Base class for a FilterExpression function + """ + + FUNC = "Unknown" + + def __init__(self, *arguments): + self.arguments = arguments + + def expr(self, item): + raise NotImplementedError + + def __repr__(self): + return "{0}({1})".format( + self.FUNC, " ".join([repr(arg) for arg in self.arguments]) + ) + + +class FuncAttrExists(Func): + FUNC = "attribute_exists" + + def __init__(self, attribute): + self.attr = attribute + super().__init__(attribute) + + def expr(self, item): + return self.attr.get_type(item) is not None + + +def FuncAttrNotExists(attribute): + return OpNot(FuncAttrExists(attribute)) + + +class FuncAttrType(Func): + FUNC = "attribute_type" + + def __init__(self, attribute, _type): + self.attr = attribute + self.type = _type + super().__init__(attribute, _type) + + def expr(self, item): + return self.attr.get_type(item) == self.type.expr(item) + + +class FuncBeginsWith(Func): + FUNC = "begins_with" + + def __init__(self, attribute, substr): + self.attr = attribute + self.substr = substr + super().__init__(attribute, substr) + + def expr(self, item): + if self.attr.get_type(item) != "S": + return False + if self.substr.get_type(item) != "S": + return False + return self.attr.expr(item).startswith(self.substr.expr(item)) + + +class FuncContains(Func): + FUNC = "contains" + + def __init__(self, attribute, operand): + self.attr = attribute + self.operand = operand + super().__init__(attribute, operand) + + def expr(self, item): + if self.attr.get_type(item) in ("S", "SS", "NS", "BS", "L"): + try: + return self.operand.expr(item) in self.attr.expr(item) + except TypeError: + return False + return False + + +def FuncNotContains(attribute, operand): + return OpNot(FuncContains(attribute, operand)) + + +class FuncSize(Func): + FUNC = "size" + + def __init__(self, attribute): + self.attr = attribute + super().__init__(attribute) + + def expr(self, item): + if self.attr.get_type(item) is None: + raise ValueError("Invalid attribute name {0}".format(self.attr)) + + if self.attr.get_type(item) in ("S", "SS", "NS", "B", "BS", "L", "M"): + return len(self.attr.expr(item)) + raise ValueError("Invalid filter expression") + + +class FuncBetween(Func): + FUNC = "BETWEEN" + + def __init__(self, attribute, start, end): + self.attr = attribute + self.start = start + self.end = end + super().__init__(attribute, start, end) + + def expr(self, item): + # In python3 None is not a valid comparator when using < or > so must be handled specially + start = self.start.expr(item) + attr = self.attr.expr(item) + end = self.end.expr(item) + # Need to verify whether start has a valid value + # Can't just check 'if start', because start could be 0, which is a valid integer + start_has_value = start is not None and (isinstance(start, int) or start) + end_has_value = end is not None and (isinstance(end, int) or end) + if start_has_value and attr and end_has_value: + return start <= attr <= end + elif start is None and attr is None: + # None is between None and None as well as None is between None and any number + return True + elif start is None and attr and end: + return attr <= end + else: + return False + + +class FuncIn(Func): + FUNC = "IN" + + def __init__(self, attribute, *possible_values): + self.attr = attribute + self.possible_values = possible_values + super().__init__(attribute, *possible_values) + + def expr(self, item): + for possible_value in self.possible_values: + if self.attr.expr(item) == possible_value.expr(item): + return True + + return False + + +COMPARATOR_CLASS = { + "<": OpLessThan, + ">": OpGreaterThan, + "<=": OpLessThanOrEqual, + ">=": OpGreaterThanOrEqual, + "=": OpEqual, + "<>": OpNotEqual, +} + +FUNC_CLASS = { + "attribute_exists": FuncAttrExists, + "attribute_not_exists": FuncAttrNotExists, + "attribute_type": FuncAttrType, + "begins_with": FuncBeginsWith, + "contains": FuncContains, + "size": FuncSize, + "between": FuncBetween, +} diff --git a/moto/dynamodb2/exceptions.py b/moto/dynamodb/exceptions.py similarity index 99% rename from moto/dynamodb2/exceptions.py rename to moto/dynamodb/exceptions.py index d5e595e64f02..90401dbf1f9b 100644 --- a/moto/dynamodb2/exceptions.py +++ b/moto/dynamodb/exceptions.py @@ -1,4 +1,4 @@ -from moto.dynamodb2.limits import HASH_KEY_MAX_LENGTH, RANGE_KEY_MAX_LENGTH +from moto.dynamodb.limits import HASH_KEY_MAX_LENGTH, RANGE_KEY_MAX_LENGTH class InvalidIndexNameError(ValueError): diff --git a/moto/dynamodb2/limits.py b/moto/dynamodb/limits.py similarity index 100% rename from moto/dynamodb2/limits.py rename to moto/dynamodb/limits.py diff --git a/moto/dynamodb2/models/__init__.py b/moto/dynamodb/models/__init__.py similarity index 99% rename from moto/dynamodb2/models/__init__.py rename to moto/dynamodb/models/__init__.py index 892cfa2af4c4..62cd5fdc2ced 100644 --- a/moto/dynamodb2/models/__init__.py +++ b/moto/dynamodb/models/__init__.py @@ -11,9 +11,9 @@ from moto.core import BaseBackend, BaseModel, CloudFormationModel from moto.core.utils import unix_time, unix_time_millis, BackendDict from moto.core.exceptions import JsonRESTError -from moto.dynamodb2.comparisons import get_filter_expression -from moto.dynamodb2.comparisons import get_expected -from moto.dynamodb2.exceptions import ( +from moto.dynamodb.comparisons import get_filter_expression +from moto.dynamodb.comparisons import get_expected +from moto.dynamodb.exceptions import ( InvalidIndexNameError, ItemSizeTooLarge, ItemSizeToUpdateTooLarge, @@ -26,12 +26,12 @@ MultipleTransactionsException, TooManyTransactionsException, ) -from moto.dynamodb2.models.utilities import bytesize -from moto.dynamodb2.models.dynamo_type import DynamoType -from moto.dynamodb2.parsing.executors import UpdateExpressionExecutor -from moto.dynamodb2.parsing.expressions import UpdateExpressionParser -from moto.dynamodb2.parsing.validators import UpdateExpressionValidator -from moto.dynamodb2.limits import HASH_KEY_MAX_LENGTH, RANGE_KEY_MAX_LENGTH +from moto.dynamodb.models.utilities import bytesize +from moto.dynamodb.models.dynamo_type import DynamoType +from moto.dynamodb.parsing.executors import UpdateExpressionExecutor +from moto.dynamodb.parsing.expressions import UpdateExpressionParser +from moto.dynamodb.parsing.validators import UpdateExpressionValidator +from moto.dynamodb.limits import HASH_KEY_MAX_LENGTH, RANGE_KEY_MAX_LENGTH class DynamoJsonEncoder(json.JSONEncoder): diff --git a/moto/dynamodb2/models/dynamo_type.py b/moto/dynamodb/models/dynamo_type.py similarity index 97% rename from moto/dynamodb2/models/dynamo_type.py rename to moto/dynamodb/models/dynamo_type.py index e2d35fc8538d..2bf79663c194 100644 --- a/moto/dynamodb2/models/dynamo_type.py +++ b/moto/dynamodb/models/dynamo_type.py @@ -1,6 +1,6 @@ -from moto.dynamodb2.comparisons import get_comparison_func -from moto.dynamodb2.exceptions import IncorrectDataType -from moto.dynamodb2.models.utilities import bytesize +from moto.dynamodb.comparisons import get_comparison_func +from moto.dynamodb.exceptions import IncorrectDataType +from moto.dynamodb.models.utilities import bytesize class DDBType(object): diff --git a/moto/dynamodb2/models/utilities.py b/moto/dynamodb/models/utilities.py similarity index 100% rename from moto/dynamodb2/models/utilities.py rename to moto/dynamodb/models/utilities.py diff --git a/moto/dynamodb2/parsing/README.md b/moto/dynamodb/parsing/README.md similarity index 100% rename from moto/dynamodb2/parsing/README.md rename to moto/dynamodb/parsing/README.md diff --git a/moto/dynamodb2/parsing/__init__.py b/moto/dynamodb/parsing/__init__.py similarity index 100% rename from moto/dynamodb2/parsing/__init__.py rename to moto/dynamodb/parsing/__init__.py diff --git a/moto/dynamodb2/parsing/ast_nodes.py b/moto/dynamodb/parsing/ast_nodes.py similarity index 99% rename from moto/dynamodb2/parsing/ast_nodes.py rename to moto/dynamodb/parsing/ast_nodes.py index 515b180b4f5d..b8e332e3540d 100644 --- a/moto/dynamodb2/parsing/ast_nodes.py +++ b/moto/dynamodb/parsing/ast_nodes.py @@ -2,7 +2,7 @@ from abc import abstractmethod from collections import deque -from moto.dynamodb2.models import DynamoType +from moto.dynamodb.models import DynamoType from ..exceptions import TooManyAddClauses diff --git a/moto/dynamodb2/parsing/executors.py b/moto/dynamodb/parsing/executors.py similarity index 97% rename from moto/dynamodb2/parsing/executors.py rename to moto/dynamodb/parsing/executors.py index 538e26cbfa2d..2b403ff14c4e 100644 --- a/moto/dynamodb2/parsing/executors.py +++ b/moto/dynamodb/parsing/executors.py @@ -1,13 +1,13 @@ from abc import abstractmethod -from moto.dynamodb2.exceptions import ( +from moto.dynamodb.exceptions import ( IncorrectOperandType, IncorrectDataType, ProvidedKeyDoesNotExist, ) -from moto.dynamodb2.models import DynamoType -from moto.dynamodb2.models.dynamo_type import DDBTypeConversion, DDBType -from moto.dynamodb2.parsing.ast_nodes import ( +from moto.dynamodb.models import DynamoType +from moto.dynamodb.models.dynamo_type import DDBTypeConversion, DDBType +from moto.dynamodb.parsing.ast_nodes import ( UpdateExpressionSetAction, UpdateExpressionDeleteAction, UpdateExpressionRemoveAction, @@ -18,7 +18,7 @@ ExpressionSelector, ExpressionAttributeName, ) -from moto.dynamodb2.parsing.validators import ExpressionPathResolver +from moto.dynamodb.parsing.validators import ExpressionPathResolver class NodeExecutor(object): diff --git a/moto/dynamodb2/parsing/expressions.py b/moto/dynamodb/parsing/expressions.py similarity index 99% rename from moto/dynamodb2/parsing/expressions.py rename to moto/dynamodb/parsing/expressions.py index e9c53a15e24e..5657ebf3b6d2 100644 --- a/moto/dynamodb2/parsing/expressions.py +++ b/moto/dynamodb/parsing/expressions.py @@ -3,7 +3,7 @@ import abc from collections import deque -from moto.dynamodb2.parsing.ast_nodes import ( +from moto.dynamodb.parsing.ast_nodes import ( UpdateExpression, UpdateExpressionSetClause, UpdateExpressionSetActions, @@ -28,8 +28,8 @@ UpdateExpressionDeleteActions, UpdateExpressionDeleteClause, ) -from moto.dynamodb2.exceptions import InvalidTokenException, InvalidUpdateExpression -from moto.dynamodb2.parsing.tokens import Token, ExpressionTokenizer +from moto.dynamodb.exceptions import InvalidTokenException, InvalidUpdateExpression +from moto.dynamodb.parsing.tokens import Token, ExpressionTokenizer logger = logging.getLogger(__name__) diff --git a/moto/dynamodb2/parsing/reserved_keywords.py b/moto/dynamodb/parsing/reserved_keywords.py similarity index 100% rename from moto/dynamodb2/parsing/reserved_keywords.py rename to moto/dynamodb/parsing/reserved_keywords.py diff --git a/moto/dynamodb2/parsing/reserved_keywords.txt b/moto/dynamodb/parsing/reserved_keywords.txt similarity index 100% rename from moto/dynamodb2/parsing/reserved_keywords.txt rename to moto/dynamodb/parsing/reserved_keywords.txt diff --git a/moto/dynamodb2/parsing/tokens.py b/moto/dynamodb/parsing/tokens.py similarity index 99% rename from moto/dynamodb2/parsing/tokens.py rename to moto/dynamodb/parsing/tokens.py index 33a25d5c0d16..a83cf7ef37be 100644 --- a/moto/dynamodb2/parsing/tokens.py +++ b/moto/dynamodb/parsing/tokens.py @@ -1,6 +1,6 @@ import re -from moto.dynamodb2.exceptions import ( +from moto.dynamodb.exceptions import ( InvalidTokenException, InvalidExpressionAttributeNameKey, ) diff --git a/moto/dynamodb2/parsing/validators.py b/moto/dynamodb/parsing/validators.py similarity index 98% rename from moto/dynamodb2/parsing/validators.py rename to moto/dynamodb/parsing/validators.py index a03658188ad1..3969665b0b5a 100644 --- a/moto/dynamodb2/parsing/validators.py +++ b/moto/dynamodb/parsing/validators.py @@ -4,7 +4,7 @@ from abc import abstractmethod from copy import deepcopy -from moto.dynamodb2.exceptions import ( +from moto.dynamodb.exceptions import ( AttributeIsReservedKeyword, ExpressionAttributeValueNotDefined, AttributeDoesNotExist, @@ -15,8 +15,8 @@ EmptyKeyAttributeException, UpdateHashRangeKeyException, ) -from moto.dynamodb2.models import DynamoType -from moto.dynamodb2.parsing.ast_nodes import ( +from moto.dynamodb.models import DynamoType +from moto.dynamodb.parsing.ast_nodes import ( ExpressionAttribute, UpdateExpressionPath, UpdateExpressionSetAction, @@ -34,7 +34,7 @@ ExpressionValueOperator, ExpressionSelector, ) -from moto.dynamodb2.parsing.reserved_keywords import ReservedKeywords +from moto.dynamodb.parsing.reserved_keywords import ReservedKeywords class ExpressionAttributeValueProcessor(DepthFirstTraverser): diff --git a/moto/dynamodb/responses.py b/moto/dynamodb/responses.py index 4584ae29a85a..1d9afe455d04 100644 --- a/moto/dynamodb/responses.py +++ b/moto/dynamodb/responses.py @@ -1,8 +1,102 @@ +import copy import json +import re + +import itertools +from functools import wraps from moto.core.responses import BaseResponse -from moto.core.utils import camelcase_to_underscores -from .models import dynamodb_backend, dynamo_json_dump +from moto.core.utils import camelcase_to_underscores, amz_crc32, amzn_request_id +from .exceptions import ( + InvalidIndexNameError, + MockValidationException, + TransactionCanceledException, +) +from moto.dynamodb.models import dynamodb_backends, dynamo_json_dump + + +TRANSACTION_MAX_ITEMS = 25 + + +def include_consumed_capacity(val=1.0): + def _inner(f): + @wraps(f) + def _wrapper(*args, **kwargs): + (handler,) = args + expected_capacity = handler.body.get("ReturnConsumedCapacity", "NONE") + if expected_capacity not in ["NONE", "TOTAL", "INDEXES"]: + type_ = "ValidationException" + message = "1 validation error detected: Value '{}' at 'returnConsumedCapacity' failed to satisfy constraint: Member must satisfy enum value set: [INDEXES, TOTAL, NONE]".format( + expected_capacity + ) + return ( + 400, + handler.response_headers, + dynamo_json_dump({"__type": type_, "message": message}), + ) + table_name = handler.body.get("TableName", "") + index_name = handler.body.get("IndexName", None) + + response = f(*args, **kwargs) + + if isinstance(response, str): + body = json.loads(response) + + if expected_capacity == "TOTAL": + body["ConsumedCapacity"] = { + "TableName": table_name, + "CapacityUnits": val, + } + elif expected_capacity == "INDEXES": + body["ConsumedCapacity"] = { + "TableName": table_name, + "CapacityUnits": val, + "Table": {"CapacityUnits": val}, + } + if index_name: + body["ConsumedCapacity"]["LocalSecondaryIndexes"] = { + index_name: {"CapacityUnits": val} + } + + return dynamo_json_dump(body) + + return response + + return _wrapper + + return _inner + + +def put_has_empty_keys(field_updates, table): + if table: + key_names = table.attribute_keys + + # string/binary fields with empty string as value + empty_str_fields = [ + key + for (key, val) in field_updates.items() + if next(iter(val.keys())) in ["S", "B"] and next(iter(val.values())) == "" + ] + return any([keyname in empty_str_fields for keyname in key_names]) + return False + + +def get_empty_str_error(): + er = "com.amazonaws.dynamodb.v20111205#ValidationException" + return ( + 400, + {"server": "amazon.com"}, + dynamo_json_dump( + { + "__type": er, + "message": ( + "One or more parameter values were " + "invalid: An AttributeValue may not " + "contain an empty string" + ), + } + ), + ) class DynamoHandler(BaseResponse): @@ -17,9 +111,23 @@ def get_endpoint_name(self, headers): if match: return match.split(".")[1] - def error(self, type_, status=400): - return status, self.response_headers, dynamo_json_dump({"__type": type_}) + def error(self, type_, message, status=400): + return ( + status, + self.response_headers, + dynamo_json_dump({"__type": type_, "message": message}), + ) + @property + def dynamodb_backend(self): + """ + :return: DynamoDB2 Backend + :rtype: moto.dynamodb2.models.DynamoDBBackend + """ + return dynamodb_backends[self.region] + + @amz_crc32 + @amzn_request_id def call_action(self): self.body = json.loads(self.body or "{}") endpoint = self.get_endpoint_name(self.headers) @@ -38,192 +146,695 @@ def call_action(self): def list_tables(self): body = self.body - limit = body.get("Limit") - if body.get("ExclusiveStartTableName"): - last = body.get("ExclusiveStartTableName") - start = list(dynamodb_backend.tables.keys()).index(last) + 1 - else: - start = 0 - all_tables = list(dynamodb_backend.tables.keys()) - if limit: - tables = all_tables[start : start + limit] - else: - tables = all_tables[start:] + limit = body.get("Limit", 100) + exclusive_start_table_name = body.get("ExclusiveStartTableName") + tables, last_eval = self.dynamodb_backend.list_tables( + limit, exclusive_start_table_name + ) + response = {"TableNames": tables} - if limit and len(all_tables) > start + limit: - response["LastEvaluatedTableName"] = tables[-1] + if last_eval: + response["LastEvaluatedTableName"] = last_eval + return dynamo_json_dump(response) def create_table(self): body = self.body - name = body["TableName"] - + # get the table name + table_name = body["TableName"] + # check billing mode and get the throughput + if "BillingMode" in body.keys() and body["BillingMode"] == "PAY_PER_REQUEST": + if "ProvisionedThroughput" in body.keys(): + er = "com.amazonaws.dynamodb.v20111205#ValidationException" + return self.error( + er, + "ProvisionedThroughput cannot be specified when BillingMode is PAY_PER_REQUEST", + ) + throughput = None + billing_mode = "PAY_PER_REQUEST" + else: # Provisioned (default billing mode) + throughput = body.get("ProvisionedThroughput") + if throughput is None: + return self.error( + "ValidationException", + "One or more parameter values were invalid: ReadCapacityUnits and WriteCapacityUnits must both be specified when BillingMode is PROVISIONED", + ) + billing_mode = "PROVISIONED" + # getting ServerSideEncryption details + sse_spec = body.get("SSESpecification") + # getting the schema key_schema = body["KeySchema"] - hash_key = key_schema["HashKeyElement"] - hash_key_attr = hash_key["AttributeName"] - hash_key_type = hash_key["AttributeType"] + # getting attribute definition + attr = body["AttributeDefinitions"] + # getting the indexes + global_indexes = body.get("GlobalSecondaryIndexes") + if global_indexes == []: + return self.error( + "ValidationException", + "One or more parameter values were invalid: List of GlobalSecondaryIndexes is empty", + ) + global_indexes = global_indexes or [] + local_secondary_indexes = body.get("LocalSecondaryIndexes") + if local_secondary_indexes == []: + return self.error( + "ValidationException", + "One or more parameter values were invalid: List of LocalSecondaryIndexes is empty", + ) + local_secondary_indexes = local_secondary_indexes or [] + # Verify AttributeDefinitions list all + expected_attrs = [] + expected_attrs.extend([key["AttributeName"] for key in key_schema]) + expected_attrs.extend( + schema["AttributeName"] + for schema in itertools.chain( + *list(idx["KeySchema"] for idx in local_secondary_indexes) + ) + ) + expected_attrs.extend( + schema["AttributeName"] + for schema in itertools.chain( + *list(idx["KeySchema"] for idx in global_indexes) + ) + ) + expected_attrs = list(set(expected_attrs)) + expected_attrs.sort() + actual_attrs = [item["AttributeName"] for item in attr] + actual_attrs.sort() + if actual_attrs != expected_attrs: + return self._throw_attr_error( + actual_attrs, expected_attrs, global_indexes or local_secondary_indexes + ) + # get the stream specification + streams = body.get("StreamSpecification") + # Get any tags + tags = body.get("Tags", []) - range_key = key_schema.get("RangeKeyElement", {}) - range_key_attr = range_key.get("AttributeName") - range_key_type = range_key.get("AttributeType") + table = self.dynamodb_backend.create_table( + table_name, + schema=key_schema, + throughput=throughput, + attr=attr, + global_indexes=global_indexes, + indexes=local_secondary_indexes, + streams=streams, + billing_mode=billing_mode, + sse_specification=sse_spec, + tags=tags, + ) + if table is not None: + return dynamo_json_dump(table.describe()) + else: + er = "com.amazonaws.dynamodb.v20111205#ResourceInUseException" + return self.error(er, "Resource in use") - throughput = body["ProvisionedThroughput"] - read_units = throughput["ReadCapacityUnits"] - write_units = throughput["WriteCapacityUnits"] + def _throw_attr_error(self, actual_attrs, expected_attrs, indexes): + def dump_list(list_): + return str(list_).replace("'", "") - table = dynamodb_backend.create_table( - name, - hash_key_attr=hash_key_attr, - hash_key_type=hash_key_type, - range_key_attr=range_key_attr, - range_key_type=range_key_type, - read_capacity=int(read_units), - write_capacity=int(write_units), - ) - return dynamo_json_dump(table.describe) + er = "com.amazonaws.dynamodb.v20111205#ValidationException" + err_head = "One or more parameter values were invalid: " + if len(actual_attrs) > len(expected_attrs): + if indexes: + return self.error( + er, + err_head + + "Some AttributeDefinitions are not used. AttributeDefinitions: " + + dump_list(actual_attrs) + + ", keys used: " + + dump_list(expected_attrs), + ) + else: + return self.error( + er, + err_head + + "Number of attributes in KeySchema does not exactly match number of attributes defined in AttributeDefinitions", + ) + elif len(actual_attrs) < len(expected_attrs): + if indexes: + return self.error( + er, + err_head + + "Some index key attributes are not defined in AttributeDefinitions. Keys: " + + dump_list(list(set(expected_attrs) - set(actual_attrs))) + + ", AttributeDefinitions: " + + dump_list(actual_attrs), + ) + else: + return self.error( + er, "Invalid KeySchema: Some index key attribute have no definition" + ) + else: + if indexes: + return self.error( + er, + err_head + + "Some index key attributes are not defined in AttributeDefinitions. Keys: " + + dump_list(list(set(expected_attrs) - set(actual_attrs))) + + ", AttributeDefinitions: " + + dump_list(actual_attrs), + ) + else: + return self.error( + er, + err_head + + "Some index key attributes are not defined in AttributeDefinitions. Keys: " + + dump_list(expected_attrs) + + ", AttributeDefinitions: " + + dump_list(actual_attrs), + ) def delete_table(self): name = self.body["TableName"] - table = dynamodb_backend.delete_table(name) - if table: - return dynamo_json_dump(table.describe) + table = self.dynamodb_backend.delete_table(name) + if table is not None: + return dynamo_json_dump(table.describe()) else: er = "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException" - return self.error(er) + return self.error(er, "Requested resource not found") + + def describe_endpoints(self): + response = {"Endpoints": self.dynamodb_backend.describe_endpoints()} + return dynamo_json_dump(response) + + def tag_resource(self): + table_arn = self.body["ResourceArn"] + tags = self.body["Tags"] + self.dynamodb_backend.tag_resource(table_arn, tags) + return "" + + def untag_resource(self): + table_arn = self.body["ResourceArn"] + tags = self.body["TagKeys"] + self.dynamodb_backend.untag_resource(table_arn, tags) + return "" + + def list_tags_of_resource(self): + try: + table_arn = self.body["ResourceArn"] + all_tags = self.dynamodb_backend.list_tags_of_resource(table_arn) + all_tag_keys = [tag["Key"] for tag in all_tags] + marker = self.body.get("NextToken") + if marker: + start = all_tag_keys.index(marker) + 1 + else: + start = 0 + max_items = 10 # there is no default, but using 10 to make testing easier + tags_resp = all_tags[start : start + max_items] + next_marker = None + if len(all_tags) > start + max_items: + next_marker = tags_resp[-1]["Key"] + if next_marker: + return json.dumps({"Tags": tags_resp, "NextToken": next_marker}) + return json.dumps({"Tags": tags_resp}) + except AttributeError: + er = "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException" + return self.error(er, "Requested resource not found") def update_table(self): name = self.body["TableName"] - throughput = self.body["ProvisionedThroughput"] - new_read_units = throughput["ReadCapacityUnits"] - new_write_units = throughput["WriteCapacityUnits"] - table = dynamodb_backend.update_table_throughput( - name, new_read_units, new_write_units - ) - return dynamo_json_dump(table.describe) + attr_definitions = self.body.get("AttributeDefinitions", None) + global_index = self.body.get("GlobalSecondaryIndexUpdates", None) + throughput = self.body.get("ProvisionedThroughput", None) + billing_mode = self.body.get("BillingMode", None) + stream_spec = self.body.get("StreamSpecification", None) + try: + table = self.dynamodb_backend.update_table( + name=name, + attr_definitions=attr_definitions, + global_index=global_index, + throughput=throughput, + billing_mode=billing_mode, + stream_spec=stream_spec, + ) + return dynamo_json_dump(table.describe()) + except ValueError: + er = "com.amazonaws.dynamodb.v20111205#ResourceInUseException" + return self.error(er, "Cannot enable stream") def describe_table(self): name = self.body["TableName"] try: - table = dynamodb_backend.tables[name] + table = self.dynamodb_backend.describe_table(name) + return dynamo_json_dump(table) except KeyError: er = "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException" - return self.error(er) - return dynamo_json_dump(table.describe) + return self.error(er, "Requested resource not found") + @include_consumed_capacity() def put_item(self): name = self.body["TableName"] item = self.body["Item"] - result = dynamodb_backend.put_item(name, item) + return_values = self.body.get("ReturnValues", "NONE") + + if return_values not in ("ALL_OLD", "NONE"): + er = "com.amazonaws.dynamodb.v20111205#ValidationException" + return self.error(er, "Return values set to invalid value") + + if put_has_empty_keys(item, self.dynamodb_backend.get_table(name)): + return get_empty_str_error() + + overwrite = "Expected" not in self.body + if not overwrite: + expected = self.body["Expected"] + else: + expected = None + + if return_values == "ALL_OLD": + existing_item = self.dynamodb_backend.get_item(name, item) + if existing_item: + existing_attributes = existing_item.to_json()["Attributes"] + else: + existing_attributes = {} + + # Attempt to parse simple ConditionExpressions into an Expected + # expression + condition_expression = self.body.get("ConditionExpression") + expression_attribute_names = self.body.get("ExpressionAttributeNames", {}) + expression_attribute_values = self.body.get("ExpressionAttributeValues", {}) + + if condition_expression: + overwrite = False + + try: + result = self.dynamodb_backend.put_item( + name, + item, + expected, + condition_expression, + expression_attribute_names, + expression_attribute_values, + overwrite, + ) + except MockValidationException as mve: + er = "com.amazonaws.dynamodb.v20111205#ValidationException" + return self.error(er, mve.exception_msg) + except KeyError as ke: + er = "com.amazonaws.dynamodb.v20111205#ValidationException" + return self.error(er, ke.args[0]) + except ValueError as ve: + er = "com.amazonaws.dynamodb.v20111205#ConditionalCheckFailedException" + return self.error(er, str(ve)) + if result: item_dict = result.to_json() - item_dict["ConsumedCapacityUnits"] = 1 + if return_values == "ALL_OLD": + item_dict["Attributes"] = existing_attributes + else: + item_dict.pop("Attributes", None) return dynamo_json_dump(item_dict) else: er = "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException" - return self.error(er) + return self.error(er, "Requested resource not found") def batch_write_item(self): table_batches = self.body["RequestItems"] for table_name, table_requests in table_batches.items(): for table_request in table_requests: - request_type = list(table_request)[0] + request_type = list(table_request.keys())[0] request = list(table_request.values())[0] - if request_type == "PutRequest": item = request["Item"] - dynamodb_backend.put_item(table_name, item) + res = self.dynamodb_backend.put_item(table_name, item) + if not res: + return self.error( + "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException", + "Requested resource not found", + ) elif request_type == "DeleteRequest": - key = request["Key"] - hash_key = key["HashKeyElement"] - range_key = key.get("RangeKeyElement") - item = dynamodb_backend.delete_item(table_name, hash_key, range_key) + keys = request["Key"] + item = self.dynamodb_backend.delete_item(table_name, keys) response = { - "Responses": { - "Thread": {"ConsumedCapacityUnits": 1.0}, - "Reply": {"ConsumedCapacityUnits": 1.0}, - }, + "ConsumedCapacity": [ + { + "TableName": table_name, + "CapacityUnits": 1.0, + "Table": {"CapacityUnits": 1.0}, + } + for table_name, table_requests in table_batches.items() + ], + "ItemCollectionMetrics": {}, "UnprocessedItems": {}, } return dynamo_json_dump(response) + @include_consumed_capacity(0.5) def get_item(self): name = self.body["TableName"] + table = self.dynamodb_backend.get_table(name) + if table is None: + return self.error( + "com.amazonaws.dynamodb.v20120810#ResourceNotFoundException", + "Requested resource not found", + ) key = self.body["Key"] - hash_key = key["HashKeyElement"] - range_key = key.get("RangeKeyElement") - attrs_to_get = self.body.get("AttributesToGet") + projection_expression = self.body.get("ProjectionExpression") + expression_attribute_names = self.body.get("ExpressionAttributeNames") + if expression_attribute_names == {}: + if projection_expression is None: + er = "ValidationException" + return self.error( + er, + "ExpressionAttributeNames can only be specified when using expressions", + ) + else: + er = "ValidationException" + return self.error(er, "ExpressionAttributeNames must not be empty") + + expression_attribute_names = expression_attribute_names or {} + projection_expression = self._adjust_projection_expression( + projection_expression, expression_attribute_names + ) + try: - item = dynamodb_backend.get_item(name, hash_key, range_key) + item = self.dynamodb_backend.get_item(name, key, projection_expression) except ValueError: er = "com.amazon.coral.validate#ValidationException" - return self.error(er, status=400) + return self.error(er, "Validation Exception") if item: - item_dict = item.describe_attrs(attrs_to_get) - item_dict["ConsumedCapacityUnits"] = 0.5 + item_dict = item.describe_attrs(attributes=None) return dynamo_json_dump(item_dict) else: # Item not found - er = "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException" - return self.error(er, status=404) + return dynamo_json_dump({}) def batch_get_item(self): table_batches = self.body["RequestItems"] - results = {"Responses": {"UnprocessedKeys": {}}} + results = {"ConsumedCapacity": [], "Responses": {}, "UnprocessedKeys": {}} + + # Validation: Can only request up to 100 items at the same time + # Scenario 1: We're requesting more than a 100 keys from a single table + for table_name, table_request in table_batches.items(): + if len(table_request["Keys"]) > 100: + return self.error( + "com.amazonaws.dynamodb.v20111205#ValidationException", + "1 validation error detected: Value at 'requestItems." + + table_name + + ".member.keys' failed to satisfy constraint: Member must have length less than or equal to 100", + ) + # Scenario 2: We're requesting more than a 100 keys across all tables + nr_of_keys_across_all_tables = sum( + [len(req["Keys"]) for _, req in table_batches.items()] + ) + if nr_of_keys_across_all_tables > 100: + return self.error( + "com.amazonaws.dynamodb.v20111205#ValidationException", + "Too many items requested for the BatchGetItem call", + ) for table_name, table_request in table_batches.items(): - items = [] keys = table_request["Keys"] + if self._contains_duplicates(keys): + er = "com.amazon.coral.validate#ValidationException" + return self.error(er, "Provided list of item keys contains duplicates") attributes_to_get = table_request.get("AttributesToGet") + projection_expression = table_request.get("ProjectionExpression") + expression_attribute_names = table_request.get( + "ExpressionAttributeNames", {} + ) + + projection_expression = self._adjust_projection_expression( + projection_expression, expression_attribute_names + ) + + results["Responses"][table_name] = [] for key in keys: - hash_key = key["HashKeyElement"] - range_key = key.get("RangeKeyElement") - item = dynamodb_backend.get_item(table_name, hash_key, range_key) + try: + item = self.dynamodb_backend.get_item( + table_name, key, projection_expression + ) + except ValueError: + return self.error( + "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException", + "Requested resource not found", + ) if item: item_describe = item.describe_attrs(attributes_to_get) - items.append(item_describe) - results["Responses"][table_name] = { - "Items": items, - "ConsumedCapacityUnits": 1, - } + results["Responses"][table_name].append(item_describe["Item"]) + + results["ConsumedCapacity"].append( + {"CapacityUnits": len(keys), "TableName": table_name} + ) return dynamo_json_dump(results) + def _contains_duplicates(self, keys): + unique_keys = [] + for k in keys: + if k in unique_keys: + return True + else: + unique_keys.append(k) + return False + + @include_consumed_capacity() def query(self): name = self.body["TableName"] - hash_key = self.body["HashKeyValue"] - range_condition = self.body.get("RangeKeyCondition") - if range_condition: - range_comparison = range_condition["ComparisonOperator"] - range_values = range_condition["AttributeValueList"] - else: - range_comparison = None - range_values = [] + key_condition_expression = self.body.get("KeyConditionExpression") + projection_expression = self.body.get("ProjectionExpression") + expression_attribute_names = self.body.get("ExpressionAttributeNames", {}) + filter_expression = self.body.get("FilterExpression") + expression_attribute_values = self.body.get("ExpressionAttributeValues", {}) - items, _ = dynamodb_backend.query( - name, hash_key, range_comparison, range_values + projection_expression = self._adjust_projection_expression( + projection_expression, expression_attribute_names ) + filter_kwargs = {} + + if key_condition_expression: + value_alias_map = self.body.get("ExpressionAttributeValues", {}) + + table = self.dynamodb_backend.get_table(name) + + # If table does not exist + if table is None: + return self.error( + "com.amazonaws.dynamodb.v20120810#ResourceNotFoundException", + "Requested resource not found", + ) + + index_name = self.body.get("IndexName") + if index_name: + all_indexes = (table.global_indexes or []) + (table.indexes or []) + indexes_by_name = dict((i.name, i) for i in all_indexes) + if index_name not in indexes_by_name: + er = "com.amazonaws.dynamodb.v20120810#ResourceNotFoundException" + return self.error( + er, + "Invalid index: {} for table: {}. Available indexes are: {}".format( + index_name, name, ", ".join(indexes_by_name.keys()) + ), + ) + + index = indexes_by_name[index_name].schema + else: + index = table.schema + + reverse_attribute_lookup = dict( + (v, k) for k, v in self.body.get("ExpressionAttributeNames", {}).items() + ) + + if " and " in key_condition_expression.lower(): + expressions = re.split( + " AND ", key_condition_expression, maxsplit=1, flags=re.IGNORECASE + ) + + index_hash_key = [key for key in index if key["KeyType"] == "HASH"][0] + hash_key_var = reverse_attribute_lookup.get( + index_hash_key["AttributeName"], index_hash_key["AttributeName"] + ) + hash_key_regex = r"(^|[\s(]){0}\b".format(hash_key_var) + i, hash_key_expression = next( + ( + (i, e) + for i, e in enumerate(expressions) + if re.search(hash_key_regex, e) + ), + (None, None), + ) + if hash_key_expression is None: + return self.error( + "ValidationException", + "Query condition missed key schema element: {}".format( + hash_key_var + ), + ) + hash_key_expression = hash_key_expression.strip("()") + expressions.pop(i) + + # TODO implement more than one range expression and OR operators + range_key_expression = expressions[0].strip("()") + # Split expression, and account for all kinds of whitespacing around commas and brackets + range_key_expression_components = re.split( + r"\s*\(\s*|\s*,\s*|\s", range_key_expression + ) + # Skip whitespace + range_key_expression_components = [ + c for c in range_key_expression_components if c + ] + range_comparison = range_key_expression_components[1] + + if " and " in range_key_expression.lower(): + range_comparison = "BETWEEN" + # [range_key, between, x, and, y] + range_values = [ + value_alias_map[range_key_expression_components[2]], + value_alias_map[range_key_expression_components[4]], + ] + supplied_range_key = range_key_expression_components[0] + elif "begins_with" in range_key_expression: + range_comparison = "BEGINS_WITH" + # [begins_with, range_key, x] + range_values = [ + value_alias_map[range_key_expression_components[-1]] + ] + supplied_range_key = range_key_expression_components[1] + elif "begins_with" in range_key_expression.lower(): + function_used = range_key_expression[ + range_key_expression.lower().index("begins_with") : len( + "begins_with" + ) + ] + return self.error( + "com.amazonaws.dynamodb.v20111205#ValidationException", + "Invalid KeyConditionExpression: Invalid function name; function: {}".format( + function_used + ), + ) + else: + # [range_key, =, x] + range_values = [value_alias_map[range_key_expression_components[2]]] + supplied_range_key = range_key_expression_components[0] + + supplied_range_key = expression_attribute_names.get( + supplied_range_key, supplied_range_key + ) + range_keys = [ + k["AttributeName"] for k in index if k["KeyType"] == "RANGE" + ] + if supplied_range_key not in range_keys: + return self.error( + "ValidationException", + "Query condition missed key schema element: {}".format( + range_keys[0] + ), + ) + else: + hash_key_expression = key_condition_expression.strip("()") + range_comparison = None + range_values = [] + + if not re.search("[^<>]=", hash_key_expression): + return self.error( + "com.amazonaws.dynamodb.v20111205#ValidationException", + "Query key condition not supported", + ) + hash_key_value_alias = hash_key_expression.split("=")[1].strip() + # Temporary fix until we get proper KeyConditionExpression function + hash_key = value_alias_map.get( + hash_key_value_alias, {"S": hash_key_value_alias} + ) + else: + # 'KeyConditions': {u'forum_name': {u'ComparisonOperator': u'EQ', u'AttributeValueList': [{u'S': u'the-key'}]}} + key_conditions = self.body.get("KeyConditions") + query_filters = self.body.get("QueryFilter") + + if not (key_conditions or query_filters): + return self.error( + "com.amazonaws.dynamodb.v20111205#ValidationException", + "Either KeyConditions or QueryFilter should be present", + ) + + if key_conditions: + ( + hash_key_name, + range_key_name, + ) = self.dynamodb_backend.get_table_keys_name( + name, key_conditions.keys() + ) + for key, value in key_conditions.items(): + if key not in (hash_key_name, range_key_name): + filter_kwargs[key] = value + if hash_key_name is None: + er = "'com.amazonaws.dynamodb.v20120810#ResourceNotFoundException" + return self.error(er, "Requested resource not found") + hash_key = key_conditions[hash_key_name]["AttributeValueList"][0] + if len(key_conditions) == 1: + range_comparison = None + range_values = [] + else: + if range_key_name is None and not filter_kwargs: + er = "com.amazon.coral.validate#ValidationException" + return self.error(er, "Validation Exception") + else: + range_condition = key_conditions.get(range_key_name) + if range_condition: + range_comparison = range_condition["ComparisonOperator"] + range_values = range_condition["AttributeValueList"] + else: + range_comparison = None + range_values = [] + if query_filters: + filter_kwargs.update(query_filters) + index_name = self.body.get("IndexName") + exclusive_start_key = self.body.get("ExclusiveStartKey") + limit = self.body.get("Limit") + scan_index_forward = self.body.get("ScanIndexForward") + items, scanned_count, last_evaluated_key = self.dynamodb_backend.query( + name, + hash_key, + range_comparison, + range_values, + limit, + exclusive_start_key, + scan_index_forward, + projection_expression, + index_name=index_name, + expr_names=expression_attribute_names, + expr_values=expression_attribute_values, + filter_expression=filter_expression, + **filter_kwargs + ) if items is None: er = "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException" - return self.error(er) + return self.error(er, "Requested resource not found") result = { "Count": len(items), - "Items": [item.attrs for item in items], - "ConsumedCapacityUnits": 1, + "ScannedCount": scanned_count, } - # Implement this when we do pagination - # if not last_page: - # result["LastEvaluatedKey"] = { - # "HashKeyElement": items[-1].hash_key, - # "RangeKeyElement": items[-1].range_key, - # } + if self.body.get("Select", "").upper() != "COUNT": + result["Items"] = [item.attrs for item in items] + + if last_evaluated_key is not None: + result["LastEvaluatedKey"] = last_evaluated_key + return dynamo_json_dump(result) + def _adjust_projection_expression(self, projection_expression, expr_attr_names): + def _adjust(expression): + return ( + expr_attr_names[expression] + if expression in expr_attr_names + else expression + ) + + if projection_expression and expr_attr_names: + expressions = [x.strip() for x in projection_expression.split(",")] + return ",".join( + [ + ".".join([_adjust(expr) for expr in nested_expr.split(".")]) + for nested_expr in expressions + ] + ) + + return projection_expression + + @include_consumed_capacity() def scan(self): name = self.body["TableName"] @@ -236,59 +847,413 @@ def scan(self): comparison_values = scan_filter.get("AttributeValueList", []) filters[attribute_name] = (comparison_operator, comparison_values) - items, scanned_count, _ = dynamodb_backend.scan(name, filters) + filter_expression = self.body.get("FilterExpression") + expression_attribute_values = self.body.get("ExpressionAttributeValues", {}) + expression_attribute_names = self.body.get("ExpressionAttributeNames", {}) + projection_expression = self.body.get("ProjectionExpression", "") + exclusive_start_key = self.body.get("ExclusiveStartKey") + limit = self.body.get("Limit") + index_name = self.body.get("IndexName") + try: + items, scanned_count, last_evaluated_key = self.dynamodb_backend.scan( + name, + filters, + limit, + exclusive_start_key, + filter_expression, + expression_attribute_names, + expression_attribute_values, + index_name, + projection_expression, + ) + except InvalidIndexNameError as err: + er = "com.amazonaws.dynamodb.v20111205#ValidationException" + return self.error(er, str(err)) + except ValueError as err: + er = "com.amazonaws.dynamodb.v20111205#ValidationError" + return self.error(er, "Bad Filter Expression: {0}".format(err)) + except Exception as err: + er = "com.amazonaws.dynamodb.v20111205#InternalFailure" + return self.error(er, "Internal error. {0}".format(err)) + + # Items should be a list, at least an empty one. Is None if table does not exist. + # Should really check this at the beginning if items is None: er = "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException" - return self.error(er) + return self.error(er, "Requested resource not found") result = { "Count": len(items), - "Items": [item.attrs for item in items if item], - "ConsumedCapacityUnits": 1, + "Items": [item.attrs for item in items], "ScannedCount": scanned_count, } - - # Implement this when we do pagination - # if not last_page: - # result["LastEvaluatedKey"] = { - # "HashKeyElement": items[-1].hash_key, - # "RangeKeyElement": items[-1].range_key, - # } + if last_evaluated_key is not None: + result["LastEvaluatedKey"] = last_evaluated_key return dynamo_json_dump(result) def delete_item(self): name = self.body["TableName"] key = self.body["Key"] - hash_key = key["HashKeyElement"] - range_key = key.get("RangeKeyElement") - return_values = self.body.get("ReturnValues", "") - item = dynamodb_backend.delete_item(name, hash_key, range_key) - if item: - if return_values == "ALL_OLD": - item_dict = item.to_json() - else: - item_dict = {"Attributes": []} - item_dict["ConsumedCapacityUnits"] = 0.5 - return dynamo_json_dump(item_dict) + return_values = self.body.get("ReturnValues", "NONE") + if return_values not in ("ALL_OLD", "NONE"): + er = "com.amazonaws.dynamodb.v20111205#ValidationException" + return self.error(er, "Return values set to invalid value") + + table = self.dynamodb_backend.get_table(name) + if not table: + er = "com.amazonaws.dynamodb.v20120810#ConditionalCheckFailedException" + return self.error( + er, "A condition specified in the operation could not be evaluated." + ) + + # Attempt to parse simple ConditionExpressions into an Expected + # expression + condition_expression = self.body.get("ConditionExpression") + expression_attribute_names = self.body.get("ExpressionAttributeNames", {}) + expression_attribute_values = self.body.get("ExpressionAttributeValues", {}) + + try: + item = self.dynamodb_backend.delete_item( + name, + key, + expression_attribute_names, + expression_attribute_values, + condition_expression, + ) + except ValueError: + er = "com.amazonaws.dynamodb.v20111205#ConditionalCheckFailedException" + return self.error( + er, "A condition specified in the operation could not be evaluated." + ) + + if item and return_values == "ALL_OLD": + item_dict = item.to_json() else: - er = "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException" - return self.error(er) + item_dict = {"Attributes": {}} + item_dict["ConsumedCapacityUnits"] = 0.5 + return dynamo_json_dump(item_dict) def update_item(self): name = self.body["TableName"] key = self.body["Key"] - hash_key = key["HashKeyElement"] - range_key = key.get("RangeKeyElement") - updates = self.body["AttributeUpdates"] + return_values = self.body.get("ReturnValues", "NONE") + update_expression = self.body.get("UpdateExpression", "").strip() + attribute_updates = self.body.get("AttributeUpdates") + if update_expression and attribute_updates: + er = "com.amazonaws.dynamodb.v20111205#ValidationException" + return self.error( + er, + "Can not use both expression and non-expression parameters in the same request: Non-expression parameters: {AttributeUpdates} Expression parameters: {UpdateExpression}", + ) + # We need to copy the item in order to avoid it being modified by the update_item operation + try: + existing_item = copy.deepcopy(self.dynamodb_backend.get_item(name, key)) + except ValueError: + return self.error( + "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException", + "Requested resource not found", + ) + if existing_item: + existing_attributes = existing_item.to_json()["Attributes"] + else: + existing_attributes = {} - item = dynamodb_backend.update_item(name, hash_key, range_key, updates) + if return_values not in ( + "NONE", + "ALL_OLD", + "ALL_NEW", + "UPDATED_OLD", + "UPDATED_NEW", + ): + er = "com.amazonaws.dynamodb.v20111205#ValidationException" + return self.error(er, "Return values set to invalid value") - if item: - item_dict = item.to_json() - item_dict["ConsumedCapacityUnits"] = 0.5 + if "Expected" in self.body: + expected = self.body["Expected"] + else: + expected = None - return dynamo_json_dump(item_dict) + # Attempt to parse simple ConditionExpressions into an Expected + # expression + condition_expression = self.body.get("ConditionExpression") + expression_attribute_names = self.body.get("ExpressionAttributeNames", {}) + expression_attribute_values = self.body.get("ExpressionAttributeValues", {}) + + try: + item = self.dynamodb_backend.update_item( + name, + key, + update_expression=update_expression, + attribute_updates=attribute_updates, + expression_attribute_names=expression_attribute_names, + expression_attribute_values=expression_attribute_values, + expected=expected, + condition_expression=condition_expression, + ) + except MockValidationException as mve: + er = "com.amazonaws.dynamodb.v20111205#ValidationException" + return self.error(er, mve.exception_msg) + except ValueError: + er = "com.amazonaws.dynamodb.v20111205#ConditionalCheckFailedException" + return self.error(er, "The conditional request failed") + except TypeError: + er = "com.amazonaws.dynamodb.v20111205#ValidationException" + return self.error(er, "Validation Exception") + + item_dict = item.to_json() + item_dict["ConsumedCapacity"] = {"TableName": name, "CapacityUnits": 0.5} + unchanged_attributes = { + k + for k in existing_attributes.keys() + if existing_attributes[k] == item_dict["Attributes"].get(k) + } + changed_attributes = ( + set(existing_attributes.keys()) + .union(item_dict["Attributes"].keys()) + .difference(unchanged_attributes) + ) + + if return_values == "NONE": + item_dict["Attributes"] = {} + elif return_values == "ALL_OLD": + item_dict["Attributes"] = existing_attributes + elif return_values == "UPDATED_OLD": + item_dict["Attributes"] = { + k: v for k, v in existing_attributes.items() if k in changed_attributes + } + elif return_values == "UPDATED_NEW": + item_dict["Attributes"] = self._build_updated_new_attributes( + existing_attributes, item_dict["Attributes"] + ) + return dynamo_json_dump(item_dict) + + def _build_updated_new_attributes(self, original, changed): + if type(changed) != type(original): + return changed else: - er = "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException" - return self.error(er) + if type(changed) is dict: + return { + key: self._build_updated_new_attributes( + original.get(key, None), changed[key] + ) + for key in changed.keys() + if key not in original or changed[key] != original[key] + } + elif type(changed) in (set, list): + if len(changed) != len(original): + return changed + else: + return [ + self._build_updated_new_attributes( + original[index], changed[index] + ) + for index in range(len(changed)) + ] + else: + return changed + + def describe_limits(self): + return json.dumps( + { + "AccountMaxReadCapacityUnits": 20000, + "TableMaxWriteCapacityUnits": 10000, + "AccountMaxWriteCapacityUnits": 20000, + "TableMaxReadCapacityUnits": 10000, + } + ) + + def update_time_to_live(self): + name = self.body["TableName"] + ttl_spec = self.body["TimeToLiveSpecification"] + + self.dynamodb_backend.update_time_to_live(name, ttl_spec) + + return json.dumps({"TimeToLiveSpecification": ttl_spec}) + + def describe_time_to_live(self): + name = self.body["TableName"] + + ttl_spec = self.dynamodb_backend.describe_time_to_live(name) + + return json.dumps({"TimeToLiveDescription": ttl_spec}) + + def transact_get_items(self): + transact_items = self.body["TransactItems"] + responses = list() + + if len(transact_items) > TRANSACTION_MAX_ITEMS: + msg = "1 validation error detected: Value '[" + err_list = list() + request_id = 268435456 + for _ in transact_items: + request_id += 1 + hex_request_id = format(request_id, "x") + err_list.append( + "com.amazonaws.dynamodb.v20120810.TransactGetItem@%s" + % hex_request_id + ) + msg += ", ".join(err_list) + msg += ( + "'] at 'transactItems' failed to satisfy constraint: " + "Member must have length less than or equal to %s" + % TRANSACTION_MAX_ITEMS + ) + + return self.error("ValidationException", msg) + + ret_consumed_capacity = self.body.get("ReturnConsumedCapacity", "NONE") + consumed_capacity = dict() + + for transact_item in transact_items: + + table_name = transact_item["Get"]["TableName"] + key = transact_item["Get"]["Key"] + try: + item = self.dynamodb_backend.get_item(table_name, key) + except ValueError: + er = "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException" + return self.error(er, "Requested resource not found") + + if not item: + responses.append({}) + continue + + item_describe = item.describe_attrs(False) + responses.append(item_describe) + + table_capacity = consumed_capacity.get(table_name, {}) + table_capacity["TableName"] = table_name + capacity_units = table_capacity.get("CapacityUnits", 0) + 2.0 + table_capacity["CapacityUnits"] = capacity_units + read_capacity_units = table_capacity.get("ReadCapacityUnits", 0) + 2.0 + table_capacity["ReadCapacityUnits"] = read_capacity_units + consumed_capacity[table_name] = table_capacity + + if ret_consumed_capacity == "INDEXES": + table_capacity["Table"] = { + "CapacityUnits": capacity_units, + "ReadCapacityUnits": read_capacity_units, + } + + result = dict() + result.update({"Responses": responses}) + if ret_consumed_capacity != "NONE": + result.update({"ConsumedCapacity": [v for v in consumed_capacity.values()]}) + + return dynamo_json_dump(result) + + def transact_write_items(self): + transact_items = self.body["TransactItems"] + try: + self.dynamodb_backend.transact_write_items(transact_items) + except TransactionCanceledException as e: + er = "com.amazonaws.dynamodb.v20111205#TransactionCanceledException" + return self.error(er, str(e)) + except MockValidationException as mve: + er = "com.amazonaws.dynamodb.v20111205#ValidationException" + return self.error(er, mve.exception_msg) + response = {"ConsumedCapacity": [], "ItemCollectionMetrics": {}} + return dynamo_json_dump(response) + + def describe_continuous_backups(self): + name = self.body["TableName"] + + if self.dynamodb_backend.get_table(name) is None: + return self.error( + "com.amazonaws.dynamodb.v20111205#TableNotFoundException", + "Table not found: {}".format(name), + ) + + response = self.dynamodb_backend.describe_continuous_backups(name) + + return json.dumps({"ContinuousBackupsDescription": response}) + + def update_continuous_backups(self): + name = self.body["TableName"] + point_in_time_spec = self.body["PointInTimeRecoverySpecification"] + + if self.dynamodb_backend.get_table(name) is None: + return self.error( + "com.amazonaws.dynamodb.v20111205#TableNotFoundException", + "Table not found: {}".format(name), + ) + + response = self.dynamodb_backend.update_continuous_backups( + name, point_in_time_spec + ) + + return json.dumps({"ContinuousBackupsDescription": response}) + + def list_backups(self): + body = self.body + table_name = body.get("TableName") + backups = self.dynamodb_backend.list_backups(table_name) + response = {"BackupSummaries": [backup.summary for backup in backups]} + return dynamo_json_dump(response) + + def create_backup(self): + body = self.body + table_name = body.get("TableName") + backup_name = body.get("BackupName") + try: + backup = self.dynamodb_backend.create_backup(table_name, backup_name) + response = {"BackupDetails": backup.details} + return dynamo_json_dump(response) + except KeyError: + er = "com.amazonaws.dynamodb.v20111205#TableNotFoundException" + return self.error(er, "Table not found: %s" % table_name) + + def delete_backup(self): + body = self.body + backup_arn = body.get("BackupArn") + try: + backup = self.dynamodb_backend.delete_backup(backup_arn) + response = {"BackupDescription": backup.description} + return dynamo_json_dump(response) + except KeyError: + er = "com.amazonaws.dynamodb.v20111205#BackupNotFoundException" + return self.error(er, "Backup not found: %s" % backup_arn) + + def describe_backup(self): + body = self.body + backup_arn = body.get("BackupArn") + try: + backup = self.dynamodb_backend.describe_backup(backup_arn) + response = {"BackupDescription": backup.description} + return dynamo_json_dump(response) + except KeyError: + er = "com.amazonaws.dynamodb.v20111205#BackupNotFoundException" + return self.error(er, "Backup not found: %s" % backup_arn) + + def restore_table_from_backup(self): + body = self.body + target_table_name = body.get("TargetTableName") + backup_arn = body.get("BackupArn") + try: + restored_table = self.dynamodb_backend.restore_table_from_backup( + target_table_name, backup_arn + ) + return dynamo_json_dump(restored_table.describe()) + except KeyError: + er = "com.amazonaws.dynamodb.v20111205#BackupNotFoundException" + return self.error(er, "Backup not found: %s" % backup_arn) + except ValueError: + er = "com.amazonaws.dynamodb.v20111205#TableAlreadyExistsException" + return self.error(er, "Table already exists: %s" % target_table_name) + + def restore_table_to_point_in_time(self): + body = self.body + target_table_name = body.get("TargetTableName") + source_table_name = body.get("SourceTableName") + try: + restored_table = self.dynamodb_backend.restore_table_to_point_in_time( + target_table_name, source_table_name + ) + return dynamo_json_dump(restored_table.describe()) + except KeyError: + er = "com.amazonaws.dynamodb.v20111205#SourceTableNotFoundException" + return self.error(er, "Source table not found: %s" % source_table_name) + except ValueError: + er = "com.amazonaws.dynamodb.v20111205#TableAlreadyExistsException" + return self.error(er, "Table already exists: %s" % target_table_name) diff --git a/moto/dynamodb2/__init__.py b/moto/dynamodb2/__init__.py deleted file mode 100644 index 87e738e064b4..000000000000 --- a/moto/dynamodb2/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from moto.dynamodb2.models import dynamodb_backends as dynamodb_backends2 -from ..core.models import base_decorator - -dynamodb_backend2 = dynamodb_backends2["us-east-1"] -mock_dynamodb2 = base_decorator(dynamodb_backends2) diff --git a/moto/dynamodb2/comparisons.py b/moto/dynamodb2/comparisons.py deleted file mode 100644 index c6e0dd6b20d3..000000000000 --- a/moto/dynamodb2/comparisons.py +++ /dev/null @@ -1,1222 +0,0 @@ -import re -from collections import deque -from collections import namedtuple - -from moto.dynamodb2.exceptions import ConditionAttributeIsReservedKeyword -from moto.dynamodb2.parsing.reserved_keywords import ReservedKeywords - - -def get_filter_expression(expr, names, values): - """ - Parse a filter expression into an Op. - - Examples - expr = 'Id > 5 AND attribute_exists(test) AND Id BETWEEN 5 AND 6 OR length < 6 AND contains(test, 1) AND 5 IN (4,5, 6) OR (Id < 5 AND 5 > Id)' - expr = 'Id > 5 AND Subs < 7' - """ - parser = ConditionExpressionParser(expr, names, values) - return parser.parse() - - -def get_expected(expected): - """ - Parse a filter expression into an Op. - - Examples - expr = 'Id > 5 AND attribute_exists(test) AND Id BETWEEN 5 AND 6 OR length < 6 AND contains(test, 1) AND 5 IN (4,5, 6) OR (Id < 5 AND 5 > Id)' - expr = 'Id > 5 AND Subs < 7' - """ - ops = { - "EQ": OpEqual, - "NE": OpNotEqual, - "LE": OpLessThanOrEqual, - "LT": OpLessThan, - "GE": OpGreaterThanOrEqual, - "GT": OpGreaterThan, - "NOT_NULL": FuncAttrExists, - "NULL": FuncAttrNotExists, - "CONTAINS": FuncContains, - "NOT_CONTAINS": FuncNotContains, - "BEGINS_WITH": FuncBeginsWith, - "IN": FuncIn, - "BETWEEN": FuncBetween, - } - - # NOTE: Always uses ConditionalOperator=AND - conditions = [] - for key, cond in expected.items(): - path = AttributePath([key]) - if "Exists" in cond: - if cond["Exists"]: - conditions.append(FuncAttrExists(path)) - else: - conditions.append(FuncAttrNotExists(path)) - elif "Value" in cond: - conditions.append(OpEqual(path, AttributeValue(cond["Value"]))) - elif "ComparisonOperator" in cond: - operator_name = cond["ComparisonOperator"] - values = [AttributeValue(v) for v in cond.get("AttributeValueList", [])] - OpClass = ops[operator_name] - conditions.append(OpClass(path, *values)) - - # NOTE: Ignore ConditionalOperator - ConditionalOp = OpAnd - if conditions: - output = conditions[0] - for condition in conditions[1:]: - output = ConditionalOp(output, condition) - else: - return OpDefault(None, None) - - return output - - -class Op(object): - """ - Base class for a FilterExpression operator - """ - - OP = "" - - def __init__(self, lhs, rhs): - self.lhs = lhs - self.rhs = rhs - - def expr(self, item): - raise NotImplementedError("Expr not defined for {0}".format(type(self))) - - def __repr__(self): - return "({0} {1} {2})".format(self.lhs, self.OP, self.rhs) - - -# TODO add tests for all of these - -EQ_FUNCTION = lambda item_value, test_value: item_value == test_value # noqa -NE_FUNCTION = lambda item_value, test_value: item_value != test_value # noqa -LE_FUNCTION = lambda item_value, test_value: item_value <= test_value # noqa -LT_FUNCTION = lambda item_value, test_value: item_value < test_value # noqa -GE_FUNCTION = lambda item_value, test_value: item_value >= test_value # noqa -GT_FUNCTION = lambda item_value, test_value: item_value > test_value # noqa - -COMPARISON_FUNCS = { - "EQ": EQ_FUNCTION, - "=": EQ_FUNCTION, - "NE": NE_FUNCTION, - "!=": NE_FUNCTION, - "LE": LE_FUNCTION, - "<=": LE_FUNCTION, - "LT": LT_FUNCTION, - "<": LT_FUNCTION, - "GE": GE_FUNCTION, - ">=": GE_FUNCTION, - "GT": GT_FUNCTION, - ">": GT_FUNCTION, - # NULL means the value should not exist at all - "NULL": lambda item_value: False, - # NOT_NULL means the value merely has to exist, and values of None are valid - "NOT_NULL": lambda item_value: True, - "CONTAINS": lambda item_value, test_value: test_value in item_value, - "NOT_CONTAINS": lambda item_value, test_value: test_value not in item_value, - "BEGINS_WITH": lambda item_value, test_value: item_value.startswith(test_value), - "IN": lambda item_value, *test_values: item_value in test_values, - "BETWEEN": lambda item_value, lower_test_value, upper_test_value: lower_test_value - <= item_value - <= upper_test_value, -} - - -def get_comparison_func(range_comparison): - return COMPARISON_FUNCS.get(range_comparison) - - -class RecursionStopIteration(StopIteration): - pass - - -class ConditionExpressionParser: - def __init__( - self, - condition_expression, - expression_attribute_names, - expression_attribute_values, - ): - self.condition_expression = condition_expression - self.expression_attribute_names = expression_attribute_names - self.expression_attribute_values = expression_attribute_values - - def parse(self): - """Returns a syntax tree for the expression. - - The tree, and all of the nodes in the tree are a tuple of - - kind: str - - children/value: - list of nodes for parent nodes - value for leaf nodes - - Raises ValueError if the condition expression is invalid - Raises KeyError if expression attribute names/values are invalid - - Here are the types of nodes that can be returned. - The types of child nodes are denoted with a colon (:). - An arbitrary number of children is denoted with ... - - Condition: - ('OR', [lhs : Condition, rhs : Condition]) - ('AND', [lhs: Condition, rhs: Condition]) - ('NOT', [argument: Condition]) - ('PARENTHESES', [argument: Condition]) - ('FUNCTION', [('LITERAL', function_name: str), argument: Operand, ...]) - ('BETWEEN', [query: Operand, low: Operand, high: Operand]) - ('IN', [query: Operand, possible_value: Operand, ...]) - ('COMPARISON', [lhs: Operand, ('LITERAL', comparator: str), rhs: Operand]) - - Operand: - ('EXPRESSION_ATTRIBUTE_VALUE', value: dict, e.g. {'S': 'foobar'}) - ('PATH', [('LITERAL', path_element: str), ...]) - NOTE: Expression attribute names will be expanded - ('FUNCTION', [('LITERAL', 'size'), argument: Operand]) - - Literal: - ('LITERAL', value: str) - - """ - if not self.condition_expression: - return OpDefault(None, None) - nodes = self._lex_condition_expression() - nodes = self._parse_paths(nodes) - # NOTE: The docs say that functions should be parsed after - # IN, BETWEEN, and comparisons like <=. - # However, these expressions are invalid as function arguments, - # so it is okay to parse functions first. This needs to be done - # to interpret size() correctly as an operand. - nodes = self._apply_functions(nodes) - nodes = self._apply_comparator(nodes) - nodes = self._apply_in(nodes) - nodes = self._apply_between(nodes) - nodes = self._apply_parens_and_booleans(nodes) - node = nodes[0] - op = self._make_op_condition(node) - return op - - class Kind: - """Enum defining types of nodes in the syntax tree.""" - - # Condition nodes - # --------------- - OR = "OR" - AND = "AND" - NOT = "NOT" - PARENTHESES = "PARENTHESES" - FUNCTION = "FUNCTION" - BETWEEN = "BETWEEN" - IN = "IN" - COMPARISON = "COMPARISON" - - # Operand nodes - # ------------- - EXPRESSION_ATTRIBUTE_VALUE = "EXPRESSION_ATTRIBUTE_VALUE" - PATH = "PATH" - - # Literal nodes - # -------------- - LITERAL = "LITERAL" - - class Nonterminal: - """Enum defining nonterminals for productions.""" - - CONDITION = "CONDITION" - OPERAND = "OPERAND" - COMPARATOR = "COMPARATOR" - FUNCTION_NAME = "FUNCTION_NAME" - IDENTIFIER = "IDENTIFIER" - AND = "AND" - OR = "OR" - NOT = "NOT" - BETWEEN = "BETWEEN" - IN = "IN" - COMMA = "COMMA" - LEFT_PAREN = "LEFT_PAREN" - RIGHT_PAREN = "RIGHT_PAREN" - WHITESPACE = "WHITESPACE" - - Node = namedtuple("Node", ["nonterminal", "kind", "text", "value", "children"]) - - @classmethod - def raise_exception_if_keyword(cls, attribute): - if attribute.upper() in ReservedKeywords.get_reserved_keywords(): - raise ConditionAttributeIsReservedKeyword(attribute) - - def _lex_condition_expression(self): - nodes = deque() - remaining_expression = self.condition_expression - while remaining_expression: - node, remaining_expression = self._lex_one_node(remaining_expression) - if node.nonterminal == self.Nonterminal.WHITESPACE: - continue - nodes.append(node) - return nodes - - def _lex_one_node(self, remaining_expression): - # TODO: Handle indexing like [1] - attribute_regex = r"(:|#)?[A-z0-9\-_]+" - patterns = [ - (self.Nonterminal.WHITESPACE, re.compile(r"^ +")), - ( - self.Nonterminal.COMPARATOR, - re.compile( - "^(" - # Put long expressions first for greedy matching - "<>|" - "<=|" - ">=|" - "=|" - "<|" - ">)" - ), - ), - ( - self.Nonterminal.OPERAND, - re.compile( - r"^{attribute_regex}(\.{attribute_regex}|\[[0-9]\])*".format( - attribute_regex=attribute_regex - ) - ), - ), - (self.Nonterminal.COMMA, re.compile(r"^,")), - (self.Nonterminal.LEFT_PAREN, re.compile(r"^\(")), - (self.Nonterminal.RIGHT_PAREN, re.compile(r"^\)")), - ] - - for nonterminal, pattern in patterns: - match = pattern.match(remaining_expression) - if match: - match_text = match.group() - break - else: # pragma: no cover - raise ValueError( - "Cannot parse condition starting at:{}".format(remaining_expression) - ) - - node = self.Node( - nonterminal=nonterminal, - kind=self.Kind.LITERAL, - text=match_text, - value=match_text, - children=[], - ) - - remaining_expression = remaining_expression[len(match_text) :] - - return node, remaining_expression - - def _parse_paths(self, nodes): - output = deque() - - while nodes: - node = nodes.popleft() - - if node.nonterminal == self.Nonterminal.OPERAND: - path = node.value.replace("[", ".[").split(".") - children = [self._parse_path_element(name) for name in path] - if len(children) == 1: - child = children[0] - if child.nonterminal != self.Nonterminal.IDENTIFIER: - output.append(child) - continue - else: - for child in children: - self._assert( - child.nonterminal == self.Nonterminal.IDENTIFIER, - "Cannot use {} in path".format(child.text), - [node], - ) - output.append( - self.Node( - nonterminal=self.Nonterminal.OPERAND, - kind=self.Kind.PATH, - text=node.text, - value=None, - children=children, - ) - ) - else: - output.append(node) - return output - - def _parse_path_element(self, name): - reserved = { - "and": self.Nonterminal.AND, - "or": self.Nonterminal.OR, - "in": self.Nonterminal.IN, - "between": self.Nonterminal.BETWEEN, - "not": self.Nonterminal.NOT, - } - - functions = { - "attribute_exists", - "attribute_not_exists", - "attribute_type", - "begins_with", - "contains", - "size", - } - - if name.lower() in reserved: - # e.g. AND - nonterminal = reserved[name.lower()] - return self.Node( - nonterminal=nonterminal, - kind=self.Kind.LITERAL, - text=name, - value=name, - children=[], - ) - elif name in functions: - # e.g. attribute_exists - return self.Node( - nonterminal=self.Nonterminal.FUNCTION_NAME, - kind=self.Kind.LITERAL, - text=name, - value=name, - children=[], - ) - elif name.startswith(":"): - # e.g. :value0 - return self.Node( - nonterminal=self.Nonterminal.OPERAND, - kind=self.Kind.EXPRESSION_ATTRIBUTE_VALUE, - text=name, - value=self._lookup_expression_attribute_value(name), - children=[], - ) - elif name.startswith("#"): - # e.g. #name0 - return self.Node( - nonterminal=self.Nonterminal.IDENTIFIER, - kind=self.Kind.LITERAL, - text=name, - value=self._lookup_expression_attribute_name(name), - children=[], - ) - elif name.startswith("["): - # e.g. [123] - if not name.endswith("]"): # pragma: no cover - raise ValueError("Bad path element {}".format(name)) - return self.Node( - nonterminal=self.Nonterminal.IDENTIFIER, - kind=self.Kind.LITERAL, - text=name, - value=int(name[1:-1]), - children=[], - ) - else: - # e.g. ItemId - self.raise_exception_if_keyword(name) - return self.Node( - nonterminal=self.Nonterminal.IDENTIFIER, - kind=self.Kind.LITERAL, - text=name, - value=name, - children=[], - ) - - def _lookup_expression_attribute_value(self, name): - return self.expression_attribute_values[name] - - def _lookup_expression_attribute_name(self, name): - return self.expression_attribute_names[name] - - # NOTE: The following constructions are ordered from high precedence to low precedence - # according to - # https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.OperatorsAndFunctions.html#Expressions.OperatorsAndFunctions.Precedence - # - # = <> < <= > >= - # IN - # BETWEEN - # attribute_exists attribute_not_exists begins_with contains - # Parentheses - # NOT - # AND - # OR - # - # The grammar is taken from - # https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.OperatorsAndFunctions.html#Expressions.OperatorsAndFunctions.Syntax - # - # condition-expression ::= - # operand comparator operand - # operand BETWEEN operand AND operand - # operand IN ( operand (',' operand (, ...) )) - # function - # condition AND condition - # condition OR condition - # NOT condition - # ( condition ) - # - # comparator ::= - # = - # <> - # < - # <= - # > - # >= - # - # function ::= - # attribute_exists (path) - # attribute_not_exists (path) - # attribute_type (path, type) - # begins_with (path, substr) - # contains (path, operand) - # size (path) - - def _matches(self, nodes, production): - """Check if the nodes start with the given production. - - Parameters - ---------- - nodes: list of Node - production: list of str - The name of a Nonterminal, or '*' for anything - - """ - if len(nodes) < len(production): - return False - for i in range(len(production)): - if production[i] == "*": - continue - expected = getattr(self.Nonterminal, production[i]) - if nodes[i].nonterminal != expected: - return False - return True - - def _apply_comparator(self, nodes): - """Apply condition := operand comparator operand.""" - output = deque() - - while nodes: - if self._matches(nodes, ["*", "COMPARATOR"]): - self._assert( - self._matches(nodes, ["OPERAND", "COMPARATOR", "OPERAND"]), - "Bad comparison", - list(nodes)[:3], - ) - lhs = nodes.popleft() - comparator = nodes.popleft() - rhs = nodes.popleft() - nodes.appendleft( - self.Node( - nonterminal=self.Nonterminal.CONDITION, - kind=self.Kind.COMPARISON, - text=" ".join([lhs.text, comparator.text, rhs.text]), - value=None, - children=[lhs, comparator, rhs], - ) - ) - else: - output.append(nodes.popleft()) - return output - - def _apply_in(self, nodes): - """Apply condition := operand IN ( operand , ... ).""" - output = deque() - while nodes: - if self._matches(nodes, ["*", "IN"]): - self._assert( - self._matches(nodes, ["OPERAND", "IN", "LEFT_PAREN"]), - "Bad IN expression", - list(nodes)[:3], - ) - lhs = nodes.popleft() - in_node = nodes.popleft() - left_paren = nodes.popleft() - all_children = [lhs, in_node, left_paren] - rhs = [] - while True: - if self._matches(nodes, ["OPERAND", "COMMA"]): - operand = nodes.popleft() - separator = nodes.popleft() - all_children += [operand, separator] - rhs.append(operand) - elif self._matches(nodes, ["OPERAND", "RIGHT_PAREN"]): - operand = nodes.popleft() - separator = nodes.popleft() - all_children += [operand, separator] - rhs.append(operand) - break # Close - else: - self._assert(False, "Bad IN expression starting at", nodes) - nodes.appendleft( - self.Node( - nonterminal=self.Nonterminal.CONDITION, - kind=self.Kind.IN, - text=" ".join([t.text for t in all_children]), - value=None, - children=[lhs] + rhs, - ) - ) - else: - output.append(nodes.popleft()) - return output - - def _apply_between(self, nodes): - """Apply condition := operand BETWEEN operand AND operand.""" - output = deque() - while nodes: - if self._matches(nodes, ["*", "BETWEEN"]): - self._assert( - self._matches( - nodes, ["OPERAND", "BETWEEN", "OPERAND", "AND", "OPERAND"] - ), - "Bad BETWEEN expression", - list(nodes)[:5], - ) - lhs = nodes.popleft() - between_node = nodes.popleft() - low = nodes.popleft() - and_node = nodes.popleft() - high = nodes.popleft() - all_children = [lhs, between_node, low, and_node, high] - nodes.appendleft( - self.Node( - nonterminal=self.Nonterminal.CONDITION, - kind=self.Kind.BETWEEN, - text=" ".join([t.text for t in all_children]), - value=None, - children=[lhs, low, high], - ) - ) - else: - output.append(nodes.popleft()) - return output - - def _apply_functions(self, nodes): - """Apply condition := function_name (operand , ...).""" - output = deque() - either_kind = {self.Kind.PATH, self.Kind.EXPRESSION_ATTRIBUTE_VALUE} - expected_argument_kind_map = { - "attribute_exists": [{self.Kind.PATH}], - "attribute_not_exists": [{self.Kind.PATH}], - "attribute_type": [either_kind, {self.Kind.EXPRESSION_ATTRIBUTE_VALUE}], - "begins_with": [either_kind, either_kind], - "contains": [either_kind, either_kind], - "size": [{self.Kind.PATH}], - } - while nodes: - if self._matches(nodes, ["FUNCTION_NAME"]): - self._assert( - self._matches( - nodes, ["FUNCTION_NAME", "LEFT_PAREN", "OPERAND", "*"] - ), - "Bad function expression at", - list(nodes)[:4], - ) - function_name = nodes.popleft() - left_paren = nodes.popleft() - all_children = [function_name, left_paren] - arguments = [] - while True: - if self._matches(nodes, ["OPERAND", "COMMA"]): - operand = nodes.popleft() - separator = nodes.popleft() - all_children += [operand, separator] - arguments.append(operand) - elif self._matches(nodes, ["OPERAND", "RIGHT_PAREN"]): - operand = nodes.popleft() - separator = nodes.popleft() - all_children += [operand, separator] - arguments.append(operand) - break # Close paren - else: - self._assert( - False, - "Bad function expression", - all_children + list(nodes)[:2], - ) - expected_kinds = expected_argument_kind_map[function_name.value] - self._assert( - len(arguments) == len(expected_kinds), - "Wrong number of arguments in", - all_children, - ) - for i in range(len(expected_kinds)): - self._assert( - arguments[i].kind in expected_kinds[i], - "Wrong type for argument %d in" % i, - all_children, - ) - if function_name.value == "size": - nonterminal = self.Nonterminal.OPERAND - else: - nonterminal = self.Nonterminal.CONDITION - nodes.appendleft( - self.Node( - nonterminal=nonterminal, - kind=self.Kind.FUNCTION, - text=" ".join([t.text for t in all_children]), - value=None, - children=[function_name] + arguments, - ) - ) - else: - output.append(nodes.popleft()) - return output - - def _apply_parens_and_booleans(self, nodes, left_paren=None): - """Apply condition := ( condition ) and booleans.""" - output = deque() - while nodes: - if self._matches(nodes, ["LEFT_PAREN"]): - parsed = self._apply_parens_and_booleans( - nodes, left_paren=nodes.popleft() - ) - self._assert(len(parsed) >= 1, "Failed to close parentheses at", nodes) - parens = parsed.popleft() - self._assert( - parens.kind == self.Kind.PARENTHESES, - "Failed to close parentheses at", - nodes, - ) - output.append(parens) - nodes = parsed - elif self._matches(nodes, ["RIGHT_PAREN"]): - self._assert(left_paren is not None, "Unmatched ) at", nodes) - close_paren = nodes.popleft() - children = self._apply_booleans(output) - all_children = [left_paren] + list(children) + [close_paren] - return deque( - [ - self.Node( - nonterminal=self.Nonterminal.CONDITION, - kind=self.Kind.PARENTHESES, - text=" ".join([t.text for t in all_children]), - value=None, - children=list(children), - ) - ] - + list(nodes) - ) - else: - output.append(nodes.popleft()) - - self._assert(left_paren is None, "Unmatched ( at", list(output)) - return self._apply_booleans(output) - - def _apply_booleans(self, nodes): - """Apply and, or, and not constructions.""" - nodes = self._apply_not(nodes) - nodes = self._apply_and(nodes) - nodes = self._apply_or(nodes) - # The expression should reduce to a single condition - self._assert(len(nodes) == 1, "Unexpected expression at", list(nodes)[1:]) - self._assert( - nodes[0].nonterminal == self.Nonterminal.CONDITION, - "Incomplete condition", - nodes, - ) - return nodes - - def _apply_not(self, nodes): - """Apply condition := NOT condition.""" - output = deque() - while nodes: - if self._matches(nodes, ["NOT"]): - self._assert( - self._matches(nodes, ["NOT", "CONDITION"]), - "Bad NOT expression", - list(nodes)[:2], - ) - not_node = nodes.popleft() - child = nodes.popleft() - nodes.appendleft( - self.Node( - nonterminal=self.Nonterminal.CONDITION, - kind=self.Kind.NOT, - text=" ".join([not_node.text, child.text]), - value=None, - children=[child], - ) - ) - else: - output.append(nodes.popleft()) - - return output - - def _apply_and(self, nodes): - """Apply condition := condition AND condition.""" - output = deque() - while nodes: - if self._matches(nodes, ["*", "AND"]): - self._assert( - self._matches(nodes, ["CONDITION", "AND", "CONDITION"]), - "Bad AND expression", - list(nodes)[:3], - ) - lhs = nodes.popleft() - and_node = nodes.popleft() - rhs = nodes.popleft() - all_children = [lhs, and_node, rhs] - nodes.appendleft( - self.Node( - nonterminal=self.Nonterminal.CONDITION, - kind=self.Kind.AND, - text=" ".join([t.text for t in all_children]), - value=None, - children=[lhs, rhs], - ) - ) - else: - output.append(nodes.popleft()) - - return output - - def _apply_or(self, nodes): - """Apply condition := condition OR condition.""" - output = deque() - while nodes: - if self._matches(nodes, ["*", "OR"]): - self._assert( - self._matches(nodes, ["CONDITION", "OR", "CONDITION"]), - "Bad OR expression", - list(nodes)[:3], - ) - lhs = nodes.popleft() - or_node = nodes.popleft() - rhs = nodes.popleft() - all_children = [lhs, or_node, rhs] - nodes.appendleft( - self.Node( - nonterminal=self.Nonterminal.CONDITION, - kind=self.Kind.OR, - text=" ".join([t.text for t in all_children]), - value=None, - children=[lhs, rhs], - ) - ) - else: - output.append(nodes.popleft()) - - return output - - def _make_operand(self, node): - if node.kind == self.Kind.PATH: - return AttributePath([child.value for child in node.children]) - elif node.kind == self.Kind.EXPRESSION_ATTRIBUTE_VALUE: - return AttributeValue(node.value) - elif node.kind == self.Kind.FUNCTION: - # size() - function_node = node.children[0] - arguments = node.children[1:] - function_name = function_node.value - arguments = [self._make_operand(arg) for arg in arguments] - return FUNC_CLASS[function_name](*arguments) - else: # pragma: no cover - raise ValueError("Unknown operand: %r" % node) - - def _make_op_condition(self, node): - if node.kind == self.Kind.OR: - lhs, rhs = node.children - return OpOr(self._make_op_condition(lhs), self._make_op_condition(rhs)) - elif node.kind == self.Kind.AND: - lhs, rhs = node.children - return OpAnd(self._make_op_condition(lhs), self._make_op_condition(rhs)) - elif node.kind == self.Kind.NOT: - (child,) = node.children - return OpNot(self._make_op_condition(child)) - elif node.kind == self.Kind.PARENTHESES: - (child,) = node.children - return self._make_op_condition(child) - elif node.kind == self.Kind.FUNCTION: - function_node = node.children[0] - arguments = node.children[1:] - function_name = function_node.value - arguments = [self._make_operand(arg) for arg in arguments] - return FUNC_CLASS[function_name](*arguments) - elif node.kind == self.Kind.BETWEEN: - query, low, high = node.children - return FuncBetween( - self._make_operand(query), - self._make_operand(low), - self._make_operand(high), - ) - elif node.kind == self.Kind.IN: - query = node.children[0] - possible_values = node.children[1:] - query = self._make_operand(query) - possible_values = [self._make_operand(v) for v in possible_values] - return FuncIn(query, *possible_values) - elif node.kind == self.Kind.COMPARISON: - lhs, comparator, rhs = node.children - return COMPARATOR_CLASS[comparator.value]( - self._make_operand(lhs), self._make_operand(rhs) - ) - else: # pragma: no cover - raise ValueError("Unknown expression node kind %r" % node.kind) - - def _assert(self, condition, message, nodes): - if not condition: - raise ValueError(message + " " + " ".join([t.text for t in nodes])) - - -class Operand(object): - def expr(self, item): - raise NotImplementedError - - def get_type(self, item): - raise NotImplementedError - - -class AttributePath(Operand): - def __init__(self, path): - """Initialize the AttributePath. - - Parameters - ---------- - path: list of int/str - - """ - assert len(path) >= 1 - self.path = path - - def _get_attr(self, item): - if item is None: - return None - - base = self.path[0] - if base not in item.attrs: - return None - attr = item.attrs[base] - - for name in self.path[1:]: - attr = attr.child_attr(name) - if attr is None: - return None - - return attr - - def expr(self, item): - attr = self._get_attr(item) - if attr is None: - return None - else: - return attr.cast_value - - def get_type(self, item): - attr = self._get_attr(item) - if attr is None: - return None - else: - return attr.type - - def __repr__(self): - return ".".join(self.path) - - -class AttributeValue(Operand): - def __init__(self, value): - """Initialize the AttributePath. - - Parameters - ---------- - value: dict - e.g. {'N': '1.234'} - - """ - self.type = list(value.keys())[0] - self.value = value[self.type] - - def expr(self, item): - # TODO: Reuse DynamoType code - if self.type == "N": - try: - return int(self.value) - except ValueError: - return float(self.value) - elif self.type in ["SS", "NS", "BS"]: - sub_type = self.type[0] - return set([AttributeValue({sub_type: v}).expr(item) for v in self.value]) - elif self.type == "L": - return [AttributeValue(v).expr(item) for v in self.value] - elif self.type == "M": - return dict( - [(k, AttributeValue(v).expr(item)) for k, v in self.value.items()] - ) - else: - return self.value - return self.value - - def get_type(self, item): - return self.type - - def __repr__(self): - return repr(self.value) - - -class OpDefault(Op): - OP = "NONE" - - def expr(self, item): - """If no condition is specified, always True.""" - return True - - -class OpNot(Op): - OP = "NOT" - - def __init__(self, lhs): - super().__init__(lhs, None) - - def expr(self, item): - lhs = self.lhs.expr(item) - return not lhs - - def __str__(self): - return "({0} {1})".format(self.OP, self.lhs) - - -class OpAnd(Op): - OP = "AND" - - def expr(self, item): - lhs = self.lhs.expr(item) - return lhs and self.rhs.expr(item) - - -class OpLessThan(Op): - OP = "<" - - def expr(self, item): - lhs = self.lhs.expr(item) - rhs = self.rhs.expr(item) - # In python3 None is not a valid comparator when using < or > so must be handled specially - if lhs is not None and rhs is not None: - return lhs < rhs - else: - return False - - -class OpGreaterThan(Op): - OP = ">" - - def expr(self, item): - lhs = self.lhs.expr(item) - rhs = self.rhs.expr(item) - # In python3 None is not a valid comparator when using < or > so must be handled specially - if lhs is not None and rhs is not None: - return lhs > rhs - else: - return False - - -class OpEqual(Op): - OP = "=" - - def expr(self, item): - lhs = self.lhs.expr(item) - rhs = self.rhs.expr(item) - return lhs == rhs - - -class OpNotEqual(Op): - OP = "<>" - - def expr(self, item): - lhs = self.lhs.expr(item) - rhs = self.rhs.expr(item) - return lhs != rhs - - -class OpLessThanOrEqual(Op): - OP = "<=" - - def expr(self, item): - lhs = self.lhs.expr(item) - rhs = self.rhs.expr(item) - # In python3 None is not a valid comparator when using < or > so must be handled specially - if lhs is not None and rhs is not None: - return lhs <= rhs - else: - return False - - -class OpGreaterThanOrEqual(Op): - OP = ">=" - - def expr(self, item): - lhs = self.lhs.expr(item) - rhs = self.rhs.expr(item) - # In python3 None is not a valid comparator when using < or > so must be handled specially - if lhs is not None and rhs is not None: - return lhs >= rhs - else: - return False - - -class OpOr(Op): - OP = "OR" - - def expr(self, item): - lhs = self.lhs.expr(item) - return lhs or self.rhs.expr(item) - - -class Func(object): - """ - Base class for a FilterExpression function - """ - - FUNC = "Unknown" - - def __init__(self, *arguments): - self.arguments = arguments - - def expr(self, item): - raise NotImplementedError - - def __repr__(self): - return "{0}({1})".format( - self.FUNC, " ".join([repr(arg) for arg in self.arguments]) - ) - - -class FuncAttrExists(Func): - FUNC = "attribute_exists" - - def __init__(self, attribute): - self.attr = attribute - super().__init__(attribute) - - def expr(self, item): - return self.attr.get_type(item) is not None - - -def FuncAttrNotExists(attribute): - return OpNot(FuncAttrExists(attribute)) - - -class FuncAttrType(Func): - FUNC = "attribute_type" - - def __init__(self, attribute, _type): - self.attr = attribute - self.type = _type - super().__init__(attribute, _type) - - def expr(self, item): - return self.attr.get_type(item) == self.type.expr(item) - - -class FuncBeginsWith(Func): - FUNC = "begins_with" - - def __init__(self, attribute, substr): - self.attr = attribute - self.substr = substr - super().__init__(attribute, substr) - - def expr(self, item): - if self.attr.get_type(item) != "S": - return False - if self.substr.get_type(item) != "S": - return False - return self.attr.expr(item).startswith(self.substr.expr(item)) - - -class FuncContains(Func): - FUNC = "contains" - - def __init__(self, attribute, operand): - self.attr = attribute - self.operand = operand - super().__init__(attribute, operand) - - def expr(self, item): - if self.attr.get_type(item) in ("S", "SS", "NS", "BS", "L"): - try: - return self.operand.expr(item) in self.attr.expr(item) - except TypeError: - return False - return False - - -def FuncNotContains(attribute, operand): - return OpNot(FuncContains(attribute, operand)) - - -class FuncSize(Func): - FUNC = "size" - - def __init__(self, attribute): - self.attr = attribute - super().__init__(attribute) - - def expr(self, item): - if self.attr.get_type(item) is None: - raise ValueError("Invalid attribute name {0}".format(self.attr)) - - if self.attr.get_type(item) in ("S", "SS", "NS", "B", "BS", "L", "M"): - return len(self.attr.expr(item)) - raise ValueError("Invalid filter expression") - - -class FuncBetween(Func): - FUNC = "BETWEEN" - - def __init__(self, attribute, start, end): - self.attr = attribute - self.start = start - self.end = end - super().__init__(attribute, start, end) - - def expr(self, item): - # In python3 None is not a valid comparator when using < or > so must be handled specially - start = self.start.expr(item) - attr = self.attr.expr(item) - end = self.end.expr(item) - # Need to verify whether start has a valid value - # Can't just check 'if start', because start could be 0, which is a valid integer - start_has_value = start is not None and (isinstance(start, int) or start) - end_has_value = end is not None and (isinstance(end, int) or end) - if start_has_value and attr and end_has_value: - return start <= attr <= end - elif start is None and attr is None: - # None is between None and None as well as None is between None and any number - return True - elif start is None and attr and end: - return attr <= end - else: - return False - - -class FuncIn(Func): - FUNC = "IN" - - def __init__(self, attribute, *possible_values): - self.attr = attribute - self.possible_values = possible_values - super().__init__(attribute, *possible_values) - - def expr(self, item): - for possible_value in self.possible_values: - if self.attr.expr(item) == possible_value.expr(item): - return True - - return False - - -COMPARATOR_CLASS = { - "<": OpLessThan, - ">": OpGreaterThan, - "<=": OpLessThanOrEqual, - ">=": OpGreaterThanOrEqual, - "=": OpEqual, - "<>": OpNotEqual, -} - -FUNC_CLASS = { - "attribute_exists": FuncAttrExists, - "attribute_not_exists": FuncAttrNotExists, - "attribute_type": FuncAttrType, - "begins_with": FuncBeginsWith, - "contains": FuncContains, - "size": FuncSize, - "between": FuncBetween, -} diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py deleted file mode 100644 index 3e0b36aabdac..000000000000 --- a/moto/dynamodb2/responses.py +++ /dev/null @@ -1,1259 +0,0 @@ -import copy -import json -import re - -import itertools -from functools import wraps - -from moto.core.responses import BaseResponse -from moto.core.utils import camelcase_to_underscores, amz_crc32, amzn_request_id -from .exceptions import ( - InvalidIndexNameError, - MockValidationException, - TransactionCanceledException, -) -from moto.dynamodb2.models import dynamodb_backends, dynamo_json_dump - - -TRANSACTION_MAX_ITEMS = 25 - - -def include_consumed_capacity(val=1.0): - def _inner(f): - @wraps(f) - def _wrapper(*args, **kwargs): - (handler,) = args - expected_capacity = handler.body.get("ReturnConsumedCapacity", "NONE") - if expected_capacity not in ["NONE", "TOTAL", "INDEXES"]: - type_ = "ValidationException" - message = "1 validation error detected: Value '{}' at 'returnConsumedCapacity' failed to satisfy constraint: Member must satisfy enum value set: [INDEXES, TOTAL, NONE]".format( - expected_capacity - ) - return ( - 400, - handler.response_headers, - dynamo_json_dump({"__type": type_, "message": message}), - ) - table_name = handler.body.get("TableName", "") - index_name = handler.body.get("IndexName", None) - - response = f(*args, **kwargs) - - if isinstance(response, str): - body = json.loads(response) - - if expected_capacity == "TOTAL": - body["ConsumedCapacity"] = { - "TableName": table_name, - "CapacityUnits": val, - } - elif expected_capacity == "INDEXES": - body["ConsumedCapacity"] = { - "TableName": table_name, - "CapacityUnits": val, - "Table": {"CapacityUnits": val}, - } - if index_name: - body["ConsumedCapacity"]["LocalSecondaryIndexes"] = { - index_name: {"CapacityUnits": val} - } - - return dynamo_json_dump(body) - - return response - - return _wrapper - - return _inner - - -def put_has_empty_keys(field_updates, table): - if table: - key_names = table.attribute_keys - - # string/binary fields with empty string as value - empty_str_fields = [ - key - for (key, val) in field_updates.items() - if next(iter(val.keys())) in ["S", "B"] and next(iter(val.values())) == "" - ] - return any([keyname in empty_str_fields for keyname in key_names]) - return False - - -def get_empty_str_error(): - er = "com.amazonaws.dynamodb.v20111205#ValidationException" - return ( - 400, - {"server": "amazon.com"}, - dynamo_json_dump( - { - "__type": er, - "message": ( - "One or more parameter values were " - "invalid: An AttributeValue may not " - "contain an empty string" - ), - } - ), - ) - - -class DynamoHandler(BaseResponse): - def get_endpoint_name(self, headers): - """Parses request headers and extracts part od the X-Amz-Target - that corresponds to a method of DynamoHandler - - ie: X-Amz-Target: DynamoDB_20111205.ListTables -> ListTables - """ - # Headers are case-insensitive. Probably a better way to do this. - match = headers.get("x-amz-target") or headers.get("X-Amz-Target") - if match: - return match.split(".")[1] - - def error(self, type_, message, status=400): - return ( - status, - self.response_headers, - dynamo_json_dump({"__type": type_, "message": message}), - ) - - @property - def dynamodb_backend(self): - """ - :return: DynamoDB2 Backend - :rtype: moto.dynamodb2.models.DynamoDBBackend - """ - return dynamodb_backends[self.region] - - @amz_crc32 - @amzn_request_id - def call_action(self): - self.body = json.loads(self.body or "{}") - endpoint = self.get_endpoint_name(self.headers) - if endpoint: - endpoint = camelcase_to_underscores(endpoint) - response = getattr(self, endpoint)() - if isinstance(response, str): - return 200, self.response_headers, response - - else: - status_code, new_headers, response_content = response - self.response_headers.update(new_headers) - return status_code, self.response_headers, response_content - else: - return 404, self.response_headers, "" - - def list_tables(self): - body = self.body - limit = body.get("Limit", 100) - exclusive_start_table_name = body.get("ExclusiveStartTableName") - tables, last_eval = self.dynamodb_backend.list_tables( - limit, exclusive_start_table_name - ) - - response = {"TableNames": tables} - if last_eval: - response["LastEvaluatedTableName"] = last_eval - - return dynamo_json_dump(response) - - def create_table(self): - body = self.body - # get the table name - table_name = body["TableName"] - # check billing mode and get the throughput - if "BillingMode" in body.keys() and body["BillingMode"] == "PAY_PER_REQUEST": - if "ProvisionedThroughput" in body.keys(): - er = "com.amazonaws.dynamodb.v20111205#ValidationException" - return self.error( - er, - "ProvisionedThroughput cannot be specified when BillingMode is PAY_PER_REQUEST", - ) - throughput = None - billing_mode = "PAY_PER_REQUEST" - else: # Provisioned (default billing mode) - throughput = body.get("ProvisionedThroughput") - if throughput is None: - return self.error( - "ValidationException", - "One or more parameter values were invalid: ReadCapacityUnits and WriteCapacityUnits must both be specified when BillingMode is PROVISIONED", - ) - billing_mode = "PROVISIONED" - # getting ServerSideEncryption details - sse_spec = body.get("SSESpecification") - # getting the schema - key_schema = body["KeySchema"] - # getting attribute definition - attr = body["AttributeDefinitions"] - # getting the indexes - global_indexes = body.get("GlobalSecondaryIndexes") - if global_indexes == []: - return self.error( - "ValidationException", - "One or more parameter values were invalid: List of GlobalSecondaryIndexes is empty", - ) - global_indexes = global_indexes or [] - local_secondary_indexes = body.get("LocalSecondaryIndexes") - if local_secondary_indexes == []: - return self.error( - "ValidationException", - "One or more parameter values were invalid: List of LocalSecondaryIndexes is empty", - ) - local_secondary_indexes = local_secondary_indexes or [] - # Verify AttributeDefinitions list all - expected_attrs = [] - expected_attrs.extend([key["AttributeName"] for key in key_schema]) - expected_attrs.extend( - schema["AttributeName"] - for schema in itertools.chain( - *list(idx["KeySchema"] for idx in local_secondary_indexes) - ) - ) - expected_attrs.extend( - schema["AttributeName"] - for schema in itertools.chain( - *list(idx["KeySchema"] for idx in global_indexes) - ) - ) - expected_attrs = list(set(expected_attrs)) - expected_attrs.sort() - actual_attrs = [item["AttributeName"] for item in attr] - actual_attrs.sort() - if actual_attrs != expected_attrs: - return self._throw_attr_error( - actual_attrs, expected_attrs, global_indexes or local_secondary_indexes - ) - # get the stream specification - streams = body.get("StreamSpecification") - # Get any tags - tags = body.get("Tags", []) - - table = self.dynamodb_backend.create_table( - table_name, - schema=key_schema, - throughput=throughput, - attr=attr, - global_indexes=global_indexes, - indexes=local_secondary_indexes, - streams=streams, - billing_mode=billing_mode, - sse_specification=sse_spec, - tags=tags, - ) - if table is not None: - return dynamo_json_dump(table.describe()) - else: - er = "com.amazonaws.dynamodb.v20111205#ResourceInUseException" - return self.error(er, "Resource in use") - - def _throw_attr_error(self, actual_attrs, expected_attrs, indexes): - def dump_list(list_): - return str(list_).replace("'", "") - - er = "com.amazonaws.dynamodb.v20111205#ValidationException" - err_head = "One or more parameter values were invalid: " - if len(actual_attrs) > len(expected_attrs): - if indexes: - return self.error( - er, - err_head - + "Some AttributeDefinitions are not used. AttributeDefinitions: " - + dump_list(actual_attrs) - + ", keys used: " - + dump_list(expected_attrs), - ) - else: - return self.error( - er, - err_head - + "Number of attributes in KeySchema does not exactly match number of attributes defined in AttributeDefinitions", - ) - elif len(actual_attrs) < len(expected_attrs): - if indexes: - return self.error( - er, - err_head - + "Some index key attributes are not defined in AttributeDefinitions. Keys: " - + dump_list(list(set(expected_attrs) - set(actual_attrs))) - + ", AttributeDefinitions: " - + dump_list(actual_attrs), - ) - else: - return self.error( - er, "Invalid KeySchema: Some index key attribute have no definition" - ) - else: - if indexes: - return self.error( - er, - err_head - + "Some index key attributes are not defined in AttributeDefinitions. Keys: " - + dump_list(list(set(expected_attrs) - set(actual_attrs))) - + ", AttributeDefinitions: " - + dump_list(actual_attrs), - ) - else: - return self.error( - er, - err_head - + "Some index key attributes are not defined in AttributeDefinitions. Keys: " - + dump_list(expected_attrs) - + ", AttributeDefinitions: " - + dump_list(actual_attrs), - ) - - def delete_table(self): - name = self.body["TableName"] - table = self.dynamodb_backend.delete_table(name) - if table is not None: - return dynamo_json_dump(table.describe()) - else: - er = "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException" - return self.error(er, "Requested resource not found") - - def describe_endpoints(self): - response = {"Endpoints": self.dynamodb_backend.describe_endpoints()} - return dynamo_json_dump(response) - - def tag_resource(self): - table_arn = self.body["ResourceArn"] - tags = self.body["Tags"] - self.dynamodb_backend.tag_resource(table_arn, tags) - return "" - - def untag_resource(self): - table_arn = self.body["ResourceArn"] - tags = self.body["TagKeys"] - self.dynamodb_backend.untag_resource(table_arn, tags) - return "" - - def list_tags_of_resource(self): - try: - table_arn = self.body["ResourceArn"] - all_tags = self.dynamodb_backend.list_tags_of_resource(table_arn) - all_tag_keys = [tag["Key"] for tag in all_tags] - marker = self.body.get("NextToken") - if marker: - start = all_tag_keys.index(marker) + 1 - else: - start = 0 - max_items = 10 # there is no default, but using 10 to make testing easier - tags_resp = all_tags[start : start + max_items] - next_marker = None - if len(all_tags) > start + max_items: - next_marker = tags_resp[-1]["Key"] - if next_marker: - return json.dumps({"Tags": tags_resp, "NextToken": next_marker}) - return json.dumps({"Tags": tags_resp}) - except AttributeError: - er = "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException" - return self.error(er, "Requested resource not found") - - def update_table(self): - name = self.body["TableName"] - attr_definitions = self.body.get("AttributeDefinitions", None) - global_index = self.body.get("GlobalSecondaryIndexUpdates", None) - throughput = self.body.get("ProvisionedThroughput", None) - billing_mode = self.body.get("BillingMode", None) - stream_spec = self.body.get("StreamSpecification", None) - try: - table = self.dynamodb_backend.update_table( - name=name, - attr_definitions=attr_definitions, - global_index=global_index, - throughput=throughput, - billing_mode=billing_mode, - stream_spec=stream_spec, - ) - return dynamo_json_dump(table.describe()) - except ValueError: - er = "com.amazonaws.dynamodb.v20111205#ResourceInUseException" - return self.error(er, "Cannot enable stream") - - def describe_table(self): - name = self.body["TableName"] - try: - table = self.dynamodb_backend.describe_table(name) - return dynamo_json_dump(table) - except KeyError: - er = "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException" - return self.error(er, "Requested resource not found") - - @include_consumed_capacity() - def put_item(self): - name = self.body["TableName"] - item = self.body["Item"] - return_values = self.body.get("ReturnValues", "NONE") - - if return_values not in ("ALL_OLD", "NONE"): - er = "com.amazonaws.dynamodb.v20111205#ValidationException" - return self.error(er, "Return values set to invalid value") - - if put_has_empty_keys(item, self.dynamodb_backend.get_table(name)): - return get_empty_str_error() - - overwrite = "Expected" not in self.body - if not overwrite: - expected = self.body["Expected"] - else: - expected = None - - if return_values == "ALL_OLD": - existing_item = self.dynamodb_backend.get_item(name, item) - if existing_item: - existing_attributes = existing_item.to_json()["Attributes"] - else: - existing_attributes = {} - - # Attempt to parse simple ConditionExpressions into an Expected - # expression - condition_expression = self.body.get("ConditionExpression") - expression_attribute_names = self.body.get("ExpressionAttributeNames", {}) - expression_attribute_values = self.body.get("ExpressionAttributeValues", {}) - - if condition_expression: - overwrite = False - - try: - result = self.dynamodb_backend.put_item( - name, - item, - expected, - condition_expression, - expression_attribute_names, - expression_attribute_values, - overwrite, - ) - except MockValidationException as mve: - er = "com.amazonaws.dynamodb.v20111205#ValidationException" - return self.error(er, mve.exception_msg) - except KeyError as ke: - er = "com.amazonaws.dynamodb.v20111205#ValidationException" - return self.error(er, ke.args[0]) - except ValueError as ve: - er = "com.amazonaws.dynamodb.v20111205#ConditionalCheckFailedException" - return self.error(er, str(ve)) - - if result: - item_dict = result.to_json() - if return_values == "ALL_OLD": - item_dict["Attributes"] = existing_attributes - else: - item_dict.pop("Attributes", None) - return dynamo_json_dump(item_dict) - else: - er = "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException" - return self.error(er, "Requested resource not found") - - def batch_write_item(self): - table_batches = self.body["RequestItems"] - - for table_name, table_requests in table_batches.items(): - for table_request in table_requests: - request_type = list(table_request.keys())[0] - request = list(table_request.values())[0] - if request_type == "PutRequest": - item = request["Item"] - res = self.dynamodb_backend.put_item(table_name, item) - if not res: - return self.error( - "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException", - "Requested resource not found", - ) - elif request_type == "DeleteRequest": - keys = request["Key"] - item = self.dynamodb_backend.delete_item(table_name, keys) - - response = { - "ConsumedCapacity": [ - { - "TableName": table_name, - "CapacityUnits": 1.0, - "Table": {"CapacityUnits": 1.0}, - } - for table_name, table_requests in table_batches.items() - ], - "ItemCollectionMetrics": {}, - "UnprocessedItems": {}, - } - - return dynamo_json_dump(response) - - @include_consumed_capacity(0.5) - def get_item(self): - name = self.body["TableName"] - table = self.dynamodb_backend.get_table(name) - if table is None: - return self.error( - "com.amazonaws.dynamodb.v20120810#ResourceNotFoundException", - "Requested resource not found", - ) - key = self.body["Key"] - projection_expression = self.body.get("ProjectionExpression") - expression_attribute_names = self.body.get("ExpressionAttributeNames") - if expression_attribute_names == {}: - if projection_expression is None: - er = "ValidationException" - return self.error( - er, - "ExpressionAttributeNames can only be specified when using expressions", - ) - else: - er = "ValidationException" - return self.error(er, "ExpressionAttributeNames must not be empty") - - expression_attribute_names = expression_attribute_names or {} - projection_expression = self._adjust_projection_expression( - projection_expression, expression_attribute_names - ) - - try: - item = self.dynamodb_backend.get_item(name, key, projection_expression) - except ValueError: - er = "com.amazon.coral.validate#ValidationException" - return self.error(er, "Validation Exception") - if item: - item_dict = item.describe_attrs(attributes=None) - return dynamo_json_dump(item_dict) - else: - # Item not found - return dynamo_json_dump({}) - - def batch_get_item(self): - table_batches = self.body["RequestItems"] - - results = {"ConsumedCapacity": [], "Responses": {}, "UnprocessedKeys": {}} - - # Validation: Can only request up to 100 items at the same time - # Scenario 1: We're requesting more than a 100 keys from a single table - for table_name, table_request in table_batches.items(): - if len(table_request["Keys"]) > 100: - return self.error( - "com.amazonaws.dynamodb.v20111205#ValidationException", - "1 validation error detected: Value at 'requestItems." - + table_name - + ".member.keys' failed to satisfy constraint: Member must have length less than or equal to 100", - ) - # Scenario 2: We're requesting more than a 100 keys across all tables - nr_of_keys_across_all_tables = sum( - [len(req["Keys"]) for _, req in table_batches.items()] - ) - if nr_of_keys_across_all_tables > 100: - return self.error( - "com.amazonaws.dynamodb.v20111205#ValidationException", - "Too many items requested for the BatchGetItem call", - ) - - for table_name, table_request in table_batches.items(): - keys = table_request["Keys"] - if self._contains_duplicates(keys): - er = "com.amazon.coral.validate#ValidationException" - return self.error(er, "Provided list of item keys contains duplicates") - attributes_to_get = table_request.get("AttributesToGet") - projection_expression = table_request.get("ProjectionExpression") - expression_attribute_names = table_request.get( - "ExpressionAttributeNames", {} - ) - - projection_expression = self._adjust_projection_expression( - projection_expression, expression_attribute_names - ) - - results["Responses"][table_name] = [] - for key in keys: - try: - item = self.dynamodb_backend.get_item( - table_name, key, projection_expression - ) - except ValueError: - return self.error( - "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException", - "Requested resource not found", - ) - if item: - item_describe = item.describe_attrs(attributes_to_get) - results["Responses"][table_name].append(item_describe["Item"]) - - results["ConsumedCapacity"].append( - {"CapacityUnits": len(keys), "TableName": table_name} - ) - return dynamo_json_dump(results) - - def _contains_duplicates(self, keys): - unique_keys = [] - for k in keys: - if k in unique_keys: - return True - else: - unique_keys.append(k) - return False - - @include_consumed_capacity() - def query(self): - name = self.body["TableName"] - key_condition_expression = self.body.get("KeyConditionExpression") - projection_expression = self.body.get("ProjectionExpression") - expression_attribute_names = self.body.get("ExpressionAttributeNames", {}) - filter_expression = self.body.get("FilterExpression") - expression_attribute_values = self.body.get("ExpressionAttributeValues", {}) - - projection_expression = self._adjust_projection_expression( - projection_expression, expression_attribute_names - ) - - filter_kwargs = {} - - if key_condition_expression: - value_alias_map = self.body.get("ExpressionAttributeValues", {}) - - table = self.dynamodb_backend.get_table(name) - - # If table does not exist - if table is None: - return self.error( - "com.amazonaws.dynamodb.v20120810#ResourceNotFoundException", - "Requested resource not found", - ) - - index_name = self.body.get("IndexName") - if index_name: - all_indexes = (table.global_indexes or []) + (table.indexes or []) - indexes_by_name = dict((i.name, i) for i in all_indexes) - if index_name not in indexes_by_name: - er = "com.amazonaws.dynamodb.v20120810#ResourceNotFoundException" - return self.error( - er, - "Invalid index: {} for table: {}. Available indexes are: {}".format( - index_name, name, ", ".join(indexes_by_name.keys()) - ), - ) - - index = indexes_by_name[index_name].schema - else: - index = table.schema - - reverse_attribute_lookup = dict( - (v, k) for k, v in self.body.get("ExpressionAttributeNames", {}).items() - ) - - if " and " in key_condition_expression.lower(): - expressions = re.split( - " AND ", key_condition_expression, maxsplit=1, flags=re.IGNORECASE - ) - - index_hash_key = [key for key in index if key["KeyType"] == "HASH"][0] - hash_key_var = reverse_attribute_lookup.get( - index_hash_key["AttributeName"], index_hash_key["AttributeName"] - ) - hash_key_regex = r"(^|[\s(]){0}\b".format(hash_key_var) - i, hash_key_expression = next( - ( - (i, e) - for i, e in enumerate(expressions) - if re.search(hash_key_regex, e) - ), - (None, None), - ) - if hash_key_expression is None: - return self.error( - "ValidationException", - "Query condition missed key schema element: {}".format( - hash_key_var - ), - ) - hash_key_expression = hash_key_expression.strip("()") - expressions.pop(i) - - # TODO implement more than one range expression and OR operators - range_key_expression = expressions[0].strip("()") - # Split expression, and account for all kinds of whitespacing around commas and brackets - range_key_expression_components = re.split( - r"\s*\(\s*|\s*,\s*|\s", range_key_expression - ) - # Skip whitespace - range_key_expression_components = [ - c for c in range_key_expression_components if c - ] - range_comparison = range_key_expression_components[1] - - if " and " in range_key_expression.lower(): - range_comparison = "BETWEEN" - # [range_key, between, x, and, y] - range_values = [ - value_alias_map[range_key_expression_components[2]], - value_alias_map[range_key_expression_components[4]], - ] - supplied_range_key = range_key_expression_components[0] - elif "begins_with" in range_key_expression: - range_comparison = "BEGINS_WITH" - # [begins_with, range_key, x] - range_values = [ - value_alias_map[range_key_expression_components[-1]] - ] - supplied_range_key = range_key_expression_components[1] - elif "begins_with" in range_key_expression.lower(): - function_used = range_key_expression[ - range_key_expression.lower().index("begins_with") : len( - "begins_with" - ) - ] - return self.error( - "com.amazonaws.dynamodb.v20111205#ValidationException", - "Invalid KeyConditionExpression: Invalid function name; function: {}".format( - function_used - ), - ) - else: - # [range_key, =, x] - range_values = [value_alias_map[range_key_expression_components[2]]] - supplied_range_key = range_key_expression_components[0] - - supplied_range_key = expression_attribute_names.get( - supplied_range_key, supplied_range_key - ) - range_keys = [ - k["AttributeName"] for k in index if k["KeyType"] == "RANGE" - ] - if supplied_range_key not in range_keys: - return self.error( - "ValidationException", - "Query condition missed key schema element: {}".format( - range_keys[0] - ), - ) - else: - hash_key_expression = key_condition_expression.strip("()") - range_comparison = None - range_values = [] - - if not re.search("[^<>]=", hash_key_expression): - return self.error( - "com.amazonaws.dynamodb.v20111205#ValidationException", - "Query key condition not supported", - ) - hash_key_value_alias = hash_key_expression.split("=")[1].strip() - # Temporary fix until we get proper KeyConditionExpression function - hash_key = value_alias_map.get( - hash_key_value_alias, {"S": hash_key_value_alias} - ) - else: - # 'KeyConditions': {u'forum_name': {u'ComparisonOperator': u'EQ', u'AttributeValueList': [{u'S': u'the-key'}]}} - key_conditions = self.body.get("KeyConditions") - query_filters = self.body.get("QueryFilter") - - if not (key_conditions or query_filters): - return self.error( - "com.amazonaws.dynamodb.v20111205#ValidationException", - "Either KeyConditions or QueryFilter should be present", - ) - - if key_conditions: - ( - hash_key_name, - range_key_name, - ) = self.dynamodb_backend.get_table_keys_name( - name, key_conditions.keys() - ) - for key, value in key_conditions.items(): - if key not in (hash_key_name, range_key_name): - filter_kwargs[key] = value - if hash_key_name is None: - er = "'com.amazonaws.dynamodb.v20120810#ResourceNotFoundException" - return self.error(er, "Requested resource not found") - hash_key = key_conditions[hash_key_name]["AttributeValueList"][0] - if len(key_conditions) == 1: - range_comparison = None - range_values = [] - else: - if range_key_name is None and not filter_kwargs: - er = "com.amazon.coral.validate#ValidationException" - return self.error(er, "Validation Exception") - else: - range_condition = key_conditions.get(range_key_name) - if range_condition: - range_comparison = range_condition["ComparisonOperator"] - range_values = range_condition["AttributeValueList"] - else: - range_comparison = None - range_values = [] - if query_filters: - filter_kwargs.update(query_filters) - index_name = self.body.get("IndexName") - exclusive_start_key = self.body.get("ExclusiveStartKey") - limit = self.body.get("Limit") - scan_index_forward = self.body.get("ScanIndexForward") - items, scanned_count, last_evaluated_key = self.dynamodb_backend.query( - name, - hash_key, - range_comparison, - range_values, - limit, - exclusive_start_key, - scan_index_forward, - projection_expression, - index_name=index_name, - expr_names=expression_attribute_names, - expr_values=expression_attribute_values, - filter_expression=filter_expression, - **filter_kwargs - ) - if items is None: - er = "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException" - return self.error(er, "Requested resource not found") - - result = { - "Count": len(items), - "ScannedCount": scanned_count, - } - - if self.body.get("Select", "").upper() != "COUNT": - result["Items"] = [item.attrs for item in items] - - if last_evaluated_key is not None: - result["LastEvaluatedKey"] = last_evaluated_key - - return dynamo_json_dump(result) - - def _adjust_projection_expression(self, projection_expression, expr_attr_names): - def _adjust(expression): - return ( - expr_attr_names[expression] - if expression in expr_attr_names - else expression - ) - - if projection_expression and expr_attr_names: - expressions = [x.strip() for x in projection_expression.split(",")] - return ",".join( - [ - ".".join([_adjust(expr) for expr in nested_expr.split(".")]) - for nested_expr in expressions - ] - ) - - return projection_expression - - @include_consumed_capacity() - def scan(self): - name = self.body["TableName"] - - filters = {} - scan_filters = self.body.get("ScanFilter", {}) - for attribute_name, scan_filter in scan_filters.items(): - # Keys are attribute names. Values are tuples of (comparison, - # comparison_value) - comparison_operator = scan_filter["ComparisonOperator"] - comparison_values = scan_filter.get("AttributeValueList", []) - filters[attribute_name] = (comparison_operator, comparison_values) - - filter_expression = self.body.get("FilterExpression") - expression_attribute_values = self.body.get("ExpressionAttributeValues", {}) - expression_attribute_names = self.body.get("ExpressionAttributeNames", {}) - projection_expression = self.body.get("ProjectionExpression", "") - exclusive_start_key = self.body.get("ExclusiveStartKey") - limit = self.body.get("Limit") - index_name = self.body.get("IndexName") - - try: - items, scanned_count, last_evaluated_key = self.dynamodb_backend.scan( - name, - filters, - limit, - exclusive_start_key, - filter_expression, - expression_attribute_names, - expression_attribute_values, - index_name, - projection_expression, - ) - except InvalidIndexNameError as err: - er = "com.amazonaws.dynamodb.v20111205#ValidationException" - return self.error(er, str(err)) - except ValueError as err: - er = "com.amazonaws.dynamodb.v20111205#ValidationError" - return self.error(er, "Bad Filter Expression: {0}".format(err)) - except Exception as err: - er = "com.amazonaws.dynamodb.v20111205#InternalFailure" - return self.error(er, "Internal error. {0}".format(err)) - - # Items should be a list, at least an empty one. Is None if table does not exist. - # Should really check this at the beginning - if items is None: - er = "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException" - return self.error(er, "Requested resource not found") - - result = { - "Count": len(items), - "Items": [item.attrs for item in items], - "ScannedCount": scanned_count, - } - if last_evaluated_key is not None: - result["LastEvaluatedKey"] = last_evaluated_key - return dynamo_json_dump(result) - - def delete_item(self): - name = self.body["TableName"] - key = self.body["Key"] - return_values = self.body.get("ReturnValues", "NONE") - if return_values not in ("ALL_OLD", "NONE"): - er = "com.amazonaws.dynamodb.v20111205#ValidationException" - return self.error(er, "Return values set to invalid value") - - table = self.dynamodb_backend.get_table(name) - if not table: - er = "com.amazonaws.dynamodb.v20120810#ConditionalCheckFailedException" - return self.error( - er, "A condition specified in the operation could not be evaluated." - ) - - # Attempt to parse simple ConditionExpressions into an Expected - # expression - condition_expression = self.body.get("ConditionExpression") - expression_attribute_names = self.body.get("ExpressionAttributeNames", {}) - expression_attribute_values = self.body.get("ExpressionAttributeValues", {}) - - try: - item = self.dynamodb_backend.delete_item( - name, - key, - expression_attribute_names, - expression_attribute_values, - condition_expression, - ) - except ValueError: - er = "com.amazonaws.dynamodb.v20111205#ConditionalCheckFailedException" - return self.error( - er, "A condition specified in the operation could not be evaluated." - ) - - if item and return_values == "ALL_OLD": - item_dict = item.to_json() - else: - item_dict = {"Attributes": {}} - item_dict["ConsumedCapacityUnits"] = 0.5 - return dynamo_json_dump(item_dict) - - def update_item(self): - name = self.body["TableName"] - key = self.body["Key"] - return_values = self.body.get("ReturnValues", "NONE") - update_expression = self.body.get("UpdateExpression", "").strip() - attribute_updates = self.body.get("AttributeUpdates") - if update_expression and attribute_updates: - er = "com.amazonaws.dynamodb.v20111205#ValidationException" - return self.error( - er, - "Can not use both expression and non-expression parameters in the same request: Non-expression parameters: {AttributeUpdates} Expression parameters: {UpdateExpression}", - ) - # We need to copy the item in order to avoid it being modified by the update_item operation - try: - existing_item = copy.deepcopy(self.dynamodb_backend.get_item(name, key)) - except ValueError: - return self.error( - "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException", - "Requested resource not found", - ) - if existing_item: - existing_attributes = existing_item.to_json()["Attributes"] - else: - existing_attributes = {} - - if return_values not in ( - "NONE", - "ALL_OLD", - "ALL_NEW", - "UPDATED_OLD", - "UPDATED_NEW", - ): - er = "com.amazonaws.dynamodb.v20111205#ValidationException" - return self.error(er, "Return values set to invalid value") - - if "Expected" in self.body: - expected = self.body["Expected"] - else: - expected = None - - # Attempt to parse simple ConditionExpressions into an Expected - # expression - condition_expression = self.body.get("ConditionExpression") - expression_attribute_names = self.body.get("ExpressionAttributeNames", {}) - expression_attribute_values = self.body.get("ExpressionAttributeValues", {}) - - try: - item = self.dynamodb_backend.update_item( - name, - key, - update_expression=update_expression, - attribute_updates=attribute_updates, - expression_attribute_names=expression_attribute_names, - expression_attribute_values=expression_attribute_values, - expected=expected, - condition_expression=condition_expression, - ) - except MockValidationException as mve: - er = "com.amazonaws.dynamodb.v20111205#ValidationException" - return self.error(er, mve.exception_msg) - except ValueError: - er = "com.amazonaws.dynamodb.v20111205#ConditionalCheckFailedException" - return self.error(er, "The conditional request failed") - except TypeError: - er = "com.amazonaws.dynamodb.v20111205#ValidationException" - return self.error(er, "Validation Exception") - - item_dict = item.to_json() - item_dict["ConsumedCapacity"] = {"TableName": name, "CapacityUnits": 0.5} - unchanged_attributes = { - k - for k in existing_attributes.keys() - if existing_attributes[k] == item_dict["Attributes"].get(k) - } - changed_attributes = ( - set(existing_attributes.keys()) - .union(item_dict["Attributes"].keys()) - .difference(unchanged_attributes) - ) - - if return_values == "NONE": - item_dict["Attributes"] = {} - elif return_values == "ALL_OLD": - item_dict["Attributes"] = existing_attributes - elif return_values == "UPDATED_OLD": - item_dict["Attributes"] = { - k: v for k, v in existing_attributes.items() if k in changed_attributes - } - elif return_values == "UPDATED_NEW": - item_dict["Attributes"] = self._build_updated_new_attributes( - existing_attributes, item_dict["Attributes"] - ) - return dynamo_json_dump(item_dict) - - def _build_updated_new_attributes(self, original, changed): - if type(changed) != type(original): - return changed - else: - if type(changed) is dict: - return { - key: self._build_updated_new_attributes( - original.get(key, None), changed[key] - ) - for key in changed.keys() - if key not in original or changed[key] != original[key] - } - elif type(changed) in (set, list): - if len(changed) != len(original): - return changed - else: - return [ - self._build_updated_new_attributes( - original[index], changed[index] - ) - for index in range(len(changed)) - ] - else: - return changed - - def describe_limits(self): - return json.dumps( - { - "AccountMaxReadCapacityUnits": 20000, - "TableMaxWriteCapacityUnits": 10000, - "AccountMaxWriteCapacityUnits": 20000, - "TableMaxReadCapacityUnits": 10000, - } - ) - - def update_time_to_live(self): - name = self.body["TableName"] - ttl_spec = self.body["TimeToLiveSpecification"] - - self.dynamodb_backend.update_time_to_live(name, ttl_spec) - - return json.dumps({"TimeToLiveSpecification": ttl_spec}) - - def describe_time_to_live(self): - name = self.body["TableName"] - - ttl_spec = self.dynamodb_backend.describe_time_to_live(name) - - return json.dumps({"TimeToLiveDescription": ttl_spec}) - - def transact_get_items(self): - transact_items = self.body["TransactItems"] - responses = list() - - if len(transact_items) > TRANSACTION_MAX_ITEMS: - msg = "1 validation error detected: Value '[" - err_list = list() - request_id = 268435456 - for _ in transact_items: - request_id += 1 - hex_request_id = format(request_id, "x") - err_list.append( - "com.amazonaws.dynamodb.v20120810.TransactGetItem@%s" - % hex_request_id - ) - msg += ", ".join(err_list) - msg += ( - "'] at 'transactItems' failed to satisfy constraint: " - "Member must have length less than or equal to %s" - % TRANSACTION_MAX_ITEMS - ) - - return self.error("ValidationException", msg) - - ret_consumed_capacity = self.body.get("ReturnConsumedCapacity", "NONE") - consumed_capacity = dict() - - for transact_item in transact_items: - - table_name = transact_item["Get"]["TableName"] - key = transact_item["Get"]["Key"] - try: - item = self.dynamodb_backend.get_item(table_name, key) - except ValueError: - er = "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException" - return self.error(er, "Requested resource not found") - - if not item: - responses.append({}) - continue - - item_describe = item.describe_attrs(False) - responses.append(item_describe) - - table_capacity = consumed_capacity.get(table_name, {}) - table_capacity["TableName"] = table_name - capacity_units = table_capacity.get("CapacityUnits", 0) + 2.0 - table_capacity["CapacityUnits"] = capacity_units - read_capacity_units = table_capacity.get("ReadCapacityUnits", 0) + 2.0 - table_capacity["ReadCapacityUnits"] = read_capacity_units - consumed_capacity[table_name] = table_capacity - - if ret_consumed_capacity == "INDEXES": - table_capacity["Table"] = { - "CapacityUnits": capacity_units, - "ReadCapacityUnits": read_capacity_units, - } - - result = dict() - result.update({"Responses": responses}) - if ret_consumed_capacity != "NONE": - result.update({"ConsumedCapacity": [v for v in consumed_capacity.values()]}) - - return dynamo_json_dump(result) - - def transact_write_items(self): - transact_items = self.body["TransactItems"] - try: - self.dynamodb_backend.transact_write_items(transact_items) - except TransactionCanceledException as e: - er = "com.amazonaws.dynamodb.v20111205#TransactionCanceledException" - return self.error(er, str(e)) - except MockValidationException as mve: - er = "com.amazonaws.dynamodb.v20111205#ValidationException" - return self.error(er, mve.exception_msg) - response = {"ConsumedCapacity": [], "ItemCollectionMetrics": {}} - return dynamo_json_dump(response) - - def describe_continuous_backups(self): - name = self.body["TableName"] - - if self.dynamodb_backend.get_table(name) is None: - return self.error( - "com.amazonaws.dynamodb.v20111205#TableNotFoundException", - "Table not found: {}".format(name), - ) - - response = self.dynamodb_backend.describe_continuous_backups(name) - - return json.dumps({"ContinuousBackupsDescription": response}) - - def update_continuous_backups(self): - name = self.body["TableName"] - point_in_time_spec = self.body["PointInTimeRecoverySpecification"] - - if self.dynamodb_backend.get_table(name) is None: - return self.error( - "com.amazonaws.dynamodb.v20111205#TableNotFoundException", - "Table not found: {}".format(name), - ) - - response = self.dynamodb_backend.update_continuous_backups( - name, point_in_time_spec - ) - - return json.dumps({"ContinuousBackupsDescription": response}) - - def list_backups(self): - body = self.body - table_name = body.get("TableName") - backups = self.dynamodb_backend.list_backups(table_name) - response = {"BackupSummaries": [backup.summary for backup in backups]} - return dynamo_json_dump(response) - - def create_backup(self): - body = self.body - table_name = body.get("TableName") - backup_name = body.get("BackupName") - try: - backup = self.dynamodb_backend.create_backup(table_name, backup_name) - response = {"BackupDetails": backup.details} - return dynamo_json_dump(response) - except KeyError: - er = "com.amazonaws.dynamodb.v20111205#TableNotFoundException" - return self.error(er, "Table not found: %s" % table_name) - - def delete_backup(self): - body = self.body - backup_arn = body.get("BackupArn") - try: - backup = self.dynamodb_backend.delete_backup(backup_arn) - response = {"BackupDescription": backup.description} - return dynamo_json_dump(response) - except KeyError: - er = "com.amazonaws.dynamodb.v20111205#BackupNotFoundException" - return self.error(er, "Backup not found: %s" % backup_arn) - - def describe_backup(self): - body = self.body - backup_arn = body.get("BackupArn") - try: - backup = self.dynamodb_backend.describe_backup(backup_arn) - response = {"BackupDescription": backup.description} - return dynamo_json_dump(response) - except KeyError: - er = "com.amazonaws.dynamodb.v20111205#BackupNotFoundException" - return self.error(er, "Backup not found: %s" % backup_arn) - - def restore_table_from_backup(self): - body = self.body - target_table_name = body.get("TargetTableName") - backup_arn = body.get("BackupArn") - try: - restored_table = self.dynamodb_backend.restore_table_from_backup( - target_table_name, backup_arn - ) - return dynamo_json_dump(restored_table.describe()) - except KeyError: - er = "com.amazonaws.dynamodb.v20111205#BackupNotFoundException" - return self.error(er, "Backup not found: %s" % backup_arn) - except ValueError: - er = "com.amazonaws.dynamodb.v20111205#TableAlreadyExistsException" - return self.error(er, "Table already exists: %s" % target_table_name) - - def restore_table_to_point_in_time(self): - body = self.body - target_table_name = body.get("TargetTableName") - source_table_name = body.get("SourceTableName") - try: - restored_table = self.dynamodb_backend.restore_table_to_point_in_time( - target_table_name, source_table_name - ) - return dynamo_json_dump(restored_table.describe()) - except KeyError: - er = "com.amazonaws.dynamodb.v20111205#SourceTableNotFoundException" - return self.error(er, "Source table not found: %s" % source_table_name) - except ValueError: - er = "com.amazonaws.dynamodb.v20111205#TableAlreadyExistsException" - return self.error(er, "Table already exists: %s" % target_table_name) diff --git a/moto/dynamodb_v20111205/__init__.py b/moto/dynamodb_v20111205/__init__.py new file mode 100644 index 000000000000..11040bb8d9c1 --- /dev/null +++ b/moto/dynamodb_v20111205/__init__.py @@ -0,0 +1,9 @@ +from .models import dynamodb_backend + +""" +An older API version of DynamoDB. +Please see the corresponding tests (tests/test_dynamodb_v20111205) on how to invoke this API. +""" + +dynamodb_backends = {"global": dynamodb_backend} +mock_dynamodb = dynamodb_backend.decorator diff --git a/moto/dynamodb_v20111205/comparisons.py b/moto/dynamodb_v20111205/comparisons.py new file mode 100644 index 000000000000..f31b9d5c32d7 --- /dev/null +++ b/moto/dynamodb_v20111205/comparisons.py @@ -0,0 +1,22 @@ +# TODO add tests for all of these +COMPARISON_FUNCS = { + "EQ": lambda item_value, test_value: item_value == test_value, + "NE": lambda item_value, test_value: item_value != test_value, + "LE": lambda item_value, test_value: item_value <= test_value, + "LT": lambda item_value, test_value: item_value < test_value, + "GE": lambda item_value, test_value: item_value >= test_value, + "GT": lambda item_value, test_value: item_value > test_value, + "NULL": lambda item_value: item_value is None, + "NOT_NULL": lambda item_value: item_value is not None, + "CONTAINS": lambda item_value, test_value: test_value in item_value, + "NOT_CONTAINS": lambda item_value, test_value: test_value not in item_value, + "BEGINS_WITH": lambda item_value, test_value: item_value.startswith(test_value), + "IN": lambda item_value, *test_values: item_value in test_values, + "BETWEEN": lambda item_value, lower_test_value, upper_test_value: lower_test_value + <= item_value + <= upper_test_value, +} + + +def get_comparison_func(range_comparison): + return COMPARISON_FUNCS.get(range_comparison) diff --git a/moto/dynamodb/models.py b/moto/dynamodb_v20111205/models.py similarity index 100% rename from moto/dynamodb/models.py rename to moto/dynamodb_v20111205/models.py diff --git a/moto/dynamodb_v20111205/responses.py b/moto/dynamodb_v20111205/responses.py new file mode 100644 index 000000000000..4584ae29a85a --- /dev/null +++ b/moto/dynamodb_v20111205/responses.py @@ -0,0 +1,294 @@ +import json + +from moto.core.responses import BaseResponse +from moto.core.utils import camelcase_to_underscores +from .models import dynamodb_backend, dynamo_json_dump + + +class DynamoHandler(BaseResponse): + def get_endpoint_name(self, headers): + """Parses request headers and extracts part od the X-Amz-Target + that corresponds to a method of DynamoHandler + + ie: X-Amz-Target: DynamoDB_20111205.ListTables -> ListTables + """ + # Headers are case-insensitive. Probably a better way to do this. + match = headers.get("x-amz-target") or headers.get("X-Amz-Target") + if match: + return match.split(".")[1] + + def error(self, type_, status=400): + return status, self.response_headers, dynamo_json_dump({"__type": type_}) + + def call_action(self): + self.body = json.loads(self.body or "{}") + endpoint = self.get_endpoint_name(self.headers) + if endpoint: + endpoint = camelcase_to_underscores(endpoint) + response = getattr(self, endpoint)() + if isinstance(response, str): + return 200, self.response_headers, response + + else: + status_code, new_headers, response_content = response + self.response_headers.update(new_headers) + return status_code, self.response_headers, response_content + else: + return 404, self.response_headers, "" + + def list_tables(self): + body = self.body + limit = body.get("Limit") + if body.get("ExclusiveStartTableName"): + last = body.get("ExclusiveStartTableName") + start = list(dynamodb_backend.tables.keys()).index(last) + 1 + else: + start = 0 + all_tables = list(dynamodb_backend.tables.keys()) + if limit: + tables = all_tables[start : start + limit] + else: + tables = all_tables[start:] + response = {"TableNames": tables} + if limit and len(all_tables) > start + limit: + response["LastEvaluatedTableName"] = tables[-1] + return dynamo_json_dump(response) + + def create_table(self): + body = self.body + name = body["TableName"] + + key_schema = body["KeySchema"] + hash_key = key_schema["HashKeyElement"] + hash_key_attr = hash_key["AttributeName"] + hash_key_type = hash_key["AttributeType"] + + range_key = key_schema.get("RangeKeyElement", {}) + range_key_attr = range_key.get("AttributeName") + range_key_type = range_key.get("AttributeType") + + throughput = body["ProvisionedThroughput"] + read_units = throughput["ReadCapacityUnits"] + write_units = throughput["WriteCapacityUnits"] + + table = dynamodb_backend.create_table( + name, + hash_key_attr=hash_key_attr, + hash_key_type=hash_key_type, + range_key_attr=range_key_attr, + range_key_type=range_key_type, + read_capacity=int(read_units), + write_capacity=int(write_units), + ) + return dynamo_json_dump(table.describe) + + def delete_table(self): + name = self.body["TableName"] + table = dynamodb_backend.delete_table(name) + if table: + return dynamo_json_dump(table.describe) + else: + er = "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException" + return self.error(er) + + def update_table(self): + name = self.body["TableName"] + throughput = self.body["ProvisionedThroughput"] + new_read_units = throughput["ReadCapacityUnits"] + new_write_units = throughput["WriteCapacityUnits"] + table = dynamodb_backend.update_table_throughput( + name, new_read_units, new_write_units + ) + return dynamo_json_dump(table.describe) + + def describe_table(self): + name = self.body["TableName"] + try: + table = dynamodb_backend.tables[name] + except KeyError: + er = "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException" + return self.error(er) + return dynamo_json_dump(table.describe) + + def put_item(self): + name = self.body["TableName"] + item = self.body["Item"] + result = dynamodb_backend.put_item(name, item) + if result: + item_dict = result.to_json() + item_dict["ConsumedCapacityUnits"] = 1 + return dynamo_json_dump(item_dict) + else: + er = "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException" + return self.error(er) + + def batch_write_item(self): + table_batches = self.body["RequestItems"] + + for table_name, table_requests in table_batches.items(): + for table_request in table_requests: + request_type = list(table_request)[0] + request = list(table_request.values())[0] + + if request_type == "PutRequest": + item = request["Item"] + dynamodb_backend.put_item(table_name, item) + elif request_type == "DeleteRequest": + key = request["Key"] + hash_key = key["HashKeyElement"] + range_key = key.get("RangeKeyElement") + item = dynamodb_backend.delete_item(table_name, hash_key, range_key) + + response = { + "Responses": { + "Thread": {"ConsumedCapacityUnits": 1.0}, + "Reply": {"ConsumedCapacityUnits": 1.0}, + }, + "UnprocessedItems": {}, + } + + return dynamo_json_dump(response) + + def get_item(self): + name = self.body["TableName"] + key = self.body["Key"] + hash_key = key["HashKeyElement"] + range_key = key.get("RangeKeyElement") + attrs_to_get = self.body.get("AttributesToGet") + try: + item = dynamodb_backend.get_item(name, hash_key, range_key) + except ValueError: + er = "com.amazon.coral.validate#ValidationException" + return self.error(er, status=400) + if item: + item_dict = item.describe_attrs(attrs_to_get) + item_dict["ConsumedCapacityUnits"] = 0.5 + return dynamo_json_dump(item_dict) + else: + # Item not found + er = "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException" + return self.error(er, status=404) + + def batch_get_item(self): + table_batches = self.body["RequestItems"] + + results = {"Responses": {"UnprocessedKeys": {}}} + + for table_name, table_request in table_batches.items(): + items = [] + keys = table_request["Keys"] + attributes_to_get = table_request.get("AttributesToGet") + for key in keys: + hash_key = key["HashKeyElement"] + range_key = key.get("RangeKeyElement") + item = dynamodb_backend.get_item(table_name, hash_key, range_key) + if item: + item_describe = item.describe_attrs(attributes_to_get) + items.append(item_describe) + results["Responses"][table_name] = { + "Items": items, + "ConsumedCapacityUnits": 1, + } + return dynamo_json_dump(results) + + def query(self): + name = self.body["TableName"] + hash_key = self.body["HashKeyValue"] + range_condition = self.body.get("RangeKeyCondition") + if range_condition: + range_comparison = range_condition["ComparisonOperator"] + range_values = range_condition["AttributeValueList"] + else: + range_comparison = None + range_values = [] + + items, _ = dynamodb_backend.query( + name, hash_key, range_comparison, range_values + ) + + if items is None: + er = "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException" + return self.error(er) + + result = { + "Count": len(items), + "Items": [item.attrs for item in items], + "ConsumedCapacityUnits": 1, + } + + # Implement this when we do pagination + # if not last_page: + # result["LastEvaluatedKey"] = { + # "HashKeyElement": items[-1].hash_key, + # "RangeKeyElement": items[-1].range_key, + # } + return dynamo_json_dump(result) + + def scan(self): + name = self.body["TableName"] + + filters = {} + scan_filters = self.body.get("ScanFilter", {}) + for attribute_name, scan_filter in scan_filters.items(): + # Keys are attribute names. Values are tuples of (comparison, + # comparison_value) + comparison_operator = scan_filter["ComparisonOperator"] + comparison_values = scan_filter.get("AttributeValueList", []) + filters[attribute_name] = (comparison_operator, comparison_values) + + items, scanned_count, _ = dynamodb_backend.scan(name, filters) + + if items is None: + er = "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException" + return self.error(er) + + result = { + "Count": len(items), + "Items": [item.attrs for item in items if item], + "ConsumedCapacityUnits": 1, + "ScannedCount": scanned_count, + } + + # Implement this when we do pagination + # if not last_page: + # result["LastEvaluatedKey"] = { + # "HashKeyElement": items[-1].hash_key, + # "RangeKeyElement": items[-1].range_key, + # } + return dynamo_json_dump(result) + + def delete_item(self): + name = self.body["TableName"] + key = self.body["Key"] + hash_key = key["HashKeyElement"] + range_key = key.get("RangeKeyElement") + return_values = self.body.get("ReturnValues", "") + item = dynamodb_backend.delete_item(name, hash_key, range_key) + if item: + if return_values == "ALL_OLD": + item_dict = item.to_json() + else: + item_dict = {"Attributes": []} + item_dict["ConsumedCapacityUnits"] = 0.5 + return dynamo_json_dump(item_dict) + else: + er = "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException" + return self.error(er) + + def update_item(self): + name = self.body["TableName"] + key = self.body["Key"] + hash_key = key["HashKeyElement"] + range_key = key.get("RangeKeyElement") + updates = self.body["AttributeUpdates"] + + item = dynamodb_backend.update_item(name, hash_key, range_key, updates) + + if item: + item_dict = item.to_json() + item_dict["ConsumedCapacityUnits"] = 0.5 + + return dynamo_json_dump(item_dict) + else: + er = "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException" + return self.error(er) diff --git a/moto/dynamodb2/urls.py b/moto/dynamodb_v20111205/urls.py similarity index 100% rename from moto/dynamodb2/urls.py rename to moto/dynamodb_v20111205/urls.py diff --git a/moto/dynamodbstreams/models.py b/moto/dynamodbstreams/models.py index 6a3797e7df60..629cd97bdc13 100644 --- a/moto/dynamodbstreams/models.py +++ b/moto/dynamodbstreams/models.py @@ -4,7 +4,7 @@ from moto.core import BaseBackend, BaseModel from moto.core.utils import BackendDict -from moto.dynamodb2.models import dynamodb_backends, DynamoJsonEncoder +from moto.dynamodb.models import dynamodb_backends, DynamoJsonEncoder class ShardIterator(BaseModel): diff --git a/moto/moto_server/werkzeug_app.py b/moto/moto_server/werkzeug_app.py index 51f5f8ea2e0a..6983c970c6ba 100644 --- a/moto/moto_server/werkzeug_app.py +++ b/moto/moto_server/werkzeug_app.py @@ -130,9 +130,11 @@ def infer_service_region_host(self, body, environ): dynamo_api_version = ( environ["HTTP_X_AMZ_TARGET"].split("_")[1].split(".")[0] ) - # If Newer API version, use dynamodb2 - if dynamo_api_version > "20111205": - host = "dynamodb2" + # Support for older API version + if dynamo_api_version <= "20111205": + host = "dynamodb_v20111205" + else: + host = "dynamodb" elif service == "sagemaker": host = "api.{service}.{region}.amazonaws.com".format( service=service, region=region diff --git a/scripts/implementation_coverage.py b/scripts/implementation_coverage.py index 39d48ad48983..0fff1f881d54 100755 --- a/scripts/implementation_coverage.py +++ b/scripts/implementation_coverage.py @@ -7,7 +7,7 @@ script_dir = os.path.dirname(os.path.abspath(__file__)) -alternative_service_names = {"lambda": "awslambda", "dynamodb": "dynamodb2"} +alternative_service_names = {"lambda": "awslambda"} def get_moto_implementation(service_name): diff --git a/tests/test_dynamodb2/conftest.py b/tests/test_dynamodb/conftest.py similarity index 91% rename from tests/test_dynamodb2/conftest.py rename to tests/test_dynamodb/conftest.py index 08bef17f13e7..5918f98912af 100644 --- a/tests/test_dynamodb2/conftest.py +++ b/tests/test_dynamodb/conftest.py @@ -1,5 +1,5 @@ import pytest -from moto.dynamodb2.models import Table +from moto.dynamodb.models import Table @pytest.fixture diff --git a/tests/test_dynamodb2/exceptions/__init__.py b/tests/test_dynamodb/exceptions/__init__.py similarity index 100% rename from tests/test_dynamodb2/exceptions/__init__.py rename to tests/test_dynamodb/exceptions/__init__.py diff --git a/tests/test_dynamodb2/exceptions/test_dynamodb_exceptions.py b/tests/test_dynamodb/exceptions/test_dynamodb_exceptions.py similarity index 100% rename from tests/test_dynamodb2/exceptions/test_dynamodb_exceptions.py rename to tests/test_dynamodb/exceptions/test_dynamodb_exceptions.py diff --git a/tests/test_dynamodb2/exceptions/test_key_length_exceptions.py b/tests/test_dynamodb/exceptions/test_key_length_exceptions.py similarity index 99% rename from tests/test_dynamodb2/exceptions/test_key_length_exceptions.py rename to tests/test_dynamodb/exceptions/test_key_length_exceptions.py index 12299e7e63bf..666a1d242f66 100644 --- a/tests/test_dynamodb2/exceptions/test_key_length_exceptions.py +++ b/tests/test_dynamodb/exceptions/test_key_length_exceptions.py @@ -6,7 +6,7 @@ from moto import mock_dynamodb2 from botocore.exceptions import ClientError -from moto.dynamodb2.limits import HASH_KEY_MAX_LENGTH, RANGE_KEY_MAX_LENGTH +from moto.dynamodb.limits import HASH_KEY_MAX_LENGTH, RANGE_KEY_MAX_LENGTH @mock_dynamodb2 diff --git a/tests/test_dynamodb/test_dynamodb.py b/tests/test_dynamodb/test_dynamodb.py index fcf165909322..575005b3b9ab 100644 --- a/tests/test_dynamodb/test_dynamodb.py +++ b/tests/test_dynamodb/test_dynamodb.py @@ -1,12 +1,5808 @@ +import uuid +from datetime import datetime +from decimal import Decimal + +import boto3 +from boto3.dynamodb.conditions import Attr, Key +import re import sure # noqa # pylint: disable=unused-import +from moto import mock_dynamodb +from moto.dynamodb import dynamodb_backends +from botocore.exceptions import ClientError + +import moto.dynamodb.comparisons +import moto.dynamodb.models + import pytest -from moto import mock_dynamodb +@mock_dynamodb +@pytest.mark.parametrize( + "names", + [[], ["TestTable"], ["TestTable1", "TestTable2"]], + ids=["no-table", "one-table", "multiple-tables"], +) +def test_list_tables_boto3(names): + conn = boto3.client("dynamodb", region_name="us-west-2") + for name in names: + conn.create_table( + TableName=name, + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + conn.list_tables()["TableNames"].should.equal(names) + + +@mock_dynamodb +def test_list_tables_paginated(): + conn = boto3.client("dynamodb", region_name="us-west-2") + for name in ["name1", "name2", "name3"]: + conn.create_table( + TableName=name, + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + res = conn.list_tables(Limit=2) + res.should.have.key("TableNames").equal(["name1", "name2"]) + res.should.have.key("LastEvaluatedTableName").equal("name2") + + res = conn.list_tables(Limit=1, ExclusiveStartTableName="name1") + res.should.have.key("TableNames").equal(["name2"]) + res.should.have.key("LastEvaluatedTableName").equal("name2") + + res = conn.list_tables(ExclusiveStartTableName="name1") + res.should.have.key("TableNames").equal(["name2", "name3"]) + res.shouldnt.have.key("LastEvaluatedTableName") + + +@mock_dynamodb +def test_describe_missing_table_boto3(): + conn = boto3.client("dynamodb", region_name="us-west-2") + with pytest.raises(ClientError) as ex: + conn.describe_table(TableName="messages") + ex.value.response["Error"]["Code"].should.equal("ResourceNotFoundException") + ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.value.response["Error"]["Message"].should.equal("Requested resource not found") + + +@mock_dynamodb +def test_list_table_tags(): + name = "TestTable" + conn = boto3.client( + "dynamodb", + region_name="us-west-2", + aws_access_key_id="ak", + aws_secret_access_key="sk", + ) + conn.create_table( + TableName=name, + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + table_description = conn.describe_table(TableName=name) + arn = table_description["Table"]["TableArn"] + + # Tag table + tags = [ + {"Key": "TestTag", "Value": "TestValue"}, + {"Key": "TestTag2", "Value": "TestValue2"}, + ] + conn.tag_resource(ResourceArn=arn, Tags=tags) + + # Check tags + resp = conn.list_tags_of_resource(ResourceArn=arn) + assert resp["Tags"] == tags + + # Remove 1 tag + conn.untag_resource(ResourceArn=arn, TagKeys=["TestTag"]) + + # Check tags + resp = conn.list_tags_of_resource(ResourceArn=arn) + assert resp["Tags"] == [{"Key": "TestTag2", "Value": "TestValue2"}] + + +@mock_dynamodb +def test_list_table_tags_empty(): + name = "TestTable" + conn = boto3.client( + "dynamodb", + region_name="us-west-2", + aws_access_key_id="ak", + aws_secret_access_key="sk", + ) + conn.create_table( + TableName=name, + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + table_description = conn.describe_table(TableName=name) + arn = table_description["Table"]["TableArn"] + resp = conn.list_tags_of_resource(ResourceArn=arn) + assert resp["Tags"] == [] + + +@mock_dynamodb +def test_list_table_tags_paginated(): + name = "TestTable" + conn = boto3.client( + "dynamodb", + region_name="us-west-2", + aws_access_key_id="ak", + aws_secret_access_key="sk", + ) + conn.create_table( + TableName=name, + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + table_description = conn.describe_table(TableName=name) + arn = table_description["Table"]["TableArn"] + for i in range(11): + tags = [{"Key": "TestTag%d" % i, "Value": "TestValue"}] + conn.tag_resource(ResourceArn=arn, Tags=tags) + resp = conn.list_tags_of_resource(ResourceArn=arn) + assert len(resp["Tags"]) == 10 + assert "NextToken" in resp.keys() + resp2 = conn.list_tags_of_resource(ResourceArn=arn, NextToken=resp["NextToken"]) + assert len(resp2["Tags"]) == 1 + assert "NextToken" not in resp2.keys() + + +@mock_dynamodb +def test_list_not_found_table_tags(): + conn = boto3.client( + "dynamodb", + region_name="us-west-2", + aws_access_key_id="ak", + aws_secret_access_key="sk", + ) + arn = "DymmyArn" + try: + conn.list_tags_of_resource(ResourceArn=arn) + except ClientError as exception: + assert exception.response["Error"]["Code"] == "ResourceNotFoundException" + + +@mock_dynamodb +def test_item_add_empty_string_hash_key_exception(): + name = "TestTable" + conn = boto3.client( + "dynamodb", + region_name="us-west-2", + aws_access_key_id="ak", + aws_secret_access_key="sk", + ) + conn.create_table( + TableName=name, + KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + with pytest.raises(ClientError) as ex: + conn.put_item( + TableName=name, + Item={ + "forum_name": {"S": ""}, + "subject": {"S": "Check this out!"}, + "Body": {"S": "http://url_to_lolcat.gif"}, + "SentBy": {"S": "someone@somewhere.edu"}, + "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"}, + }, + ) + + ex.value.response["Error"]["Code"].should.equal("ValidationException") + ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.value.response["Error"]["Message"].should.equal( + "One or more parameter values were invalid: An AttributeValue may not contain an empty string" + ) + + +@mock_dynamodb +def test_item_add_empty_string_range_key_exception(): + name = "TestTable" + conn = boto3.client( + "dynamodb", + region_name="us-west-2", + aws_access_key_id="ak", + aws_secret_access_key="sk", + ) + conn.create_table( + TableName=name, + KeySchema=[ + {"AttributeName": "forum_name", "KeyType": "HASH"}, + {"AttributeName": "ReceivedTime", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "forum_name", "AttributeType": "S"}, + {"AttributeName": "ReceivedTime", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + with pytest.raises(ClientError) as ex: + conn.put_item( + TableName=name, + Item={ + "forum_name": {"S": "LOLCat Forum"}, + "subject": {"S": "Check this out!"}, + "Body": {"S": "http://url_to_lolcat.gif"}, + "SentBy": {"S": "someone@somewhere.edu"}, + "ReceivedTime": {"S": ""}, + }, + ) + + ex.value.response["Error"]["Code"].should.equal("ValidationException") + ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.value.response["Error"]["Message"].should.equal( + "One or more parameter values were invalid: An AttributeValue may not contain an empty string" + ) + + +@mock_dynamodb +def test_item_add_empty_string_attr_no_exception(): + name = "TestTable" + conn = boto3.client( + "dynamodb", + region_name="us-west-2", + aws_access_key_id="ak", + aws_secret_access_key="sk", + ) + conn.create_table( + TableName=name, + KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + conn.put_item( + TableName=name, + Item={ + "forum_name": {"S": "LOLCat Forum"}, + "subject": {"S": "Check this out!"}, + "Body": {"S": "http://url_to_lolcat.gif"}, + "SentBy": {"S": ""}, + "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"}, + }, + ) + + +@mock_dynamodb +def test_update_item_with_empty_string_attr_no_exception(): + name = "TestTable" + conn = boto3.client( + "dynamodb", + region_name="us-west-2", + aws_access_key_id="ak", + aws_secret_access_key="sk", + ) + conn.create_table( + TableName=name, + KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + conn.put_item( + TableName=name, + Item={ + "forum_name": {"S": "LOLCat Forum"}, + "subject": {"S": "Check this out!"}, + "Body": {"S": "http://url_to_lolcat.gif"}, + "SentBy": {"S": "test"}, + "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"}, + }, + ) + + conn.update_item( + TableName=name, + Key={"forum_name": {"S": "LOLCat Forum"}}, + UpdateExpression="set Body=:Body", + ExpressionAttributeValues={":Body": {"S": ""}}, + ) + + +@mock_dynamodb +def test_query_invalid_table(): + conn = boto3.client( + "dynamodb", + region_name="us-west-2", + aws_access_key_id="ak", + aws_secret_access_key="sk", + ) + try: + conn.query( + TableName="invalid_table", + KeyConditionExpression="index1 = :partitionkeyval", + ExpressionAttributeValues={":partitionkeyval": {"S": "test"}}, + ) + except ClientError as exception: + assert exception.response["Error"]["Code"] == "ResourceNotFoundException" + + +@mock_dynamodb +def test_put_item_with_special_chars(): + name = "TestTable" + conn = boto3.client( + "dynamodb", + region_name="us-west-2", + aws_access_key_id="ak", + aws_secret_access_key="sk", + ) + + conn.create_table( + TableName=name, + KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + conn.put_item( + TableName=name, + Item={ + "forum_name": {"S": "LOLCat Forum"}, + "subject": {"S": "Check this out!"}, + "Body": {"S": "http://url_to_lolcat.gif"}, + "SentBy": {"S": "test"}, + "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"}, + '"': {"S": "foo"}, + }, + ) + + +@mock_dynamodb +def test_put_item_with_streams(): + name = "TestTable" + conn = boto3.client( + "dynamodb", + region_name="us-west-2", + aws_access_key_id="ak", + aws_secret_access_key="sk", + ) + + conn.create_table( + TableName=name, + KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], + StreamSpecification={ + "StreamEnabled": True, + "StreamViewType": "NEW_AND_OLD_IMAGES", + }, + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + conn.put_item( + TableName=name, + Item={ + "forum_name": {"S": "LOLCat Forum"}, + "subject": {"S": "Check this out!"}, + "Body": {"S": "http://url_to_lolcat.gif"}, + "SentBy": {"S": "test"}, + "Data": {"M": {"Key1": {"S": "Value1"}, "Key2": {"S": "Value2"}}}, + }, + ) + + result = conn.get_item(TableName=name, Key={"forum_name": {"S": "LOLCat Forum"}}) + + result["Item"].should.be.equal( + { + "forum_name": {"S": "LOLCat Forum"}, + "subject": {"S": "Check this out!"}, + "Body": {"S": "http://url_to_lolcat.gif"}, + "SentBy": {"S": "test"}, + "Data": {"M": {"Key1": {"S": "Value1"}, "Key2": {"S": "Value2"}}}, + } + ) + table = dynamodb_backends["us-west-2"].get_table(name) + if not table: + # There is no way to access stream data over the API, so this part can't run in server-tests mode. + return + len(table.stream_shard.items).should.be.equal(1) + stream_record = table.stream_shard.items[0].record + stream_record["eventName"].should.be.equal("INSERT") + stream_record["dynamodb"]["SizeBytes"].should.be.equal(447) + + +@mock_dynamodb +def test_basic_projection_expression_using_get_item(): + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + table = dynamodb.create_table( + TableName="users", + KeySchema=[ + {"AttributeName": "forum_name", "KeyType": "HASH"}, + {"AttributeName": "subject", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "forum_name", "AttributeType": "S"}, + {"AttributeName": "subject", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + table = dynamodb.Table("users") + + table.put_item( + Item={"forum_name": "the-key", "subject": "123", "body": "some test message"} + ) + + table.put_item( + Item={ + "forum_name": "not-the-key", + "subject": "123", + "body": "some other test message", + } + ) + result = table.get_item( + Key={"forum_name": "the-key", "subject": "123"}, + ProjectionExpression="body, subject", + ) + + result["Item"].should.be.equal({"subject": "123", "body": "some test message"}) + + # The projection expression should not remove data from storage + result = table.get_item(Key={"forum_name": "the-key", "subject": "123"}) + + result["Item"].should.be.equal( + {"forum_name": "the-key", "subject": "123", "body": "some test message"} + ) + + +@mock_dynamodb +def test_basic_projection_expressions_using_scan(): + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + dynamodb.create_table( + TableName="users", + KeySchema=[ + {"AttributeName": "forum_name", "KeyType": "HASH"}, + {"AttributeName": "subject", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "forum_name", "AttributeType": "S"}, + {"AttributeName": "subject", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + table = dynamodb.Table("users") + + table.put_item( + Item={"forum_name": "the-key", "subject": "123", "body": "some test message"} + ) + + table.put_item( + Item={ + "forum_name": "not-the-key", + "subject": "123", + "body": "some other test message", + } + ) + # Test a scan returning all items + results = table.scan( + FilterExpression=Key("forum_name").eq("the-key"), + ProjectionExpression="body, subject", + ) + + assert "body" in results["Items"][0] + assert results["Items"][0]["body"] == "some test message" + assert "subject" in results["Items"][0] + + table.put_item( + Item={ + "forum_name": "the-key", + "subject": "1234", + "body": "yet another test message", + } + ) + + results = table.scan( + FilterExpression=Key("forum_name").eq("the-key"), ProjectionExpression="body" + ) + + bodies = [item["body"] for item in results["Items"]] + bodies.should.contain("some test message") + bodies.should.contain("yet another test message") + assert "subject" not in results["Items"][0] + assert "forum_name" not in results["Items"][0] + assert "subject" not in results["Items"][1] + assert "forum_name" not in results["Items"][1] + + # The projection expression should not remove data from storage + results = table.query(KeyConditionExpression=Key("forum_name").eq("the-key")) + assert "subject" in results["Items"][0] + assert "body" in results["Items"][1] + assert "forum_name" in results["Items"][1] + + +@mock_dynamodb +def test_nested_projection_expression_using_get_item(): + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + dynamodb.create_table( + TableName="users", + KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + table = dynamodb.Table("users") + table.put_item( + Item={ + "forum_name": "key1", + "nested": { + "level1": {"id": "id1", "att": "irrelevant"}, + "level2": {"id": "id2", "include": "all"}, + "level3": {"id": "irrelevant"}, + }, + "foo": "bar", + } + ) + table.put_item( + Item={ + "forum_name": "key2", + "nested": {"id": "id2", "incode": "code2"}, + "foo": "bar", + } + ) + + # Test a get_item returning all items + result = table.get_item( + Key={"forum_name": "key1"}, + ProjectionExpression="nested.level1.id, nested.level2", + )["Item"] + result.should.equal( + {"nested": {"level1": {"id": "id1"}, "level2": {"id": "id2", "include": "all"}}} + ) + # Assert actual data has not been deleted + result = table.get_item(Key={"forum_name": "key1"})["Item"] + result.should.equal( + { + "foo": "bar", + "forum_name": "key1", + "nested": { + "level1": {"id": "id1", "att": "irrelevant"}, + "level2": {"id": "id2", "include": "all"}, + "level3": {"id": "irrelevant"}, + }, + } + ) + + +@mock_dynamodb +def test_basic_projection_expressions_using_query(): + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + dynamodb.create_table( + TableName="users", + KeySchema=[ + {"AttributeName": "forum_name", "KeyType": "HASH"}, + {"AttributeName": "subject", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "forum_name", "AttributeType": "S"}, + {"AttributeName": "subject", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + table = dynamodb.Table("users") + table.put_item( + Item={"forum_name": "the-key", "subject": "123", "body": "some test message"} + ) + table.put_item( + Item={ + "forum_name": "not-the-key", + "subject": "123", + "body": "some other test message", + } + ) + + # Test a query returning all items + result = table.query( + KeyConditionExpression=Key("forum_name").eq("the-key"), + ProjectionExpression="body, subject", + )["Items"][0] + + assert "body" in result + assert result["body"] == "some test message" + assert "subject" in result + assert "forum_name" not in result + + table.put_item( + Item={ + "forum_name": "the-key", + "subject": "1234", + "body": "yet another test message", + } + ) + + items = table.query( + KeyConditionExpression=Key("forum_name").eq("the-key"), + ProjectionExpression="body", + )["Items"] + + assert "body" in items[0] + assert "subject" not in items[0] + assert items[0]["body"] == "some test message" + assert "body" in items[1] + assert "subject" not in items[1] + assert items[1]["body"] == "yet another test message" + + # The projection expression should not remove data from storage + items = table.query(KeyConditionExpression=Key("forum_name").eq("the-key"))["Items"] + assert "subject" in items[0] + assert "body" in items[1] + assert "forum_name" in items[1] + + +@mock_dynamodb +def test_nested_projection_expression_using_query(): + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + dynamodb.create_table( + TableName="users", + KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + table = dynamodb.Table("users") + table.put_item( + Item={ + "forum_name": "key1", + "nested": { + "level1": {"id": "id1", "att": "irrelevant"}, + "level2": {"id": "id2", "include": "all"}, + "level3": {"id": "irrelevant"}, + }, + "foo": "bar", + } + ) + table.put_item( + Item={ + "forum_name": "key2", + "nested": {"id": "id2", "incode": "code2"}, + "foo": "bar", + } + ) + + # Test a query returning all items + result = table.query( + KeyConditionExpression=Key("forum_name").eq("key1"), + ProjectionExpression="nested.level1.id, nested.level2", + )["Items"][0] + + assert "nested" in result + result["nested"].should.equal( + {"level1": {"id": "id1"}, "level2": {"id": "id2", "include": "all"}} + ) + assert "foo" not in result + # Assert actual data has not been deleted + result = table.query(KeyConditionExpression=Key("forum_name").eq("key1"))["Items"][ + 0 + ] + result.should.equal( + { + "foo": "bar", + "forum_name": "key1", + "nested": { + "level1": {"id": "id1", "att": "irrelevant"}, + "level2": {"id": "id2", "include": "all"}, + "level3": {"id": "irrelevant"}, + }, + } + ) + + +@mock_dynamodb +def test_nested_projection_expression_using_scan(): + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + dynamodb.create_table( + TableName="users", + KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + table = dynamodb.Table("users") + table.put_item( + Item={ + "forum_name": "key1", + "nested": { + "level1": {"id": "id1", "att": "irrelevant"}, + "level2": {"id": "id2", "include": "all"}, + "level3": {"id": "irrelevant"}, + }, + "foo": "bar", + } + ) + table.put_item( + Item={ + "forum_name": "key2", + "nested": {"id": "id2", "incode": "code2"}, + "foo": "bar", + } + ) + + # Test a scan + results = table.scan( + FilterExpression=Key("forum_name").eq("key1"), + ProjectionExpression="nested.level1.id, nested.level2", + )["Items"] + results.should.equal( + [ + { + "nested": { + "level1": {"id": "id1"}, + "level2": {"include": "all", "id": "id2"}, + } + } + ] + ) + # Assert original data is still there + results = table.scan(FilterExpression=Key("forum_name").eq("key1"))["Items"] + results.should.equal( + [ + { + "forum_name": "key1", + "foo": "bar", + "nested": { + "level1": {"att": "irrelevant", "id": "id1"}, + "level2": {"include": "all", "id": "id2"}, + "level3": {"id": "irrelevant"}, + }, + } + ] + ) + + +@mock_dynamodb +def test_basic_projection_expression_using_get_item_with_attr_expression_names(): + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + table = dynamodb.create_table( + TableName="users", + KeySchema=[ + {"AttributeName": "forum_name", "KeyType": "HASH"}, + {"AttributeName": "subject", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "forum_name", "AttributeType": "S"}, + {"AttributeName": "subject", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + table = dynamodb.Table("users") + + table.put_item( + Item={ + "forum_name": "the-key", + "subject": "123", + "body": "some test message", + "attachment": "something", + } + ) + + table.put_item( + Item={ + "forum_name": "not-the-key", + "subject": "123", + "body": "some other test message", + "attachment": "something", + } + ) + result = table.get_item( + Key={"forum_name": "the-key", "subject": "123"}, + ProjectionExpression="#rl, #rt, subject", + ExpressionAttributeNames={"#rl": "body", "#rt": "attachment"}, + ) + + result["Item"].should.be.equal( + {"subject": "123", "body": "some test message", "attachment": "something"} + ) + + +@mock_dynamodb +def test_basic_projection_expressions_using_query_with_attr_expression_names(): + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + table = dynamodb.create_table( + TableName="users", + KeySchema=[ + {"AttributeName": "forum_name", "KeyType": "HASH"}, + {"AttributeName": "subject", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "forum_name", "AttributeType": "S"}, + {"AttributeName": "subject", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + table = dynamodb.Table("users") + + table.put_item( + Item={ + "forum_name": "the-key", + "subject": "123", + "body": "some test message", + "attachment": "something", + } + ) + + table.put_item( + Item={ + "forum_name": "not-the-key", + "subject": "123", + "body": "some other test message", + "attachment": "something", + } + ) + # Test a query returning all items + + results = table.query( + KeyConditionExpression=Key("forum_name").eq("the-key"), + ProjectionExpression="#rl, #rt, subject", + ExpressionAttributeNames={"#rl": "body", "#rt": "attachment"}, + ) + + assert "body" in results["Items"][0] + assert results["Items"][0]["body"] == "some test message" + assert "subject" in results["Items"][0] + assert results["Items"][0]["subject"] == "123" + assert "attachment" in results["Items"][0] + assert results["Items"][0]["attachment"] == "something" + + +@mock_dynamodb +def test_nested_projection_expression_using_get_item_with_attr_expression(): + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + dynamodb.create_table( + TableName="users", + KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + table = dynamodb.Table("users") + table.put_item( + Item={ + "forum_name": "key1", + "nested": { + "level1": {"id": "id1", "att": "irrelevant"}, + "level2": {"id": "id2", "include": "all"}, + "level3": {"id": "irrelevant"}, + }, + "foo": "bar", + } + ) + table.put_item( + Item={ + "forum_name": "key2", + "nested": {"id": "id2", "incode": "code2"}, + "foo": "bar", + } + ) + + # Test a get_item returning all items + result = table.get_item( + Key={"forum_name": "key1"}, + ProjectionExpression="#nst.level1.id, #nst.#lvl2", + ExpressionAttributeNames={"#nst": "nested", "#lvl2": "level2"}, + )["Item"] + result.should.equal( + {"nested": {"level1": {"id": "id1"}, "level2": {"id": "id2", "include": "all"}}} + ) + # Assert actual data has not been deleted + result = table.get_item(Key={"forum_name": "key1"})["Item"] + result.should.equal( + { + "foo": "bar", + "forum_name": "key1", + "nested": { + "level1": {"id": "id1", "att": "irrelevant"}, + "level2": {"id": "id2", "include": "all"}, + "level3": {"id": "irrelevant"}, + }, + } + ) + + +@mock_dynamodb +def test_nested_projection_expression_using_query_with_attr_expression_names(): + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + dynamodb.create_table( + TableName="users", + KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + table = dynamodb.Table("users") + table.put_item( + Item={ + "forum_name": "key1", + "nested": { + "level1": {"id": "id1", "att": "irrelevant"}, + "level2": {"id": "id2", "include": "all"}, + "level3": {"id": "irrelevant"}, + }, + "foo": "bar", + } + ) + table.put_item( + Item={ + "forum_name": "key2", + "nested": {"id": "id2", "incode": "code2"}, + "foo": "bar", + } + ) + + # Test a query returning all items + result = table.query( + KeyConditionExpression=Key("forum_name").eq("key1"), + ProjectionExpression="#nst.level1.id, #nst.#lvl2", + ExpressionAttributeNames={"#nst": "nested", "#lvl2": "level2"}, + )["Items"][0] + + assert "nested" in result + result["nested"].should.equal( + {"level1": {"id": "id1"}, "level2": {"id": "id2", "include": "all"}} + ) + assert "foo" not in result + # Assert actual data has not been deleted + result = table.query(KeyConditionExpression=Key("forum_name").eq("key1"))["Items"][ + 0 + ] + result.should.equal( + { + "foo": "bar", + "forum_name": "key1", + "nested": { + "level1": {"id": "id1", "att": "irrelevant"}, + "level2": {"id": "id2", "include": "all"}, + "level3": {"id": "irrelevant"}, + }, + } + ) + + +@mock_dynamodb +def test_basic_projection_expressions_using_scan_with_attr_expression_names(): + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + table = dynamodb.create_table( + TableName="users", + KeySchema=[ + {"AttributeName": "forum_name", "KeyType": "HASH"}, + {"AttributeName": "subject", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "forum_name", "AttributeType": "S"}, + {"AttributeName": "subject", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + table = dynamodb.Table("users") + + table.put_item( + Item={ + "forum_name": "the-key", + "subject": "123", + "body": "some test message", + "attachment": "something", + } + ) + + table.put_item( + Item={ + "forum_name": "not-the-key", + "subject": "123", + "body": "some other test message", + "attachment": "something", + } + ) + # Test a scan returning all items + + results = table.scan( + FilterExpression=Key("forum_name").eq("the-key"), + ProjectionExpression="#rl, #rt, subject", + ExpressionAttributeNames={"#rl": "body", "#rt": "attachment"}, + ) + + assert "body" in results["Items"][0] + assert "attachment" in results["Items"][0] + assert "subject" in results["Items"][0] + assert "form_name" not in results["Items"][0] + + # Test without a FilterExpression + results = table.scan( + ProjectionExpression="#rl, #rt, subject", + ExpressionAttributeNames={"#rl": "body", "#rt": "attachment"}, + ) + + assert "body" in results["Items"][0] + assert "attachment" in results["Items"][0] + assert "subject" in results["Items"][0] + assert "form_name" not in results["Items"][0] + + +@mock_dynamodb +def test_nested_projection_expression_using_scan_with_attr_expression_names(): + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + dynamodb.create_table( + TableName="users", + KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + table = dynamodb.Table("users") + table.put_item( + Item={ + "forum_name": "key1", + "nested": { + "level1": {"id": "id1", "att": "irrelevant"}, + "level2": {"id": "id2", "include": "all"}, + "level3": {"id": "irrelevant"}, + }, + "foo": "bar", + } + ) + table.put_item( + Item={ + "forum_name": "key2", + "nested": {"id": "id2", "incode": "code2"}, + "foo": "bar", + } + ) + + # Test a scan + results = table.scan( + FilterExpression=Key("forum_name").eq("key1"), + ProjectionExpression="nested.level1.id, nested.level2", + ExpressionAttributeNames={"#nst": "nested", "#lvl2": "level2"}, + )["Items"] + results.should.equal( + [ + { + "nested": { + "level1": {"id": "id1"}, + "level2": {"include": "all", "id": "id2"}, + } + } + ] + ) + # Assert original data is still there + results = table.scan(FilterExpression=Key("forum_name").eq("key1"))["Items"] + results.should.equal( + [ + { + "forum_name": "key1", + "foo": "bar", + "nested": { + "level1": {"att": "irrelevant", "id": "id1"}, + "level2": {"include": "all", "id": "id2"}, + "level3": {"id": "irrelevant"}, + }, + } + ] + ) + + +@mock_dynamodb +def test_put_empty_item(): + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + dynamodb.create_table( + AttributeDefinitions=[{"AttributeName": "structure_id", "AttributeType": "S"},], + TableName="test", + KeySchema=[{"AttributeName": "structure_id", "KeyType": "HASH"},], + ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, + ) + table = dynamodb.Table("test") + + with pytest.raises(ClientError) as ex: + table.put_item(Item={}) + ex.value.response["Error"]["Message"].should.equal( + "One or more parameter values were invalid: Missing the key structure_id in the item" + ) + ex.value.response["Error"]["Code"].should.equal("ValidationException") + + +@mock_dynamodb +def test_put_item_nonexisting_hash_key(): + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + dynamodb.create_table( + AttributeDefinitions=[{"AttributeName": "structure_id", "AttributeType": "S"},], + TableName="test", + KeySchema=[{"AttributeName": "structure_id", "KeyType": "HASH"},], + ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, + ) + table = dynamodb.Table("test") + + with pytest.raises(ClientError) as ex: + table.put_item(Item={"a_terribly_misguided_id_attribute": "abcdef"}) + ex.value.response["Error"]["Message"].should.equal( + "One or more parameter values were invalid: Missing the key structure_id in the item" + ) + ex.value.response["Error"]["Code"].should.equal("ValidationException") + + +@mock_dynamodb +def test_put_item_nonexisting_range_key(): + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + dynamodb.create_table( + AttributeDefinitions=[ + {"AttributeName": "structure_id", "AttributeType": "S"}, + {"AttributeName": "added_at", "AttributeType": "N"}, + ], + TableName="test", + KeySchema=[ + {"AttributeName": "structure_id", "KeyType": "HASH"}, + {"AttributeName": "added_at", "KeyType": "RANGE"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, + ) + table = dynamodb.Table("test") + + with pytest.raises(ClientError) as ex: + table.put_item(Item={"structure_id": "abcdef"}) + ex.value.response["Error"]["Message"].should.equal( + "One or more parameter values were invalid: Missing the key added_at in the item" + ) + ex.value.response["Error"]["Code"].should.equal("ValidationException") + + +def test_filter_expression(): + row1 = moto.dynamodb.models.Item( + hash_key=None, + range_key=None, + attrs={ + "Id": {"N": "8"}, + "Subs": {"N": "5"}, + "Des": {"S": "Some description"}, + "KV": {"SS": ["test1", "test2"]}, + }, + ) + row2 = moto.dynamodb.models.Item( + hash_key=None, + range_key=None, + attrs={ + "Id": {"N": "8"}, + "Subs": {"N": "10"}, + "Des": {"S": "A description"}, + "KV": {"SS": ["test3", "test4"]}, + }, + ) + + # NOT test 1 + filter_expr = moto.dynamodb.comparisons.get_filter_expression( + "NOT attribute_not_exists(Id)", {}, {} + ) + filter_expr.expr(row1).should.be(True) + + # NOT test 2 + filter_expr = moto.dynamodb.comparisons.get_filter_expression( + "NOT (Id = :v0)", {}, {":v0": {"N": "8"}} + ) + filter_expr.expr(row1).should.be(False) # Id = 8 so should be false + + # AND test + filter_expr = moto.dynamodb.comparisons.get_filter_expression( + "Id > :v0 AND Subs < :v1", {}, {":v0": {"N": "5"}, ":v1": {"N": "7"}} + ) + filter_expr.expr(row1).should.be(True) + filter_expr.expr(row2).should.be(False) + + # lowercase AND test + filter_expr = moto.dynamodb.comparisons.get_filter_expression( + "Id > :v0 and Subs < :v1", {}, {":v0": {"N": "5"}, ":v1": {"N": "7"}} + ) + filter_expr.expr(row1).should.be(True) + filter_expr.expr(row2).should.be(False) + + # OR test + filter_expr = moto.dynamodb.comparisons.get_filter_expression( + "Id = :v0 OR Id=:v1", {}, {":v0": {"N": "5"}, ":v1": {"N": "8"}} + ) + filter_expr.expr(row1).should.be(True) + + # BETWEEN test + filter_expr = moto.dynamodb.comparisons.get_filter_expression( + "Id BETWEEN :v0 AND :v1", {}, {":v0": {"N": "5"}, ":v1": {"N": "10"}} + ) + filter_expr.expr(row1).should.be(True) + + # BETWEEN integer test + filter_expr = moto.dynamodb.comparisons.get_filter_expression( + "Id BETWEEN :v0 AND :v1", {}, {":v0": {"N": "0"}, ":v1": {"N": "10"}} + ) + filter_expr.expr(row1).should.be(True) + + # PAREN test + filter_expr = moto.dynamodb.comparisons.get_filter_expression( + "Id = :v0 AND (Subs = :v0 OR Subs = :v1)", + {}, + {":v0": {"N": "8"}, ":v1": {"N": "5"}}, + ) + filter_expr.expr(row1).should.be(True) + + # IN test + filter_expr = moto.dynamodb.comparisons.get_filter_expression( + "Id IN (:v0, :v1, :v2)", + {}, + {":v0": {"N": "7"}, ":v1": {"N": "8"}, ":v2": {"N": "9"}}, + ) + filter_expr.expr(row1).should.be(True) + + # attribute function tests (with extra spaces) + filter_expr = moto.dynamodb.comparisons.get_filter_expression( + "attribute_exists(Id) AND attribute_not_exists (UnknownAttribute)", {}, {} + ) + filter_expr.expr(row1).should.be(True) + + filter_expr = moto.dynamodb.comparisons.get_filter_expression( + "attribute_type(Id, :v0)", {}, {":v0": {"S": "N"}} + ) + filter_expr.expr(row1).should.be(True) + + # beginswith function test + filter_expr = moto.dynamodb.comparisons.get_filter_expression( + "begins_with(Des, :v0)", {}, {":v0": {"S": "Some"}} + ) + filter_expr.expr(row1).should.be(True) + filter_expr.expr(row2).should.be(False) -def test_deprecation_warning(): - with pytest.warns(None) as record: - mock_dynamodb() - str(record[0].message).should.contain( - "Module mock_dynamodb has been deprecated, and will be repurposed in a later release" + # contains function test + filter_expr = moto.dynamodb.comparisons.get_filter_expression( + "contains(KV, :v0)", {}, {":v0": {"S": "test1"}} + ) + filter_expr.expr(row1).should.be(True) + filter_expr.expr(row2).should.be(False) + + # size function test + filter_expr = moto.dynamodb.comparisons.get_filter_expression( + "size(Des) > size(KV)", {}, {} + ) + filter_expr.expr(row1).should.be(True) + + # Expression from @batkuip + filter_expr = moto.dynamodb.comparisons.get_filter_expression( + "(#n0 < :v0 AND attribute_not_exists(#n1))", + {"#n0": "Subs", "#n1": "fanout_ts"}, + {":v0": {"N": "7"}}, + ) + filter_expr.expr(row1).should.be(True) + # Expression from to check contains on string value + filter_expr = moto.dynamodb.comparisons.get_filter_expression( + "contains(#n0, :v0)", {"#n0": "Des"}, {":v0": {"S": "Some"}} + ) + filter_expr.expr(row1).should.be(True) + filter_expr.expr(row2).should.be(False) + + +@mock_dynamodb +def test_query_filter(): + client = boto3.client("dynamodb", region_name="us-east-1") + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + client.create_table( + TableName="test1", + AttributeDefinitions=[ + {"AttributeName": "client", "AttributeType": "S"}, + {"AttributeName": "app", "AttributeType": "S"}, + ], + KeySchema=[ + {"AttributeName": "client", "KeyType": "HASH"}, + {"AttributeName": "app", "KeyType": "RANGE"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, + ) + client.put_item( + TableName="test1", + Item={ + "client": {"S": "client1"}, + "app": {"S": "app1"}, + "nested": { + "M": { + "version": {"S": "version1"}, + "contents": {"L": [{"S": "value1"}, {"S": "value2"}]}, + } + }, + }, + ) + client.put_item( + TableName="test1", + Item={ + "client": {"S": "client1"}, + "app": {"S": "app2"}, + "nested": { + "M": { + "version": {"S": "version2"}, + "contents": {"L": [{"S": "value1"}, {"S": "value2"}]}, + } + }, + }, + ) + + table = dynamodb.Table("test1") + response = table.query(KeyConditionExpression=Key("client").eq("client1")) + assert response["Count"] == 2 + + response = table.query( + KeyConditionExpression=Key("client").eq("client1"), + FilterExpression=Attr("app").eq("app2"), + ) + assert response["Count"] == 1 + assert response["Items"][0]["app"] == "app2" + response = table.query( + KeyConditionExpression=Key("client").eq("client1"), + FilterExpression=Attr("app").contains("app"), + ) + assert response["Count"] == 2 + + response = table.query( + KeyConditionExpression=Key("client").eq("client1"), + FilterExpression=Attr("nested.version").contains("version"), + ) + assert response["Count"] == 2 + + response = table.query( + KeyConditionExpression=Key("client").eq("client1"), + FilterExpression=Attr("nested.contents[0]").eq("value1"), + ) + assert response["Count"] == 2 + + +@mock_dynamodb +def test_query_filter_overlapping_expression_prefixes(): + client = boto3.client("dynamodb", region_name="us-east-1") + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + client.create_table( + TableName="test1", + AttributeDefinitions=[ + {"AttributeName": "client", "AttributeType": "S"}, + {"AttributeName": "app", "AttributeType": "S"}, + ], + KeySchema=[ + {"AttributeName": "client", "KeyType": "HASH"}, + {"AttributeName": "app", "KeyType": "RANGE"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, + ) + + client.put_item( + TableName="test1", + Item={ + "client": {"S": "client1"}, + "app": {"S": "app1"}, + "nested": { + "M": { + "version": {"S": "version1"}, + "contents": {"L": [{"S": "value1"}, {"S": "value2"}]}, + } + }, + }, + ) + + table = dynamodb.Table("test1") + response = table.query( + KeyConditionExpression=Key("client").eq("client1") & Key("app").eq("app1"), + ProjectionExpression="#1, #10, nested", + ExpressionAttributeNames={"#1": "client", "#10": "app"}, + ) + + assert response["Count"] == 1 + assert response["Items"][0] == { + "client": "client1", + "app": "app1", + "nested": {"version": "version1", "contents": ["value1", "value2"]}, + } + + +@mock_dynamodb +def test_scan_filter(): + client = boto3.client("dynamodb", region_name="us-east-1") + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + client.create_table( + TableName="test1", + AttributeDefinitions=[ + {"AttributeName": "client", "AttributeType": "S"}, + {"AttributeName": "app", "AttributeType": "S"}, + ], + KeySchema=[ + {"AttributeName": "client", "KeyType": "HASH"}, + {"AttributeName": "app", "KeyType": "RANGE"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, + ) + client.put_item( + TableName="test1", Item={"client": {"S": "client1"}, "app": {"S": "app1"}} + ) + + table = dynamodb.Table("test1") + response = table.scan(FilterExpression=Attr("app").eq("app2")) + assert response["Count"] == 0 + + response = table.scan(FilterExpression=Attr("app").eq("app1")) + assert response["Count"] == 1 + + response = table.scan(FilterExpression=Attr("app").ne("app2")) + assert response["Count"] == 1 + + response = table.scan(FilterExpression=Attr("app").ne("app1")) + assert response["Count"] == 0 + + +@mock_dynamodb +def test_scan_filter2(): + client = boto3.client("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + client.create_table( + TableName="test1", + AttributeDefinitions=[ + {"AttributeName": "client", "AttributeType": "S"}, + {"AttributeName": "app", "AttributeType": "N"}, + ], + KeySchema=[ + {"AttributeName": "client", "KeyType": "HASH"}, + {"AttributeName": "app", "KeyType": "RANGE"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, + ) + client.put_item( + TableName="test1", Item={"client": {"S": "client1"}, "app": {"N": "1"}} + ) + + response = client.scan( + TableName="test1", + Select="ALL_ATTRIBUTES", + FilterExpression="#tb >= :dt", + ExpressionAttributeNames={"#tb": "app"}, + ExpressionAttributeValues={":dt": {"N": str(1)}}, + ) + assert response["Count"] == 1 + + +@mock_dynamodb +def test_scan_filter3(): + client = boto3.client("dynamodb", region_name="us-east-1") + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + client.create_table( + TableName="test1", + AttributeDefinitions=[ + {"AttributeName": "client", "AttributeType": "S"}, + {"AttributeName": "app", "AttributeType": "N"}, + ], + KeySchema=[ + {"AttributeName": "client", "KeyType": "HASH"}, + {"AttributeName": "app", "KeyType": "RANGE"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, + ) + client.put_item( + TableName="test1", + Item={"client": {"S": "client1"}, "app": {"N": "1"}, "active": {"BOOL": True}}, + ) + + table = dynamodb.Table("test1") + response = table.scan(FilterExpression=Attr("active").eq(True)) + assert response["Count"] == 1 + + response = table.scan(FilterExpression=Attr("active").ne(True)) + assert response["Count"] == 0 + + response = table.scan(FilterExpression=Attr("active").ne(False)) + assert response["Count"] == 1 + + response = table.scan(FilterExpression=Attr("app").ne(1)) + assert response["Count"] == 0 + + response = table.scan(FilterExpression=Attr("app").ne(2)) + assert response["Count"] == 1 + + +@mock_dynamodb +def test_scan_filter4(): + client = boto3.client("dynamodb", region_name="us-east-1") + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + client.create_table( + TableName="test1", + AttributeDefinitions=[ + {"AttributeName": "client", "AttributeType": "S"}, + {"AttributeName": "app", "AttributeType": "N"}, + ], + KeySchema=[ + {"AttributeName": "client", "KeyType": "HASH"}, + {"AttributeName": "app", "KeyType": "RANGE"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, + ) + + table = dynamodb.Table("test1") + response = table.scan( + FilterExpression=Attr("epoch_ts").lt(7) & Attr("fanout_ts").not_exists() + ) + # Just testing + assert response["Count"] == 0 + + +@mock_dynamodb +def test_scan_filter_should_not_return_non_existing_attributes(): + table_name = "my-table" + item = {"partitionKey": "pk-2", "my-attr": 42} + # Create table + res = boto3.resource("dynamodb", region_name="us-east-1") + res.create_table( + TableName=table_name, + KeySchema=[{"AttributeName": "partitionKey", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "partitionKey", "AttributeType": "S"}], + BillingMode="PAY_PER_REQUEST", + ) + table = res.Table(table_name) + # Insert items + table.put_item(Item={"partitionKey": "pk-1"}) + table.put_item(Item=item) + # Verify a few operations + # Assert we only find the item that has this attribute + table.scan(FilterExpression=Attr("my-attr").lt(43))["Items"].should.equal([item]) + table.scan(FilterExpression=Attr("my-attr").lte(42))["Items"].should.equal([item]) + table.scan(FilterExpression=Attr("my-attr").gte(42))["Items"].should.equal([item]) + table.scan(FilterExpression=Attr("my-attr").gt(41))["Items"].should.equal([item]) + # Sanity check that we can't find the item if the FE is wrong + table.scan(FilterExpression=Attr("my-attr").gt(43))["Items"].should.equal([]) + + +@mock_dynamodb +def test_bad_scan_filter(): + client = boto3.client("dynamodb", region_name="us-east-1") + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + client.create_table( + TableName="test1", + AttributeDefinitions=[ + {"AttributeName": "client", "AttributeType": "S"}, + {"AttributeName": "app", "AttributeType": "S"}, + ], + KeySchema=[ + {"AttributeName": "client", "KeyType": "HASH"}, + {"AttributeName": "app", "KeyType": "RANGE"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, + ) + table = dynamodb.Table("test1") + + # Bad expression + try: + table.scan(FilterExpression="client test") + except ClientError as err: + err.response["Error"]["Code"].should.equal("ValidationError") + else: + raise RuntimeError("Should have raised ResourceInUseException") + + +@mock_dynamodb +def test_duplicate_create(): + client = boto3.client("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + client.create_table( + TableName="test1", + AttributeDefinitions=[ + {"AttributeName": "client", "AttributeType": "S"}, + {"AttributeName": "app", "AttributeType": "S"}, + ], + KeySchema=[ + {"AttributeName": "client", "KeyType": "HASH"}, + {"AttributeName": "app", "KeyType": "RANGE"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, + ) + + try: + client.create_table( + TableName="test1", + AttributeDefinitions=[ + {"AttributeName": "client", "AttributeType": "S"}, + {"AttributeName": "app", "AttributeType": "S"}, + ], + KeySchema=[ + {"AttributeName": "client", "KeyType": "HASH"}, + {"AttributeName": "app", "KeyType": "RANGE"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, + ) + except ClientError as err: + err.response["Error"]["Code"].should.equal("ResourceInUseException") + else: + raise RuntimeError("Should have raised ResourceInUseException") + + +@mock_dynamodb +def test_delete_table(): + client = boto3.client("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + client.create_table( + TableName="test1", + AttributeDefinitions=[ + {"AttributeName": "client", "AttributeType": "S"}, + {"AttributeName": "app", "AttributeType": "S"}, + ], + KeySchema=[ + {"AttributeName": "client", "KeyType": "HASH"}, + {"AttributeName": "app", "KeyType": "RANGE"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, + ) + + client.delete_table(TableName="test1") + + resp = client.list_tables() + len(resp["TableNames"]).should.equal(0) + + try: + client.delete_table(TableName="test1") + except ClientError as err: + err.response["Error"]["Code"].should.equal("ResourceNotFoundException") + else: + raise RuntimeError("Should have raised ResourceNotFoundException") + + +@mock_dynamodb +def test_delete_item(): + client = boto3.client("dynamodb", region_name="us-east-1") + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + client.create_table( + TableName="test1", + AttributeDefinitions=[ + {"AttributeName": "client", "AttributeType": "S"}, + {"AttributeName": "app", "AttributeType": "S"}, + ], + KeySchema=[ + {"AttributeName": "client", "KeyType": "HASH"}, + {"AttributeName": "app", "KeyType": "RANGE"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, + ) + client.put_item( + TableName="test1", Item={"client": {"S": "client1"}, "app": {"S": "app1"}} + ) + client.put_item( + TableName="test1", Item={"client": {"S": "client1"}, "app": {"S": "app2"}} + ) + + table = dynamodb.Table("test1") + response = table.scan() + assert response["Count"] == 2 + + # Test ReturnValues validation + with pytest.raises(ClientError) as ex: + table.delete_item( + Key={"client": "client1", "app": "app1"}, ReturnValues="ALL_NEW" + ) + err = ex.value.response["Error"] + err["Code"].should.equal("ValidationException") + err["Message"].should.equal("Return values set to invalid value") + + # Test deletion and returning old value + response = table.delete_item( + Key={"client": "client1", "app": "app1"}, ReturnValues="ALL_OLD" + ) + response["Attributes"].should.contain("client") + response["Attributes"].should.contain("app") + + response = table.scan() + assert response["Count"] == 1 + + # Test deletion returning nothing + response = table.delete_item(Key={"client": "client1", "app": "app2"}) + len(response["Attributes"]).should.equal(0) + + response = table.scan() + assert response["Count"] == 0 + + +@mock_dynamodb +def test_describe_limits(): + client = boto3.client("dynamodb", region_name="eu-central-1") + resp = client.describe_limits() + + resp["AccountMaxReadCapacityUnits"].should.equal(20000) + resp["AccountMaxWriteCapacityUnits"].should.equal(20000) + resp["TableMaxWriteCapacityUnits"].should.equal(10000) + resp["TableMaxReadCapacityUnits"].should.equal(10000) + + +@mock_dynamodb +def test_set_ttl(): + client = boto3.client("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + client.create_table( + TableName="test1", + AttributeDefinitions=[ + {"AttributeName": "client", "AttributeType": "S"}, + {"AttributeName": "app", "AttributeType": "S"}, + ], + KeySchema=[ + {"AttributeName": "client", "KeyType": "HASH"}, + {"AttributeName": "app", "KeyType": "RANGE"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, + ) + + client.update_time_to_live( + TableName="test1", + TimeToLiveSpecification={"Enabled": True, "AttributeName": "expire"}, + ) + + resp = client.describe_time_to_live(TableName="test1") + resp["TimeToLiveDescription"]["TimeToLiveStatus"].should.equal("ENABLED") + resp["TimeToLiveDescription"]["AttributeName"].should.equal("expire") + + client.update_time_to_live( + TableName="test1", + TimeToLiveSpecification={"Enabled": False, "AttributeName": "expire"}, + ) + + resp = client.describe_time_to_live(TableName="test1") + resp["TimeToLiveDescription"]["TimeToLiveStatus"].should.equal("DISABLED") + + +@mock_dynamodb +def test_describe_continuous_backups(): + # given + client = boto3.client("dynamodb", region_name="us-east-1") + table_name = client.create_table( + TableName="test", + AttributeDefinitions=[ + {"AttributeName": "client", "AttributeType": "S"}, + {"AttributeName": "app", "AttributeType": "S"}, + ], + KeySchema=[ + {"AttributeName": "client", "KeyType": "HASH"}, + {"AttributeName": "app", "KeyType": "RANGE"}, + ], + BillingMode="PAY_PER_REQUEST", + )["TableDescription"]["TableName"] + + # when + response = client.describe_continuous_backups(TableName=table_name) + + # then + response["ContinuousBackupsDescription"].should.equal( + { + "ContinuousBackupsStatus": "ENABLED", + "PointInTimeRecoveryDescription": {"PointInTimeRecoveryStatus": "DISABLED"}, + } + ) + + +@mock_dynamodb +def test_describe_continuous_backups_errors(): + # given + client = boto3.client("dynamodb", region_name="us-east-1") + + # when + with pytest.raises(Exception) as e: + client.describe_continuous_backups(TableName="not-existing-table") + + # then + ex = e.value + ex.operation_name.should.equal("DescribeContinuousBackups") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("TableNotFoundException") + ex.response["Error"]["Message"].should.equal("Table not found: not-existing-table") + + +@mock_dynamodb +def test_update_continuous_backups(): + # given + client = boto3.client("dynamodb", region_name="us-east-1") + table_name = client.create_table( + TableName="test", + AttributeDefinitions=[ + {"AttributeName": "client", "AttributeType": "S"}, + {"AttributeName": "app", "AttributeType": "S"}, + ], + KeySchema=[ + {"AttributeName": "client", "KeyType": "HASH"}, + {"AttributeName": "app", "KeyType": "RANGE"}, + ], + BillingMode="PAY_PER_REQUEST", + )["TableDescription"]["TableName"] + + # when + response = client.update_continuous_backups( + TableName=table_name, + PointInTimeRecoverySpecification={"PointInTimeRecoveryEnabled": True}, + ) + + # then + response["ContinuousBackupsDescription"]["ContinuousBackupsStatus"].should.equal( + "ENABLED" + ) + point_in_time = response["ContinuousBackupsDescription"][ + "PointInTimeRecoveryDescription" + ] + earliest_datetime = point_in_time["EarliestRestorableDateTime"] + earliest_datetime.should.be.a(datetime) + latest_datetime = point_in_time["LatestRestorableDateTime"] + latest_datetime.should.be.a(datetime) + point_in_time["PointInTimeRecoveryStatus"].should.equal("ENABLED") + + # when + # a second update should not change anything + response = client.update_continuous_backups( + TableName=table_name, + PointInTimeRecoverySpecification={"PointInTimeRecoveryEnabled": True}, + ) + + # then + response["ContinuousBackupsDescription"]["ContinuousBackupsStatus"].should.equal( + "ENABLED" + ) + point_in_time = response["ContinuousBackupsDescription"][ + "PointInTimeRecoveryDescription" + ] + point_in_time["EarliestRestorableDateTime"].should.equal(earliest_datetime) + point_in_time["LatestRestorableDateTime"].should.equal(latest_datetime) + point_in_time["PointInTimeRecoveryStatus"].should.equal("ENABLED") + + # when + response = client.update_continuous_backups( + TableName=table_name, + PointInTimeRecoverySpecification={"PointInTimeRecoveryEnabled": False}, + ) + + # then + response["ContinuousBackupsDescription"].should.equal( + { + "ContinuousBackupsStatus": "ENABLED", + "PointInTimeRecoveryDescription": {"PointInTimeRecoveryStatus": "DISABLED"}, + } + ) + + +@mock_dynamodb +def test_update_continuous_backups_errors(): + # given + client = boto3.client("dynamodb", region_name="us-east-1") + + # when + with pytest.raises(Exception) as e: + client.update_continuous_backups( + TableName="not-existing-table", + PointInTimeRecoverySpecification={"PointInTimeRecoveryEnabled": True}, + ) + + # then + ex = e.value + ex.operation_name.should.equal("UpdateContinuousBackups") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("TableNotFoundException") + ex.response["Error"]["Message"].should.equal("Table not found: not-existing-table") + + +# https://github.com/spulec/moto/issues/1043 +@mock_dynamodb +def test_query_missing_expr_names(): + client = boto3.client("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + client.create_table( + TableName="test1", + AttributeDefinitions=[ + {"AttributeName": "client", "AttributeType": "S"}, + {"AttributeName": "app", "AttributeType": "S"}, + ], + KeySchema=[ + {"AttributeName": "client", "KeyType": "HASH"}, + {"AttributeName": "app", "KeyType": "RANGE"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, + ) + client.put_item( + TableName="test1", Item={"client": {"S": "test1"}, "app": {"S": "test1"}} + ) + client.put_item( + TableName="test1", Item={"client": {"S": "test2"}, "app": {"S": "test2"}} + ) + + resp = client.query( + TableName="test1", + KeyConditionExpression="client=:client", + ExpressionAttributeValues={":client": {"S": "test1"}}, + ) + + resp["Count"].should.equal(1) + resp["Items"][0]["client"]["S"].should.equal("test1") + + resp = client.query( + TableName="test1", + KeyConditionExpression=":name=test2", + ExpressionAttributeNames={":name": "client"}, + ) + + resp["Count"].should.equal(1) + resp["Items"][0]["client"]["S"].should.equal("test2") + + +# https://github.com/spulec/moto/issues/2328 +@mock_dynamodb +def test_update_item_with_list(): + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + dynamodb.create_table( + TableName="Table", + KeySchema=[{"AttributeName": "key", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "key", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, + ) + table = dynamodb.Table("Table") + table.update_item( + Key={"key": "the-key"}, + AttributeUpdates={"list": {"Value": [1, 2], "Action": "PUT"}}, + ) + + resp = table.get_item(Key={"key": "the-key"}) + resp["Item"].should.equal({"key": "the-key", "list": [1, 2]}) + + +# https://github.com/spulec/moto/issues/2328 +@mock_dynamodb +def test_update_item_with_no_action_passed_with_list(): + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + dynamodb.create_table( + TableName="Table", + KeySchema=[{"AttributeName": "key", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "key", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, + ) + table = dynamodb.Table("Table") + table.update_item( + Key={"key": "the-key"}, + # Do not pass 'Action' key, in order to check that the + # parameter's default value will be used. + AttributeUpdates={"list": {"Value": [1, 2]}}, + ) + + resp = table.get_item(Key={"key": "the-key"}) + resp["Item"].should.equal({"key": "the-key", "list": [1, 2]}) + + +# https://github.com/spulec/moto/issues/1342 +@mock_dynamodb +def test_update_item_on_map(): + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + client = boto3.client("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + dynamodb.create_table( + TableName="users", + KeySchema=[ + {"AttributeName": "forum_name", "KeyType": "HASH"}, + {"AttributeName": "subject", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "forum_name", "AttributeType": "S"}, + {"AttributeName": "subject", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + table = dynamodb.Table("users") + + table.put_item( + Item={ + "forum_name": "the-key", + "subject": "123", + "body": {"nested": {"data": "test"}}, + } + ) + + resp = table.scan() + resp["Items"][0]["body"].should.equal({"nested": {"data": "test"}}) + + # Nonexistent nested attributes are supported for existing top-level attributes. + table.update_item( + Key={"forum_name": "the-key", "subject": "123"}, + UpdateExpression="SET body.#nested.#data = :tb", + ExpressionAttributeNames={"#nested": "nested", "#data": "data",}, + ExpressionAttributeValues={":tb": "new_value"}, + ) + # Running this against AWS DDB gives an exception so make sure it also fails.: + with pytest.raises(client.exceptions.ClientError): + # botocore.exceptions.ClientError: An error occurred (ValidationException) when calling the UpdateItem + # operation: The document path provided in the update expression is invalid for update + table.update_item( + Key={"forum_name": "the-key", "subject": "123"}, + UpdateExpression="SET body.#nested.#nonexistentnested.#data = :tb2", + ExpressionAttributeNames={ + "#nested": "nested", + "#nonexistentnested": "nonexistentnested", + "#data": "data", + }, + ExpressionAttributeValues={":tb2": "other_value"}, + ) + + table.update_item( + Key={"forum_name": "the-key", "subject": "123"}, + UpdateExpression="SET body.#nested.#nonexistentnested = :tb2", + ExpressionAttributeNames={ + "#nested": "nested", + "#nonexistentnested": "nonexistentnested", + }, + ExpressionAttributeValues={":tb2": {"data": "other_value"}}, + ) + + resp = table.scan() + resp["Items"][0]["body"].should.equal( + {"nested": {"data": "new_value", "nonexistentnested": {"data": "other_value"}}} + ) + + # Test nested value for a nonexistent attribute throws a ClientError. + with pytest.raises(client.exceptions.ClientError): + table.update_item( + Key={"forum_name": "the-key", "subject": "123"}, + UpdateExpression="SET nonexistent.#nested = :tb", + ExpressionAttributeNames={"#nested": "nested"}, + ExpressionAttributeValues={":tb": "new_value"}, + ) + + +# https://github.com/spulec/moto/issues/1358 +@mock_dynamodb +def test_update_if_not_exists(): + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + dynamodb.create_table( + TableName="users", + KeySchema=[ + {"AttributeName": "forum_name", "KeyType": "HASH"}, + {"AttributeName": "subject", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "forum_name", "AttributeType": "S"}, + {"AttributeName": "subject", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + table = dynamodb.Table("users") + + table.put_item(Item={"forum_name": "the-key", "subject": "123"}) + + table.update_item( + Key={"forum_name": "the-key", "subject": "123"}, + # if_not_exists without space + UpdateExpression="SET created_at=if_not_exists(created_at,:created_at)", + ExpressionAttributeValues={":created_at": 123}, + ) + + resp = table.scan() + assert resp["Items"][0]["created_at"] == 123 + + table.update_item( + Key={"forum_name": "the-key", "subject": "123"}, + # if_not_exists with space + UpdateExpression="SET created_at = if_not_exists (created_at, :created_at)", + ExpressionAttributeValues={":created_at": 456}, + ) + + resp = table.scan() + # Still the original value + assert resp["Items"][0]["created_at"] == 123 + + +# https://github.com/spulec/moto/issues/1937 +@mock_dynamodb +def test_update_return_attributes(): + dynamodb = boto3.client("dynamodb", region_name="us-east-1") + + dynamodb.create_table( + TableName="moto-test", + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, + ) + + def update(col, to, rv): + return dynamodb.update_item( + TableName="moto-test", + Key={"id": {"S": "foo"}}, + AttributeUpdates={col: {"Value": {"S": to}, "Action": "PUT"}}, + ReturnValues=rv, + ) + + r = update("col1", "val1", "ALL_NEW") + assert r["Attributes"] == {"id": {"S": "foo"}, "col1": {"S": "val1"}} + + r = update("col1", "val2", "ALL_OLD") + assert r["Attributes"] == {"id": {"S": "foo"}, "col1": {"S": "val1"}} + + r = update("col2", "val3", "UPDATED_NEW") + assert r["Attributes"] == {"col2": {"S": "val3"}} + + r = update("col2", "val4", "UPDATED_OLD") + assert r["Attributes"] == {"col2": {"S": "val3"}} + + r = update("col1", "val5", "NONE") + assert r["Attributes"] == {} + + with pytest.raises(ClientError) as ex: + update("col1", "val6", "WRONG") + err = ex.value.response["Error"] + err["Code"].should.equal("ValidationException") + err["Message"].should.equal("Return values set to invalid value") + + +# https://github.com/spulec/moto/issues/3448 +@mock_dynamodb +def test_update_return_updated_new_attributes_when_same(): + dynamo_client = boto3.resource("dynamodb", region_name="us-east-1") + dynamo_client.create_table( + TableName="moto-test", + KeySchema=[{"AttributeName": "HashKey1", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "HashKey1", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, + ) + + dynamodb_table = dynamo_client.Table("moto-test") + dynamodb_table.put_item( + Item={"HashKey1": "HashKeyValue1", "listValuedAttribute1": ["a", "b"]} + ) + + def update(col, to, rv): + return dynamodb_table.update_item( + TableName="moto-test", + Key={"HashKey1": "HashKeyValue1"}, + UpdateExpression="SET listValuedAttribute1=:" + col, + ExpressionAttributeValues={":" + col: to}, + ReturnValues=rv, + ) + + r = update("a", ["a", "c"], "UPDATED_NEW") + assert r["Attributes"] == {"listValuedAttribute1": ["a", "c"]} + + r = update("a", {"a", "c"}, "UPDATED_NEW") + assert r["Attributes"] == {"listValuedAttribute1": {"a", "c"}} + + r = update("a", {1, 2}, "UPDATED_NEW") + assert r["Attributes"] == {"listValuedAttribute1": {1, 2}} + + with pytest.raises(ClientError) as ex: + update("a", ["a", "c"], "WRONG") + err = ex.value.response["Error"] + err["Code"].should.equal("ValidationException") + err["Message"].should.equal("Return values set to invalid value") + + +@mock_dynamodb +def test_put_return_attributes(): + dynamodb = boto3.client("dynamodb", region_name="us-east-1") + + dynamodb.create_table( + TableName="moto-test", + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, + ) + + r = dynamodb.put_item( + TableName="moto-test", + Item={"id": {"S": "foo"}, "col1": {"S": "val1"}}, + ReturnValues="NONE", + ) + assert "Attributes" not in r + + r = dynamodb.put_item( + TableName="moto-test", + Item={"id": {"S": "foo"}, "col1": {"S": "val2"}}, + ReturnValues="ALL_OLD", + ) + assert r["Attributes"] == {"id": {"S": "foo"}, "col1": {"S": "val1"}} + + with pytest.raises(ClientError) as ex: + dynamodb.put_item( + TableName="moto-test", + Item={"id": {"S": "foo"}, "col1": {"S": "val3"}}, + ReturnValues="ALL_NEW", + ) + ex.value.response["Error"]["Code"].should.equal("ValidationException") + ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.value.response["Error"]["Message"].should.equal( + "Return values set to invalid value" + ) + + +@mock_dynamodb +def test_query_global_secondary_index_when_created_via_update_table_resource(): + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + + # Create the DynamoDB table. + dynamodb.create_table( + TableName="users", + KeySchema=[{"AttributeName": "user_id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "user_id", "AttributeType": "N"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + table = dynamodb.Table("users") + table.update( + AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], + GlobalSecondaryIndexUpdates=[ + { + "Create": { + "IndexName": "forum_name_index", + "KeySchema": [{"AttributeName": "forum_name", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "ALL"}, + "ProvisionedThroughput": { + "ReadCapacityUnits": 5, + "WriteCapacityUnits": 5, + }, + } + } + ], + ) + + next_user_id = 1 + for my_forum_name in ["cats", "dogs"]: + for my_subject in [ + "my pet is the cutest", + "wow look at what my pet did", + "don't you love my pet?", + ]: + table.put_item( + Item={ + "user_id": next_user_id, + "forum_name": my_forum_name, + "subject": my_subject, + } + ) + next_user_id += 1 + + # get all the cat users + forum_only_query_response = table.query( + IndexName="forum_name_index", + Select="ALL_ATTRIBUTES", + KeyConditionExpression=Key("forum_name").eq("cats"), + ) + forum_only_items = forum_only_query_response["Items"] + assert len(forum_only_items) == 3 + for item in forum_only_items: + assert item["forum_name"] == "cats" + + # query all cat users with a particular subject + forum_and_subject_query_results = table.query( + IndexName="forum_name_index", + Select="ALL_ATTRIBUTES", + KeyConditionExpression=Key("forum_name").eq("cats"), + FilterExpression=Attr("subject").eq("my pet is the cutest"), + ) + forum_and_subject_items = forum_and_subject_query_results["Items"] + assert len(forum_and_subject_items) == 1 + assert forum_and_subject_items[0] == { + "user_id": Decimal("1"), + "forum_name": "cats", + "subject": "my pet is the cutest", + } + + +@mock_dynamodb +def test_query_gsi_with_range_key(): + dynamodb = boto3.client("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="test", + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[ + {"AttributeName": "id", "AttributeType": "S"}, + {"AttributeName": "gsi_hash_key", "AttributeType": "S"}, + {"AttributeName": "gsi_range_key", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, + GlobalSecondaryIndexes=[ + { + "IndexName": "test_gsi", + "KeySchema": [ + {"AttributeName": "gsi_hash_key", "KeyType": "HASH"}, + {"AttributeName": "gsi_range_key", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "ALL"}, + "ProvisionedThroughput": { + "ReadCapacityUnits": 1, + "WriteCapacityUnits": 1, + }, + } + ], + ) + + dynamodb.put_item( + TableName="test", + Item={ + "id": {"S": "test1"}, + "gsi_hash_key": {"S": "key1"}, + "gsi_range_key": {"S": "range1"}, + }, + ) + dynamodb.put_item( + TableName="test", Item={"id": {"S": "test2"}, "gsi_hash_key": {"S": "key1"}} + ) + + res = dynamodb.query( + TableName="test", + IndexName="test_gsi", + KeyConditionExpression="gsi_hash_key = :gsi_hash_key and gsi_range_key = :gsi_range_key", + ExpressionAttributeValues={ + ":gsi_hash_key": {"S": "key1"}, + ":gsi_range_key": {"S": "range1"}, + }, + ) + res.should.have.key("Count").equal(1) + res.should.have.key("Items") + res["Items"][0].should.equal( + { + "id": {"S": "test1"}, + "gsi_hash_key": {"S": "key1"}, + "gsi_range_key": {"S": "range1"}, + } + ) + + +@mock_dynamodb +def test_scan_by_non_exists_index(): + dynamodb = boto3.client("dynamodb", region_name="us-east-1") + + dynamodb.create_table( + TableName="test", + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[ + {"AttributeName": "id", "AttributeType": "S"}, + {"AttributeName": "gsi_col", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, + GlobalSecondaryIndexes=[ + { + "IndexName": "test_gsi", + "KeySchema": [{"AttributeName": "gsi_col", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "ALL"}, + "ProvisionedThroughput": { + "ReadCapacityUnits": 1, + "WriteCapacityUnits": 1, + }, + } + ], + ) + + with pytest.raises(ClientError) as ex: + dynamodb.scan(TableName="test", IndexName="non_exists_index") + + ex.value.response["Error"]["Code"].should.equal("ValidationException") + ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.value.response["Error"]["Message"].should.equal( + "The table does not have the specified index: non_exists_index" + ) + + +@mock_dynamodb +def test_query_by_non_exists_index(): + dynamodb = boto3.client("dynamodb", region_name="us-east-1") + + dynamodb.create_table( + TableName="test", + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[ + {"AttributeName": "id", "AttributeType": "S"}, + {"AttributeName": "gsi_col", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, + GlobalSecondaryIndexes=[ + { + "IndexName": "test_gsi", + "KeySchema": [{"AttributeName": "gsi_col", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "ALL"}, + "ProvisionedThroughput": { + "ReadCapacityUnits": 1, + "WriteCapacityUnits": 1, + }, + } + ], + ) + + with pytest.raises(ClientError) as ex: + dynamodb.query( + TableName="test", + IndexName="non_exists_index", + KeyConditionExpression="CarModel=M", + ) + + ex.value.response["Error"]["Code"].should.equal("ResourceNotFoundException") + ex.value.response["Error"]["Message"].should.equal( + "Invalid index: non_exists_index for table: test. Available indexes are: test_gsi" + ) + + +@mock_dynamodb +def test_batch_items_returns_all(): + dynamodb = _create_user_table() + returned_items = dynamodb.batch_get_item( + RequestItems={ + "users": { + "Keys": [ + {"username": {"S": "user0"}}, + {"username": {"S": "user1"}}, + {"username": {"S": "user2"}}, + {"username": {"S": "user3"}}, + ], + "ConsistentRead": True, + } + } + )["Responses"]["users"] + assert len(returned_items) == 3 + assert [item["username"]["S"] for item in returned_items] == [ + "user1", + "user2", + "user3", + ] + + +@mock_dynamodb +def test_batch_items_throws_exception_when_requesting_100_items_for_single_table(): + dynamodb = _create_user_table() + with pytest.raises(ClientError) as ex: + dynamodb.batch_get_item( + RequestItems={ + "users": { + "Keys": [ + {"username": {"S": "user" + str(i)}} for i in range(0, 104) + ], + "ConsistentRead": True, + } + } + ) + ex.value.response["Error"]["Code"].should.equal("ValidationException") + msg = ex.value.response["Error"]["Message"] + msg.should.contain("1 validation error detected: Value") + msg.should.contain( + "at 'requestItems.users.member.keys' failed to satisfy constraint: Member must have length less than or equal to 100" + ) + + +@mock_dynamodb +def test_batch_items_throws_exception_when_requesting_100_items_across_all_tables(): + dynamodb = _create_user_table() + with pytest.raises(ClientError) as ex: + dynamodb.batch_get_item( + RequestItems={ + "users": { + "Keys": [ + {"username": {"S": "user" + str(i)}} for i in range(0, 75) + ], + "ConsistentRead": True, + }, + "users2": { + "Keys": [ + {"username": {"S": "user" + str(i)}} for i in range(0, 75) + ], + "ConsistentRead": True, + }, + } + ) + ex.value.response["Error"]["Code"].should.equal("ValidationException") + ex.value.response["Error"]["Message"].should.equal( + "Too many items requested for the BatchGetItem call" + ) + + +@mock_dynamodb +def test_batch_items_with_basic_projection_expression(): + dynamodb = _create_user_table() + returned_items = dynamodb.batch_get_item( + RequestItems={ + "users": { + "Keys": [ + {"username": {"S": "user0"}}, + {"username": {"S": "user1"}}, + {"username": {"S": "user2"}}, + {"username": {"S": "user3"}}, + ], + "ConsistentRead": True, + "ProjectionExpression": "username", + } + } + )["Responses"]["users"] + + returned_items.should.have.length_of(3) + [item["username"]["S"] for item in returned_items].should.be.equal( + ["user1", "user2", "user3"] + ) + [item.get("foo") for item in returned_items].should.be.equal([None, None, None]) + + # The projection expression should not remove data from storage + returned_items = dynamodb.batch_get_item( + RequestItems={ + "users": { + "Keys": [ + {"username": {"S": "user0"}}, + {"username": {"S": "user1"}}, + {"username": {"S": "user2"}}, + {"username": {"S": "user3"}}, + ], + "ConsistentRead": True, + } + } + )["Responses"]["users"] + + [item["username"]["S"] for item in returned_items].should.be.equal( + ["user1", "user2", "user3"] + ) + [item["foo"]["S"] for item in returned_items].should.be.equal(["bar", "bar", "bar"]) + + +@mock_dynamodb +def test_batch_items_with_basic_projection_expression_and_attr_expression_names(): + dynamodb = _create_user_table() + returned_items = dynamodb.batch_get_item( + RequestItems={ + "users": { + "Keys": [ + {"username": {"S": "user0"}}, + {"username": {"S": "user1"}}, + {"username": {"S": "user2"}}, + {"username": {"S": "user3"}}, + ], + "ConsistentRead": True, + "ProjectionExpression": "#rl", + "ExpressionAttributeNames": {"#rl": "username"}, + } + } + )["Responses"]["users"] + + returned_items.should.have.length_of(3) + [item["username"]["S"] for item in returned_items].should.be.equal( + ["user1", "user2", "user3"] + ) + [item.get("foo") for item in returned_items].should.be.equal([None, None, None]) + + +@mock_dynamodb +def test_batch_items_should_throw_exception_for_duplicate_request(): + client = _create_user_table() + with pytest.raises(ClientError) as ex: + client.batch_get_item( + RequestItems={ + "users": { + "Keys": [ + {"username": {"S": "user0"}}, + {"username": {"S": "user0"}}, + ], + "ConsistentRead": True, + } + } + ) + ex.value.response["Error"]["Code"].should.equal("ValidationException") + ex.value.response["Error"]["Message"].should.equal( + "Provided list of item keys contains duplicates" + ) + + +@mock_dynamodb +def test_index_with_unknown_attributes_should_fail(): + dynamodb = boto3.client("dynamodb", region_name="us-east-1") + + expected_exception = ( + "Some index key attributes are not defined in AttributeDefinitions." + ) + + with pytest.raises(ClientError) as ex: + dynamodb.create_table( + AttributeDefinitions=[ + {"AttributeName": "customer_nr", "AttributeType": "S"}, + {"AttributeName": "last_name", "AttributeType": "S"}, + ], + TableName="table_with_missing_attribute_definitions", + KeySchema=[ + {"AttributeName": "customer_nr", "KeyType": "HASH"}, + {"AttributeName": "last_name", "KeyType": "RANGE"}, + ], + LocalSecondaryIndexes=[ + { + "IndexName": "indexthataddsanadditionalattribute", + "KeySchema": [ + {"AttributeName": "customer_nr", "KeyType": "HASH"}, + {"AttributeName": "postcode", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "ALL"}, + } + ], + BillingMode="PAY_PER_REQUEST", + ) + + ex.value.response["Error"]["Code"].should.equal("ValidationException") + ex.value.response["Error"]["Message"].should.contain(expected_exception) + + +@mock_dynamodb +def test_update_list_index__set_existing_index(): + table_name = "test_list_index_access" + client = create_table_with_list(table_name) + client.put_item( + TableName=table_name, + Item={ + "id": {"S": "foo"}, + "itemlist": {"L": [{"S": "bar1"}, {"S": "bar2"}, {"S": "bar3"}]}, + }, + ) + client.update_item( + TableName=table_name, + Key={"id": {"S": "foo"}}, + UpdateExpression="set itemlist[1]=:Item", + ExpressionAttributeValues={":Item": {"S": "bar2_update"}}, + ) + # + result = client.get_item(TableName=table_name, Key={"id": {"S": "foo"}})["Item"] + result["id"].should.equal({"S": "foo"}) + result["itemlist"].should.equal( + {"L": [{"S": "bar1"}, {"S": "bar2_update"}, {"S": "bar3"}]} + ) + + +@mock_dynamodb +def test_update_list_index__set_existing_nested_index(): + table_name = "test_list_index_access" + client = create_table_with_list(table_name) + client.put_item( + TableName=table_name, + Item={ + "id": {"S": "foo2"}, + "itemmap": { + "M": {"itemlist": {"L": [{"S": "bar1"}, {"S": "bar2"}, {"S": "bar3"}]}} + }, + }, + ) + client.update_item( + TableName=table_name, + Key={"id": {"S": "foo2"}}, + UpdateExpression="set itemmap.itemlist[1]=:Item", + ExpressionAttributeValues={":Item": {"S": "bar2_update"}}, + ) + # + result = client.get_item(TableName=table_name, Key={"id": {"S": "foo2"}})["Item"] + result["id"].should.equal({"S": "foo2"}) + result["itemmap"]["M"]["itemlist"]["L"].should.equal( + [{"S": "bar1"}, {"S": "bar2_update"}, {"S": "bar3"}] + ) + + +@mock_dynamodb +def test_update_list_index__set_index_out_of_range(): + table_name = "test_list_index_access" + client = create_table_with_list(table_name) + client.put_item( + TableName=table_name, + Item={ + "id": {"S": "foo"}, + "itemlist": {"L": [{"S": "bar1"}, {"S": "bar2"}, {"S": "bar3"}]}, + }, + ) + client.update_item( + TableName=table_name, + Key={"id": {"S": "foo"}}, + UpdateExpression="set itemlist[10]=:Item", + ExpressionAttributeValues={":Item": {"S": "bar10"}}, + ) + # + result = client.get_item(TableName=table_name, Key={"id": {"S": "foo"}})["Item"] + assert result["id"] == {"S": "foo"} + assert result["itemlist"] == { + "L": [{"S": "bar1"}, {"S": "bar2"}, {"S": "bar3"}, {"S": "bar10"}] + } + + +@mock_dynamodb +def test_update_list_index__set_nested_index_out_of_range(): + table_name = "test_list_index_access" + client = create_table_with_list(table_name) + client.put_item( + TableName=table_name, + Item={ + "id": {"S": "foo2"}, + "itemmap": { + "M": {"itemlist": {"L": [{"S": "bar1"}, {"S": "bar2"}, {"S": "bar3"}]}} + }, + }, + ) + client.update_item( + TableName=table_name, + Key={"id": {"S": "foo2"}}, + UpdateExpression="set itemmap.itemlist[10]=:Item", + ExpressionAttributeValues={":Item": {"S": "bar10"}}, + ) + # + result = client.get_item(TableName=table_name, Key={"id": {"S": "foo2"}})["Item"] + assert result["id"] == {"S": "foo2"} + assert result["itemmap"]["M"]["itemlist"]["L"] == [ + {"S": "bar1"}, + {"S": "bar2"}, + {"S": "bar3"}, + {"S": "bar10"}, + ] + + +@mock_dynamodb +def test_update_list_index__set_double_nested_index(): + table_name = "test_list_index_access" + client = create_table_with_list(table_name) + client.put_item( + TableName=table_name, + Item={ + "id": {"S": "foo2"}, + "itemmap": { + "M": { + "itemlist": { + "L": [ + {"M": {"foo": {"S": "bar11"}, "foos": {"S": "bar12"}}}, + {"M": {"foo": {"S": "bar21"}, "foos": {"S": "bar21"}}}, + ] + } + } + }, + }, + ) + client.update_item( + TableName=table_name, + Key={"id": {"S": "foo2"}}, + UpdateExpression="set itemmap.itemlist[1].foos=:Item", + ExpressionAttributeValues={":Item": {"S": "bar22"}}, + ) + # + result = client.get_item(TableName=table_name, Key={"id": {"S": "foo2"}})["Item"] + assert result["id"] == {"S": "foo2"} + len(result["itemmap"]["M"]["itemlist"]["L"]).should.equal(2) + result["itemmap"]["M"]["itemlist"]["L"][0].should.equal( + {"M": {"foo": {"S": "bar11"}, "foos": {"S": "bar12"}}} + ) # unchanged + result["itemmap"]["M"]["itemlist"]["L"][1].should.equal( + {"M": {"foo": {"S": "bar21"}, "foos": {"S": "bar22"}}} + ) # updated + + +@mock_dynamodb +def test_update_list_index__set_index_of_a_string(): + table_name = "test_list_index_access" + client = create_table_with_list(table_name) + client.put_item( + TableName=table_name, Item={"id": {"S": "foo2"}, "itemstr": {"S": "somestring"}} + ) + with pytest.raises(ClientError) as ex: + client.update_item( + TableName=table_name, + Key={"id": {"S": "foo2"}}, + UpdateExpression="set itemstr[1]=:Item", + ExpressionAttributeValues={":Item": {"S": "string_update"}}, + ) + client.get_item(TableName=table_name, Key={"id": {"S": "foo2"}})["Item"] + + ex.value.response["Error"]["Code"].should.equal("ValidationException") + ex.value.response["Error"]["Message"].should.equal( + "The document path provided in the update expression is invalid for update" + ) + + +@mock_dynamodb +def test_remove_top_level_attribute(): + table_name = "test_remove" + client = create_table_with_list(table_name) + client.put_item( + TableName=table_name, Item={"id": {"S": "foo"}, "item": {"S": "bar"}} + ) + client.update_item( + TableName=table_name, + Key={"id": {"S": "foo"}}, + UpdateExpression="REMOVE #i", + ExpressionAttributeNames={"#i": "item"}, + ) + # + result = client.get_item(TableName=table_name, Key={"id": {"S": "foo"}})["Item"] + result.should.equal({"id": {"S": "foo"}}) + + +@mock_dynamodb +def test_remove_top_level_attribute_non_existent(): + """ + Remove statements do not require attribute to exist they silently pass + """ + table_name = "test_remove" + client = create_table_with_list(table_name) + ddb_item = {"id": {"S": "foo"}, "item": {"S": "bar"}} + client.put_item(TableName=table_name, Item=ddb_item) + client.update_item( + TableName=table_name, + Key={"id": {"S": "foo"}}, + UpdateExpression="REMOVE non_existent_attribute", + ExpressionAttributeNames={"#i": "item"}, + ) + result = client.get_item(TableName=table_name, Key={"id": {"S": "foo"}})["Item"] + result.should.equal(ddb_item) + + +@mock_dynamodb +def test_remove_list_index__remove_existing_index(): + table_name = "test_list_index_access" + client = create_table_with_list(table_name) + client.put_item( + TableName=table_name, + Item={ + "id": {"S": "foo"}, + "itemlist": {"L": [{"S": "bar1"}, {"S": "bar2"}, {"S": "bar3"}]}, + }, + ) + client.update_item( + TableName=table_name, + Key={"id": {"S": "foo"}}, + UpdateExpression="REMOVE itemlist[1]", + ) + # + result = client.get_item(TableName=table_name, Key={"id": {"S": "foo"}})["Item"] + result["id"].should.equal({"S": "foo"}) + result["itemlist"].should.equal({"L": [{"S": "bar1"}, {"S": "bar3"}]}) + + +@mock_dynamodb +def test_remove_list_index__remove_existing_nested_index(): + table_name = "test_list_index_access" + client = create_table_with_list(table_name) + client.put_item( + TableName=table_name, + Item={ + "id": {"S": "foo2"}, + "itemmap": {"M": {"itemlist": {"L": [{"S": "bar1"}, {"S": "bar2"}]}}}, + }, + ) + client.update_item( + TableName=table_name, + Key={"id": {"S": "foo2"}}, + UpdateExpression="REMOVE itemmap.itemlist[1]", + ) + # + result = client.get_item(TableName=table_name, Key={"id": {"S": "foo2"}})["Item"] + result["id"].should.equal({"S": "foo2"}) + result["itemmap"]["M"]["itemlist"]["L"].should.equal([{"S": "bar1"}]) + + +@mock_dynamodb +def test_remove_list_index__remove_existing_double_nested_index(): + table_name = "test_list_index_access" + client = create_table_with_list(table_name) + client.put_item( + TableName=table_name, + Item={ + "id": {"S": "foo2"}, + "itemmap": { + "M": { + "itemlist": { + "L": [ + {"M": {"foo00": {"S": "bar1"}, "foo01": {"S": "bar2"}}}, + {"M": {"foo10": {"S": "bar1"}, "foo11": {"S": "bar2"}}}, + ] + } + } + }, + }, + ) + client.update_item( + TableName=table_name, + Key={"id": {"S": "foo2"}}, + UpdateExpression="REMOVE itemmap.itemlist[1].foo10", + ) + # + result = client.get_item(TableName=table_name, Key={"id": {"S": "foo2"}})["Item"] + assert result["id"] == {"S": "foo2"} + assert result["itemmap"]["M"]["itemlist"]["L"][0]["M"].should.equal( + {"foo00": {"S": "bar1"}, "foo01": {"S": "bar2"}} + ) # untouched + assert result["itemmap"]["M"]["itemlist"]["L"][1]["M"].should.equal( + {"foo11": {"S": "bar2"}} + ) # changed + + +@mock_dynamodb +def test_remove_list_index__remove_index_out_of_range(): + table_name = "test_list_index_access" + client = create_table_with_list(table_name) + client.put_item( + TableName=table_name, + Item={ + "id": {"S": "foo"}, + "itemlist": {"L": [{"S": "bar1"}, {"S": "bar2"}, {"S": "bar3"}]}, + }, + ) + client.update_item( + TableName=table_name, + Key={"id": {"S": "foo"}}, + UpdateExpression="REMOVE itemlist[10]", + ) + # + result = client.get_item(TableName=table_name, Key={"id": {"S": "foo"}})["Item"] + assert result["id"] == {"S": "foo"} + assert result["itemlist"] == {"L": [{"S": "bar1"}, {"S": "bar2"}, {"S": "bar3"}]} + + +def create_table_with_list(table_name): + client = boto3.client("dynamodb", region_name="us-east-1") + client.create_table( + TableName=table_name, + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + BillingMode="PAY_PER_REQUEST", + ) + return client + + +@mock_dynamodb +def test_sorted_query_with_numerical_sort_key(): + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="CarCollection", + KeySchema=[ + {"AttributeName": "CarModel", "KeyType": "HASH"}, + {"AttributeName": "CarPrice", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "CarModel", "AttributeType": "S"}, + {"AttributeName": "CarPrice", "AttributeType": "N"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, + ) + + def create_item(price): + return {"CarModel": "M", "CarPrice": price} + + table = dynamodb.Table("CarCollection") + items = list(map(create_item, [2, 1, 10, 3])) + for item in items: + table.put_item(Item=item) + + response = table.query(KeyConditionExpression=Key("CarModel").eq("M")) + + response_items = response["Items"] + assert len(items) == len(response_items) + assert all(isinstance(item["CarPrice"], Decimal) for item in response_items) + response_prices = [item["CarPrice"] for item in response_items] + expected_prices = [Decimal(item["CarPrice"]) for item in items] + expected_prices.sort() + assert ( + expected_prices == response_prices + ), "result items are not sorted by numerical value" + + +# https://github.com/spulec/moto/issues/1874 +@mock_dynamodb +def test_item_size_is_under_400KB(): + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + client = boto3.client("dynamodb", region_name="us-east-1") + + dynamodb.create_table( + TableName="moto-test", + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, + ) + table = dynamodb.Table("moto-test") + + large_item = "x" * 410 * 1000 + assert_failure_due_to_item_size( + func=client.put_item, + TableName="moto-test", + Item={"id": {"S": "foo"}, "cont": {"S": large_item}}, + ) + assert_failure_due_to_item_size( + func=table.put_item, Item={"id": "bar", "cont": large_item} + ) + assert_failure_due_to_item_size_to_update( + func=client.update_item, + TableName="moto-test", + Key={"id": {"S": "foo2"}}, + UpdateExpression="set cont=:Item", + ExpressionAttributeValues={":Item": {"S": large_item}}, + ) + # Assert op fails when updating a nested item + assert_failure_due_to_item_size( + func=table.put_item, Item={"id": "bar", "itemlist": [{"cont": large_item}]} + ) + assert_failure_due_to_item_size( + func=client.put_item, + TableName="moto-test", + Item={ + "id": {"S": "foo"}, + "itemlist": {"L": [{"M": {"item1": {"S": large_item}}}]}, + }, + ) + + +def assert_failure_due_to_item_size(func, **kwargs): + with pytest.raises(ClientError) as ex: + func(**kwargs) + ex.value.response["Error"]["Code"].should.equal("ValidationException") + ex.value.response["Error"]["Message"].should.equal( + "Item size has exceeded the maximum allowed size" + ) + + +def assert_failure_due_to_item_size_to_update(func, **kwargs): + with pytest.raises(ClientError) as ex: + func(**kwargs) + ex.value.response["Error"]["Code"].should.equal("ValidationException") + ex.value.response["Error"]["Message"].should.equal( + "Item size to update has exceeded the maximum allowed size" + ) + + +@mock_dynamodb +def test_update_supports_complex_expression_attribute_values(): + client = boto3.client("dynamodb", region_name="us-east-1") + + client.create_table( + AttributeDefinitions=[{"AttributeName": "SHA256", "AttributeType": "S"}], + TableName="TestTable", + KeySchema=[{"AttributeName": "SHA256", "KeyType": "HASH"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + client.update_item( + TableName="TestTable", + Key={"SHA256": {"S": "sha-of-file"}}, + UpdateExpression=( + "SET MD5 = :md5," "MyStringSet = :string_set," "MyMap = :map" + ), + ExpressionAttributeValues={ + ":md5": {"S": "md5-of-file"}, + ":string_set": {"SS": ["string1", "string2"]}, + ":map": {"M": {"EntryKey": {"SS": ["thing1", "thing2"]}}}, + }, + ) + result = client.get_item( + TableName="TestTable", Key={"SHA256": {"S": "sha-of-file"}} + )["Item"] + result.should.equal( + { + "MyStringSet": {"SS": ["string1", "string2"]}, + "MyMap": {"M": {"EntryKey": {"SS": ["thing1", "thing2"]}}}, + "SHA256": {"S": "sha-of-file"}, + "MD5": {"S": "md5-of-file"}, + } + ) + + +@mock_dynamodb +def test_update_supports_list_append(): + # Verify whether the list_append operation works as expected + client = boto3.client("dynamodb", region_name="us-east-1") + + client.create_table( + AttributeDefinitions=[{"AttributeName": "SHA256", "AttributeType": "S"}], + TableName="TestTable", + KeySchema=[{"AttributeName": "SHA256", "KeyType": "HASH"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + client.put_item( + TableName="TestTable", + Item={"SHA256": {"S": "sha-of-file"}, "crontab": {"L": [{"S": "bar1"}]}}, + ) + + # Update item using list_append expression + updated_item = client.update_item( + TableName="TestTable", + Key={"SHA256": {"S": "sha-of-file"}}, + UpdateExpression="SET crontab = list_append(crontab, :i)", + ExpressionAttributeValues={":i": {"L": [{"S": "bar2"}]}}, + ReturnValues="UPDATED_NEW", + ) + + # Verify updated item is correct + updated_item["Attributes"].should.equal( + {"crontab": {"L": [{"S": "bar1"}, {"S": "bar2"}]}} + ) + # Verify item is appended to the existing list + result = client.get_item( + TableName="TestTable", Key={"SHA256": {"S": "sha-of-file"}} + )["Item"] + result.should.equal( + { + "SHA256": {"S": "sha-of-file"}, + "crontab": {"L": [{"S": "bar1"}, {"S": "bar2"}]}, + } + ) + + +@mock_dynamodb +def test_update_supports_nested_list_append(): + # Verify whether we can append a list that's inside a map + client = boto3.client("dynamodb", region_name="us-east-1") + + client.create_table( + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + TableName="TestTable", + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + client.put_item( + TableName="TestTable", + Item={ + "id": {"S": "nested_list_append"}, + "a": {"M": {"b": {"L": [{"S": "bar1"}]}}}, + }, + ) + + # Update item using list_append expression + updated_item = client.update_item( + TableName="TestTable", + Key={"id": {"S": "nested_list_append"}}, + UpdateExpression="SET a.#b = list_append(a.#b, :i)", + ExpressionAttributeValues={":i": {"L": [{"S": "bar2"}]}}, + ExpressionAttributeNames={"#b": "b"}, + ReturnValues="UPDATED_NEW", + ) + + # Verify updated item is correct + updated_item["Attributes"].should.equal( + {"a": {"M": {"b": {"L": [{"S": "bar1"}, {"S": "bar2"}]}}}} + ) + result = client.get_item( + TableName="TestTable", Key={"id": {"S": "nested_list_append"}} + )["Item"] + result.should.equal( + { + "id": {"S": "nested_list_append"}, + "a": {"M": {"b": {"L": [{"S": "bar1"}, {"S": "bar2"}]}}}, + } + ) + + +@mock_dynamodb +def test_update_supports_multiple_levels_nested_list_append(): + # Verify whether we can append a list that's inside a map that's inside a map (Inception!) + client = boto3.client("dynamodb", region_name="us-east-1") + + client.create_table( + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + TableName="TestTable", + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + client.put_item( + TableName="TestTable", + Item={ + "id": {"S": "nested_list_append"}, + "a": {"M": {"b": {"M": {"c": {"L": [{"S": "bar1"}]}}}}}, + }, + ) + + # Update item using list_append expression + updated_item = client.update_item( + TableName="TestTable", + Key={"id": {"S": "nested_list_append"}}, + UpdateExpression="SET a.#b.c = list_append(a.#b.#c, :i)", + ExpressionAttributeValues={":i": {"L": [{"S": "bar2"}]}}, + ExpressionAttributeNames={"#b": "b", "#c": "c"}, + ReturnValues="UPDATED_NEW", + ) + + # Verify updated item is correct + updated_item["Attributes"].should.equal( + {"a": {"M": {"b": {"M": {"c": {"L": [{"S": "bar1"}, {"S": "bar2"}]}}}}}} + ) + # Verify item is appended to the existing list + result = client.get_item( + TableName="TestTable", Key={"id": {"S": "nested_list_append"}} + )["Item"] + result.should.equal( + { + "id": {"S": "nested_list_append"}, + "a": {"M": {"b": {"M": {"c": {"L": [{"S": "bar1"}, {"S": "bar2"}]}}}}}, + } + ) + + +@mock_dynamodb +def test_update_supports_nested_list_append_onto_another_list(): + # Verify whether we can take the contents of one list, and use that to fill another list + # Note that the contents of the other list is completely overwritten + client = boto3.client("dynamodb", region_name="us-east-1") + + client.create_table( + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + TableName="TestTable", + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + client.put_item( + TableName="TestTable", + Item={ + "id": {"S": "list_append_another"}, + "a": {"M": {"b": {"L": [{"S": "bar1"}]}, "c": {"L": [{"S": "car1"}]}}}, + }, + ) + + # Update item using list_append expression + updated_item = client.update_item( + TableName="TestTable", + Key={"id": {"S": "list_append_another"}}, + UpdateExpression="SET a.#c = list_append(a.#b, :i)", + ExpressionAttributeValues={":i": {"L": [{"S": "bar2"}]}}, + ExpressionAttributeNames={"#b": "b", "#c": "c"}, + ReturnValues="UPDATED_NEW", + ) + + # Verify updated item is correct + updated_item["Attributes"].should.equal( + {"a": {"M": {"c": {"L": [{"S": "bar1"}, {"S": "bar2"}]}}}} + ) + # Verify item is appended to the existing list + result = client.get_item( + TableName="TestTable", Key={"id": {"S": "list_append_another"}} + )["Item"] + result.should.equal( + { + "id": {"S": "list_append_another"}, + "a": { + "M": { + "b": {"L": [{"S": "bar1"}]}, + "c": {"L": [{"S": "bar1"}, {"S": "bar2"}]}, + } + }, + } + ) + + +@mock_dynamodb +def test_update_supports_list_append_maps(): + client = boto3.client("dynamodb", region_name="us-west-1") + client.create_table( + AttributeDefinitions=[ + {"AttributeName": "id", "AttributeType": "S"}, + {"AttributeName": "rid", "AttributeType": "S"}, + ], + TableName="TestTable", + KeySchema=[ + {"AttributeName": "id", "KeyType": "HASH"}, + {"AttributeName": "rid", "KeyType": "RANGE"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + client.put_item( + TableName="TestTable", + Item={ + "id": {"S": "nested_list_append"}, + "rid": {"S": "range_key"}, + "a": {"L": [{"M": {"b": {"S": "bar1"}}}]}, + }, + ) + + # Update item using list_append expression + updated_item = client.update_item( + TableName="TestTable", + Key={"id": {"S": "nested_list_append"}, "rid": {"S": "range_key"}}, + UpdateExpression="SET a = list_append(a, :i)", + ExpressionAttributeValues={":i": {"L": [{"M": {"b": {"S": "bar2"}}}]}}, + ReturnValues="UPDATED_NEW", + ) + + # Verify updated item is correct + updated_item["Attributes"].should.equal( + {"a": {"L": [{"M": {"b": {"S": "bar1"}}}, {"M": {"b": {"S": "bar2"}}}]}} + ) + # Verify item is appended to the existing list + result = client.query( + TableName="TestTable", + KeyConditionExpression="id = :i AND begins_with(rid, :r)", + ExpressionAttributeValues={ + ":i": {"S": "nested_list_append"}, + ":r": {"S": "range_key"}, + }, + )["Items"] + result.should.equal( + [ + { + "a": {"L": [{"M": {"b": {"S": "bar1"}}}, {"M": {"b": {"S": "bar2"}}}]}, + "rid": {"S": "range_key"}, + "id": {"S": "nested_list_append"}, + } + ] + ) + + +@mock_dynamodb +def test_update_supports_nested_update_if_nested_value_not_exists(): + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + name = "TestTable" + + dynamodb.create_table( + TableName=name, + KeySchema=[{"AttributeName": "user_id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "user_id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + table = dynamodb.Table(name) + table.put_item( + Item={"user_id": "1234", "friends": {"5678": {"name": "friend_5678"}},}, + ) + table.update_item( + Key={"user_id": "1234"}, + ExpressionAttributeNames={"#friends": "friends", "#friendid": "0000",}, + ExpressionAttributeValues={":friend": {"name": "friend_0000"},}, + UpdateExpression="SET #friends.#friendid = :friend", + ReturnValues="UPDATED_NEW", + ) + item = table.get_item(Key={"user_id": "1234"})["Item"] + assert item == { + "user_id": "1234", + "friends": {"5678": {"name": "friend_5678"}, "0000": {"name": "friend_0000"},}, + } + + +@mock_dynamodb +def test_update_supports_list_append_with_nested_if_not_exists_operation(): + dynamo = boto3.resource("dynamodb", region_name="us-west-1") + table_name = "test" + + dynamo.create_table( + TableName=table_name, + AttributeDefinitions=[{"AttributeName": "Id", "AttributeType": "S"}], + KeySchema=[{"AttributeName": "Id", "KeyType": "HASH"}], + ProvisionedThroughput={"ReadCapacityUnits": 20, "WriteCapacityUnits": 20}, + ) + + table = dynamo.Table(table_name) + + table.put_item(Item={"Id": "item-id", "nest1": {"nest2": {}}}) + updated_item = table.update_item( + Key={"Id": "item-id"}, + UpdateExpression="SET nest1.nest2.event_history = list_append(if_not_exists(nest1.nest2.event_history, :empty_list), :new_value)", + ExpressionAttributeValues={":empty_list": [], ":new_value": ["some_value"]}, + ReturnValues="UPDATED_NEW", + ) + + # Verify updated item is correct + updated_item["Attributes"].should.equal( + {"nest1": {"nest2": {"event_history": ["some_value"]}}} + ) + + table.get_item(Key={"Id": "item-id"})["Item"].should.equal( + {"Id": "item-id", "nest1": {"nest2": {"event_history": ["some_value"]}}} + ) + + +@mock_dynamodb +def test_update_supports_list_append_with_nested_if_not_exists_operation_and_property_already_exists(): + dynamo = boto3.resource("dynamodb", region_name="us-west-1") + table_name = "test" + + dynamo.create_table( + TableName=table_name, + AttributeDefinitions=[{"AttributeName": "Id", "AttributeType": "S"}], + KeySchema=[{"AttributeName": "Id", "KeyType": "HASH"}], + ProvisionedThroughput={"ReadCapacityUnits": 20, "WriteCapacityUnits": 20}, + ) + + table = dynamo.Table(table_name) + + table.put_item(Item={"Id": "item-id", "event_history": ["other_value"]}) + updated_item = table.update_item( + Key={"Id": "item-id"}, + UpdateExpression="SET event_history = list_append(if_not_exists(event_history, :empty_list), :new_value)", + ExpressionAttributeValues={":empty_list": [], ":new_value": ["some_value"]}, + ReturnValues="UPDATED_NEW", + ) + + # Verify updated item is correct + updated_item["Attributes"].should.equal( + {"event_history": ["other_value", "some_value"]} + ) + + table.get_item(Key={"Id": "item-id"})["Item"].should.equal( + {"Id": "item-id", "event_history": ["other_value", "some_value"]} + ) + + +def _create_user_table(): + client = boto3.client("dynamodb", region_name="us-east-1") + client.create_table( + TableName="users", + KeySchema=[{"AttributeName": "username", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "username", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + client.put_item( + TableName="users", Item={"username": {"S": "user1"}, "foo": {"S": "bar"}} + ) + client.put_item( + TableName="users", Item={"username": {"S": "user2"}, "foo": {"S": "bar"}} + ) + client.put_item( + TableName="users", Item={"username": {"S": "user3"}, "foo": {"S": "bar"}} + ) + return client + + +@mock_dynamodb +def test_update_item_if_original_value_is_none(): + dynamo = boto3.resource("dynamodb", region_name="eu-central-1") + dynamo.create_table( + AttributeDefinitions=[{"AttributeName": "job_id", "AttributeType": "S"}], + TableName="origin-rbu-dev", + KeySchema=[{"AttributeName": "job_id", "KeyType": "HASH"}], + ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, + ) + table = dynamo.Table("origin-rbu-dev") + table.put_item(Item={"job_id": "a", "job_name": None}) + table.update_item( + Key={"job_id": "a"}, + UpdateExpression="SET job_name = :output", + ExpressionAttributeValues={":output": "updated"}, + ) + table.scan()["Items"][0]["job_name"].should.equal("updated") + + +@mock_dynamodb +def test_update_nested_item_if_original_value_is_none(): + dynamo = boto3.resource("dynamodb", region_name="eu-central-1") + dynamo.create_table( + AttributeDefinitions=[{"AttributeName": "job_id", "AttributeType": "S"}], + TableName="origin-rbu-dev", + KeySchema=[{"AttributeName": "job_id", "KeyType": "HASH"}], + ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, + ) + table = dynamo.Table("origin-rbu-dev") + table.put_item(Item={"job_id": "a", "job_details": {"job_name": None}}) + updated_item = table.update_item( + Key={"job_id": "a"}, + UpdateExpression="SET job_details.job_name = :output", + ExpressionAttributeValues={":output": "updated"}, + ReturnValues="UPDATED_NEW", + ) + + # Verify updated item is correct + updated_item["Attributes"].should.equal({"job_details": {"job_name": "updated"}}) + + table.scan()["Items"][0]["job_details"]["job_name"].should.equal("updated") + + +@mock_dynamodb +def test_allow_update_to_item_with_different_type(): + dynamo = boto3.resource("dynamodb", region_name="eu-central-1") + dynamo.create_table( + AttributeDefinitions=[{"AttributeName": "job_id", "AttributeType": "S"}], + TableName="origin-rbu-dev", + KeySchema=[{"AttributeName": "job_id", "KeyType": "HASH"}], + ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, + ) + table = dynamo.Table("origin-rbu-dev") + table.put_item(Item={"job_id": "a", "job_details": {"job_name": {"nested": "yes"}}}) + table.put_item(Item={"job_id": "b", "job_details": {"job_name": {"nested": "yes"}}}) + updated_item = table.update_item( + Key={"job_id": "a"}, + UpdateExpression="SET job_details.job_name = :output", + ExpressionAttributeValues={":output": "updated"}, + ReturnValues="UPDATED_NEW", + ) + + # Verify updated item is correct + updated_item["Attributes"].should.equal({"job_details": {"job_name": "updated"}}) + + table.get_item(Key={"job_id": "a"})["Item"]["job_details"][ + "job_name" + ].should.be.equal("updated") + table.get_item(Key={"job_id": "b"})["Item"]["job_details"][ + "job_name" + ].should.be.equal({"nested": "yes"}) + + +@mock_dynamodb +def test_query_catches_when_no_filters(): + dynamo = boto3.resource("dynamodb", region_name="eu-central-1") + dynamo.create_table( + AttributeDefinitions=[{"AttributeName": "job_id", "AttributeType": "S"}], + TableName="origin-rbu-dev", + KeySchema=[{"AttributeName": "job_id", "KeyType": "HASH"}], + ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, + ) + table = dynamo.Table("origin-rbu-dev") + + with pytest.raises(ClientError) as ex: + table.query(TableName="original-rbu-dev") + + ex.value.response["Error"]["Code"].should.equal("ValidationException") + ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.value.response["Error"]["Message"].should.equal( + "Either KeyConditions or QueryFilter should be present" + ) + + +@mock_dynamodb +def test_invalid_transact_get_items(): + + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="test1", + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + table = dynamodb.Table("test1") + table.put_item( + Item={"id": "1", "val": "1",} + ) + + table.put_item( + Item={"id": "1", "val": "2",} + ) + + client = boto3.client("dynamodb", region_name="us-east-1") + + with pytest.raises(ClientError) as ex: + client.transact_get_items( + TransactItems=[ + {"Get": {"Key": {"id": {"S": "1"}}, "TableName": "test1"}} + for i in range(26) + ] + ) + + ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.value.response["Error"]["Message"].should.match( + r"failed to satisfy constraint: Member must have length less than or equal to 25", + re.I, + ) + + with pytest.raises(ClientError) as ex: + client.transact_get_items( + TransactItems=[ + {"Get": {"Key": {"id": {"S": "1"},}, "TableName": "test1",}}, + {"Get": {"Key": {"id": {"S": "1"},}, "TableName": "non_exists_table",}}, + ] + ) + + ex.value.response["Error"]["Code"].should.equal("ResourceNotFoundException") + ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.value.response["Error"]["Message"].should.equal("Requested resource not found") + + +@mock_dynamodb +def test_valid_transact_get_items(): + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="test1", + KeySchema=[ + {"AttributeName": "id", "KeyType": "HASH"}, + {"AttributeName": "sort_key", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "id", "AttributeType": "S"}, + {"AttributeName": "sort_key", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + table1 = dynamodb.Table("test1") + table1.put_item( + Item={"id": "1", "sort_key": "1",} + ) + + table1.put_item( + Item={"id": "1", "sort_key": "2",} + ) + + dynamodb.create_table( + TableName="test2", + KeySchema=[ + {"AttributeName": "id", "KeyType": "HASH"}, + {"AttributeName": "sort_key", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "id", "AttributeType": "S"}, + {"AttributeName": "sort_key", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + table2 = dynamodb.Table("test2") + table2.put_item( + Item={"id": "1", "sort_key": "1",} + ) + + client = boto3.client("dynamodb", region_name="us-east-1") + res = client.transact_get_items( + TransactItems=[ + { + "Get": { + "Key": {"id": {"S": "1"}, "sort_key": {"S": "1"}}, + "TableName": "test1", + } + }, + { + "Get": { + "Key": {"id": {"S": "non_exists_key"}, "sort_key": {"S": "2"}}, + "TableName": "test1", + } + }, + ] + ) + res["Responses"][0]["Item"].should.equal({"id": {"S": "1"}, "sort_key": {"S": "1"}}) + len(res["Responses"]).should.equal(2) + res["Responses"][1].should.equal({}) + + res = client.transact_get_items( + TransactItems=[ + { + "Get": { + "Key": {"id": {"S": "1"}, "sort_key": {"S": "1"}}, + "TableName": "test1", + } + }, + { + "Get": { + "Key": {"id": {"S": "1"}, "sort_key": {"S": "2"}}, + "TableName": "test1", + } + }, + { + "Get": { + "Key": {"id": {"S": "1"}, "sort_key": {"S": "1"}}, + "TableName": "test2", + } + }, + ] + ) + + res["Responses"][0]["Item"].should.equal({"id": {"S": "1"}, "sort_key": {"S": "1"}}) + + res["Responses"][1]["Item"].should.equal({"id": {"S": "1"}, "sort_key": {"S": "2"}}) + + res["Responses"][2]["Item"].should.equal({"id": {"S": "1"}, "sort_key": {"S": "1"}}) + + res = client.transact_get_items( + TransactItems=[ + { + "Get": { + "Key": {"id": {"S": "1"}, "sort_key": {"S": "1"}}, + "TableName": "test1", + } + }, + { + "Get": { + "Key": {"id": {"S": "1"}, "sort_key": {"S": "2"}}, + "TableName": "test1", + } + }, + { + "Get": { + "Key": {"id": {"S": "1"}, "sort_key": {"S": "1"}}, + "TableName": "test2", + } + }, + ], + ReturnConsumedCapacity="TOTAL", + ) + + res["ConsumedCapacity"][0].should.equal( + {"TableName": "test1", "CapacityUnits": 4.0, "ReadCapacityUnits": 4.0} + ) + + res["ConsumedCapacity"][1].should.equal( + {"TableName": "test2", "CapacityUnits": 2.0, "ReadCapacityUnits": 2.0} + ) + + res = client.transact_get_items( + TransactItems=[ + { + "Get": { + "Key": {"id": {"S": "1"}, "sort_key": {"S": "1"}}, + "TableName": "test1", + } + }, + { + "Get": { + "Key": {"id": {"S": "1"}, "sort_key": {"S": "2"}}, + "TableName": "test1", + } + }, + { + "Get": { + "Key": {"id": {"S": "1"}, "sort_key": {"S": "1"}}, + "TableName": "test2", + } + }, + ], + ReturnConsumedCapacity="INDEXES", + ) + + res["ConsumedCapacity"][0].should.equal( + { + "TableName": "test1", + "CapacityUnits": 4.0, + "ReadCapacityUnits": 4.0, + "Table": {"CapacityUnits": 4.0, "ReadCapacityUnits": 4.0,}, + } + ) + + res["ConsumedCapacity"][1].should.equal( + { + "TableName": "test2", + "CapacityUnits": 2.0, + "ReadCapacityUnits": 2.0, + "Table": {"CapacityUnits": 2.0, "ReadCapacityUnits": 2.0,}, + } + ) + + +@mock_dynamodb +def test_gsi_verify_negative_number_order(): + table_schema = { + "KeySchema": [{"AttributeName": "partitionKey", "KeyType": "HASH"}], + "GlobalSecondaryIndexes": [ + { + "IndexName": "GSI-K1", + "KeySchema": [ + {"AttributeName": "gsiK1PartitionKey", "KeyType": "HASH"}, + {"AttributeName": "gsiK1SortKey", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "KEYS_ONLY",}, + } + ], + "AttributeDefinitions": [ + {"AttributeName": "partitionKey", "AttributeType": "S"}, + {"AttributeName": "gsiK1PartitionKey", "AttributeType": "S"}, + {"AttributeName": "gsiK1SortKey", "AttributeType": "N"}, + ], + } + + item1 = { + "partitionKey": "pk-1", + "gsiK1PartitionKey": "gsi-k1", + "gsiK1SortKey": Decimal("-0.6"), + } + + item2 = { + "partitionKey": "pk-2", + "gsiK1PartitionKey": "gsi-k1", + "gsiK1SortKey": Decimal("-0.7"), + } + + item3 = { + "partitionKey": "pk-3", + "gsiK1PartitionKey": "gsi-k1", + "gsiK1SortKey": Decimal("0.7"), + } + + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema + ) + table = dynamodb.Table("test-table") + table.put_item(Item=item3) + table.put_item(Item=item1) + table.put_item(Item=item2) + + resp = table.query( + KeyConditionExpression=Key("gsiK1PartitionKey").eq("gsi-k1"), + IndexName="GSI-K1", + ) + # Items should be ordered with the lowest number first + [float(item["gsiK1SortKey"]) for item in resp["Items"]].should.equal( + [-0.7, -0.6, 0.7] + ) + + +@mock_dynamodb +def test_transact_write_items_put(): + table_schema = { + "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], + } + dynamodb = boto3.client("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema + ) + # Put multiple items + dynamodb.transact_write_items( + TransactItems=[ + { + "Put": { + "Item": {"id": {"S": "foo{}".format(str(i))}, "foo": {"S": "bar"},}, + "TableName": "test-table", + } + } + for i in range(0, 5) + ] + ) + # Assert all are present + items = dynamodb.scan(TableName="test-table")["Items"] + items.should.have.length_of(5) + + +@mock_dynamodb +def test_transact_write_items_put_conditional_expressions(): + table_schema = { + "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], + } + dynamodb = boto3.client("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema + ) + dynamodb.put_item( + TableName="test-table", Item={"id": {"S": "foo2"},}, + ) + # Put multiple items + with pytest.raises(ClientError) as ex: + dynamodb.transact_write_items( + TransactItems=[ + { + "Put": { + "Item": { + "id": {"S": "foo{}".format(str(i))}, + "foo": {"S": "bar"}, + }, + "TableName": "test-table", + "ConditionExpression": "#i <> :i", + "ExpressionAttributeNames": {"#i": "id"}, + "ExpressionAttributeValues": { + ":i": { + "S": "foo2" + } # This item already exist, so the ConditionExpression should fail + }, + } + } + for i in range(0, 5) + ] + ) + # Assert the exception is correct + ex.value.response["Error"]["Code"].should.equal("TransactionCanceledException") + ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + # Assert all are present + items = dynamodb.scan(TableName="test-table")["Items"] + items.should.have.length_of(1) + items[0].should.equal({"id": {"S": "foo2"}}) + + +@mock_dynamodb +def test_transact_write_items_conditioncheck_passes(): + table_schema = { + "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], + } + dynamodb = boto3.client("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema + ) + # Insert an item without email address + dynamodb.put_item( + TableName="test-table", Item={"id": {"S": "foo"},}, + ) + # Put an email address, after verifying it doesn't exist yet + dynamodb.transact_write_items( + TransactItems=[ + { + "ConditionCheck": { + "Key": {"id": {"S": "foo"}}, + "TableName": "test-table", + "ConditionExpression": "attribute_not_exists(#e)", + "ExpressionAttributeNames": {"#e": "email_address"}, + } + }, + { + "Put": { + "Item": { + "id": {"S": "foo"}, + "email_address": {"S": "test@moto.com"}, + }, + "TableName": "test-table", + } + }, + ] + ) + # Assert all are present + items = dynamodb.scan(TableName="test-table")["Items"] + items.should.have.length_of(1) + items[0].should.equal({"email_address": {"S": "test@moto.com"}, "id": {"S": "foo"}}) + + +@mock_dynamodb +def test_transact_write_items_conditioncheck_fails(): + table_schema = { + "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], + } + dynamodb = boto3.client("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema + ) + # Insert an item with email address + dynamodb.put_item( + TableName="test-table", + Item={"id": {"S": "foo"}, "email_address": {"S": "test@moto.com"}}, + ) + # Try to put an email address, but verify whether it exists + # ConditionCheck should fail + with pytest.raises(ClientError) as ex: + dynamodb.transact_write_items( + TransactItems=[ + { + "ConditionCheck": { + "Key": {"id": {"S": "foo"}}, + "TableName": "test-table", + "ConditionExpression": "attribute_not_exists(#e)", + "ExpressionAttributeNames": {"#e": "email_address"}, + } + }, + { + "Put": { + "Item": { + "id": {"S": "foo"}, + "email_address": {"S": "update@moto.com"}, + }, + "TableName": "test-table", + } + }, + ] + ) + # Assert the exception is correct + ex.value.response["Error"]["Code"].should.equal("TransactionCanceledException") + ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + + # Assert the original email address is still present + items = dynamodb.scan(TableName="test-table")["Items"] + items.should.have.length_of(1) + items[0].should.equal({"email_address": {"S": "test@moto.com"}, "id": {"S": "foo"}}) + + +@mock_dynamodb +def test_transact_write_items_delete(): + table_schema = { + "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], + } + dynamodb = boto3.client("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema + ) + # Insert an item + dynamodb.put_item( + TableName="test-table", Item={"id": {"S": "foo"},}, + ) + # Delete the item + dynamodb.transact_write_items( + TransactItems=[ + {"Delete": {"Key": {"id": {"S": "foo"}}, "TableName": "test-table",}} + ] + ) + # Assert the item is deleted + items = dynamodb.scan(TableName="test-table")["Items"] + items.should.have.length_of(0) + + +@mock_dynamodb +def test_transact_write_items_delete_with_successful_condition_expression(): + table_schema = { + "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], + } + dynamodb = boto3.client("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema + ) + # Insert an item without email address + dynamodb.put_item( + TableName="test-table", Item={"id": {"S": "foo"},}, + ) + # ConditionExpression will pass - no email address has been specified yet + dynamodb.transact_write_items( + TransactItems=[ + { + "Delete": { + "Key": {"id": {"S": "foo"},}, + "TableName": "test-table", + "ConditionExpression": "attribute_not_exists(#e)", + "ExpressionAttributeNames": {"#e": "email_address"}, + } + } + ] + ) + # Assert the item is deleted + items = dynamodb.scan(TableName="test-table")["Items"] + items.should.have.length_of(0) + + +@mock_dynamodb +def test_transact_write_items_delete_with_failed_condition_expression(): + table_schema = { + "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], + } + dynamodb = boto3.client("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema + ) + # Insert an item with email address + dynamodb.put_item( + TableName="test-table", + Item={"id": {"S": "foo"}, "email_address": {"S": "test@moto.com"}}, + ) + # Try to delete an item that does not have an email address + # ConditionCheck should fail + with pytest.raises(ClientError) as ex: + dynamodb.transact_write_items( + TransactItems=[ + { + "Delete": { + "Key": {"id": {"S": "foo"},}, + "TableName": "test-table", + "ConditionExpression": "attribute_not_exists(#e)", + "ExpressionAttributeNames": {"#e": "email_address"}, + } + } + ] + ) + # Assert the exception is correct + ex.value.response["Error"]["Code"].should.equal("TransactionCanceledException") + ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + # Assert the original item is still present + items = dynamodb.scan(TableName="test-table")["Items"] + items.should.have.length_of(1) + items[0].should.equal({"email_address": {"S": "test@moto.com"}, "id": {"S": "foo"}}) + + +@mock_dynamodb +def test_transact_write_items_update(): + table_schema = { + "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], + } + dynamodb = boto3.client("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema + ) + # Insert an item + dynamodb.put_item(TableName="test-table", Item={"id": {"S": "foo"}}) + # Update the item + dynamodb.transact_write_items( + TransactItems=[ + { + "Update": { + "Key": {"id": {"S": "foo"}}, + "TableName": "test-table", + "UpdateExpression": "SET #e = :v", + "ExpressionAttributeNames": {"#e": "email_address"}, + "ExpressionAttributeValues": {":v": {"S": "test@moto.com"}}, + } + } + ] + ) + # Assert the item is updated + items = dynamodb.scan(TableName="test-table")["Items"] + items.should.have.length_of(1) + items[0].should.equal({"id": {"S": "foo"}, "email_address": {"S": "test@moto.com"}}) + + +@mock_dynamodb +def test_transact_write_items_update_with_failed_condition_expression(): + table_schema = { + "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], + } + dynamodb = boto3.client("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema + ) + # Insert an item with email address + dynamodb.put_item( + TableName="test-table", + Item={"id": {"S": "foo"}, "email_address": {"S": "test@moto.com"}}, + ) + # Try to update an item that does not have an email address + # ConditionCheck should fail + with pytest.raises(ClientError) as ex: + dynamodb.transact_write_items( + TransactItems=[ + { + "Update": { + "Key": {"id": {"S": "foo"}}, + "TableName": "test-table", + "UpdateExpression": "SET #e = :v", + "ConditionExpression": "attribute_not_exists(#e)", + "ExpressionAttributeNames": {"#e": "email_address"}, + "ExpressionAttributeValues": {":v": {"S": "update@moto.com"}}, + } + } + ] + ) + # Assert the exception is correct + ex.value.response["Error"]["Code"].should.equal("TransactionCanceledException") + ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + # Assert the original item is still present + items = dynamodb.scan(TableName="test-table")["Items"] + items.should.have.length_of(1) + items[0].should.equal({"email_address": {"S": "test@moto.com"}, "id": {"S": "foo"}}) + + +@mock_dynamodb +def test_dynamodb_max_1mb_limit(): + ddb = boto3.resource("dynamodb", region_name="eu-west-1") + + table_name = "populated-mock-table" + table = ddb.create_table( + TableName=table_name, + KeySchema=[ + {"AttributeName": "partition_key", "KeyType": "HASH"}, + {"AttributeName": "sort_key", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "partition_key", "AttributeType": "S"}, + {"AttributeName": "sort_key", "AttributeType": "S"}, + ], + BillingMode="PAY_PER_REQUEST", + ) + + # Populate the table + items = [ + { + "partition_key": "partition_key_val", # size=30 + "sort_key": "sort_key_value____" + str(i), # size=30 + } + for i in range(10000, 29999) + ] + with table.batch_writer() as batch: + for item in items: + batch.put_item(Item=item) + + response = table.query( + KeyConditionExpression=Key("partition_key").eq("partition_key_val") + ) + # We shouldn't get everything back - the total result set is well over 1MB + len(items).should.be.greater_than(response["Count"]) + response["LastEvaluatedKey"].shouldnt.be(None) + + +def assert_raise_syntax_error(client_error, token, near): + """ + Assert whether a client_error is as expected Syntax error. Syntax error looks like: `syntax_error_template` + + Args: + client_error(ClientError): The ClientError exception that was raised + token(str): The token that ws unexpected + near(str): The part in the expression that shows where the error occurs it generally has the preceding token the + optional separation and the problematic token. + """ + syntax_error_template = ( + 'Invalid UpdateExpression: Syntax error; token: "{token}", near: "{near}"' + ) + expected_syntax_error = syntax_error_template.format(token=token, near=near) + assert client_error.response["Error"]["Code"] == "ValidationException" + assert expected_syntax_error == client_error.response["Error"]["Message"] + + +@mock_dynamodb +def test_update_expression_with_numeric_literal_instead_of_value(): + """ + DynamoDB requires literals to be passed in as values. If they are put literally in the expression a token error will + be raised + """ + dynamodb = boto3.client("dynamodb", region_name="eu-west-1") + + dynamodb.create_table( + TableName="moto-test", + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + BillingMode="PAY_PER_REQUEST", + ) + + try: + dynamodb.update_item( + TableName="moto-test", + Key={"id": {"S": "1"}}, + UpdateExpression="SET MyStr = myNum + 1", + ) + assert False, "Validation exception not thrown" + except dynamodb.exceptions.ClientError as e: + assert_raise_syntax_error(e, "1", "+ 1") + + +@mock_dynamodb +def test_update_expression_with_multiple_set_clauses_must_be_comma_separated(): + """ + An UpdateExpression can have multiple set clauses but if they are passed in without the separating comma. + """ + dynamodb = boto3.client("dynamodb", region_name="eu-west-1") + + dynamodb.create_table( + TableName="moto-test", + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + BillingMode="PAY_PER_REQUEST", + ) + + try: + dynamodb.update_item( + TableName="moto-test", + Key={"id": {"S": "1"}}, + UpdateExpression="SET MyStr = myNum Mystr2 myNum2", + ) + assert False, "Validation exception not thrown" + except dynamodb.exceptions.ClientError as e: + assert_raise_syntax_error(e, "Mystr2", "myNum Mystr2 myNum2") + + +@mock_dynamodb +def test_list_tables_exclusive_start_table_name_empty(): + client = boto3.client("dynamodb", region_name="us-east-1") + + resp = client.list_tables(Limit=1, ExclusiveStartTableName="whatever") + + len(resp["TableNames"]).should.equal(0) + + +def assert_correct_client_error( + client_error, code, message_template, message_values=None, braces=None +): + """ + Assert whether a client_error is as expected. Allow for a list of values to be passed into the message + + Args: + client_error(ClientError): The ClientError exception that was raised + code(str): The code for the error (e.g. ValidationException) + message_template(str): Error message template. if message_values is not None then this template has a {values} + as placeholder. For example: + 'Value provided in ExpressionAttributeValues unused in expressions: keys: {values}' + message_values(list of str|None): The values that are passed in the error message + braces(list of str|None): List of length 2 with opening and closing brace for the values. By default it will be + surrounded by curly brackets + """ + braces = braces or ["{", "}"] + assert client_error.response["Error"]["Code"] == code + if message_values is not None: + values_string = "{open_brace}(?P.*){close_brace}".format( + open_brace=braces[0], close_brace=braces[1] + ) + re_msg = re.compile(message_template.format(values=values_string)) + match_result = re_msg.match(client_error.response["Error"]["Message"]) + assert match_result is not None + values_string = match_result.groupdict()["values"] + values = [key for key in values_string.split(", ")] + assert len(message_values) == len(values) + for value in message_values: + assert value in values + else: + assert client_error.response["Error"]["Message"] == message_template + + +def create_simple_table_and_return_client(): + dynamodb = boto3.client("dynamodb", region_name="eu-west-1") + dynamodb.create_table( + TableName="moto-test", + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"},], + ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, + ) + dynamodb.put_item( + TableName="moto-test", + Item={"id": {"S": "1"}, "myNum": {"N": "1"}, "MyStr": {"S": "1"},}, + ) + return dynamodb + + +# https://github.com/spulec/moto/issues/2806 +# https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_UpdateItem.html +# #DDB-UpdateItem-request-UpdateExpression +@mock_dynamodb +def test_update_item_with_attribute_in_right_hand_side_and_operation(): + dynamodb = create_simple_table_and_return_client() + + dynamodb.update_item( + TableName="moto-test", + Key={"id": {"S": "1"}}, + UpdateExpression="SET myNum = myNum+:val", + ExpressionAttributeValues={":val": {"N": "3"}}, + ) + + result = dynamodb.get_item(TableName="moto-test", Key={"id": {"S": "1"}}) + assert result["Item"]["myNum"]["N"] == "4" + + dynamodb.update_item( + TableName="moto-test", + Key={"id": {"S": "1"}}, + UpdateExpression="SET myNum = myNum - :val", + ExpressionAttributeValues={":val": {"N": "1"}}, + ) + result = dynamodb.get_item(TableName="moto-test", Key={"id": {"S": "1"}}) + assert result["Item"]["myNum"]["N"] == "3" + + +@mock_dynamodb +def test_non_existing_attribute_should_raise_exception(): + """ + Does error message get correctly raised if attribute is referenced but it does not exist for the item. + """ + dynamodb = create_simple_table_and_return_client() + + try: + dynamodb.update_item( + TableName="moto-test", + Key={"id": {"S": "1"}}, + UpdateExpression="SET MyStr = no_attr + MyStr", + ) + assert False, "Validation exception not thrown" + except dynamodb.exceptions.ClientError as e: + assert_correct_client_error( + e, + "ValidationException", + "The provided expression refers to an attribute that does not exist in the item", + ) + + +@mock_dynamodb +def test_update_expression_with_plus_in_attribute_name(): + """ + Does error message get correctly raised if attribute contains a plus and is passed in without an AttributeName. And + lhs & rhs are not attribute IDs by themselve. + """ + dynamodb = create_simple_table_and_return_client() + + dynamodb.put_item( + TableName="moto-test", + Item={"id": {"S": "1"}, "my+Num": {"S": "1"}, "MyStr": {"S": "aaa"},}, + ) + try: + dynamodb.update_item( + TableName="moto-test", + Key={"id": {"S": "1"}}, + UpdateExpression="SET MyStr = my+Num", + ) + assert False, "Validation exception not thrown" + except dynamodb.exceptions.ClientError as e: + assert_correct_client_error( + e, + "ValidationException", + "The provided expression refers to an attribute that does not exist in the item", + ) + + +@mock_dynamodb +def test_update_expression_with_minus_in_attribute_name(): + """ + Does error message get correctly raised if attribute contains a minus and is passed in without an AttributeName. And + lhs & rhs are not attribute IDs by themselve. + """ + dynamodb = create_simple_table_and_return_client() + + dynamodb.put_item( + TableName="moto-test", + Item={"id": {"S": "1"}, "my-Num": {"S": "1"}, "MyStr": {"S": "aaa"},}, + ) + try: + dynamodb.update_item( + TableName="moto-test", + Key={"id": {"S": "1"}}, + UpdateExpression="SET MyStr = my-Num", + ) + assert False, "Validation exception not thrown" + except dynamodb.exceptions.ClientError as e: + assert_correct_client_error( + e, + "ValidationException", + "The provided expression refers to an attribute that does not exist in the item", + ) + + +@mock_dynamodb +def test_update_expression_with_space_in_attribute_name(): + """ + Does error message get correctly raised if attribute contains a space and is passed in without an AttributeName. And + lhs & rhs are not attribute IDs by themselves. + """ + dynamodb = create_simple_table_and_return_client() + + dynamodb.put_item( + TableName="moto-test", + Item={"id": {"S": "1"}, "my Num": {"S": "1"}, "MyStr": {"S": "aaa"},}, + ) + + try: + dynamodb.update_item( + TableName="moto-test", + Key={"id": {"S": "1"}}, + UpdateExpression="SET MyStr = my Num", + ) + assert False, "Validation exception not thrown" + except dynamodb.exceptions.ClientError as e: + assert_raise_syntax_error(e, "Num", "my Num") + + +@mock_dynamodb +def test_summing_up_2_strings_raises_exception(): + """ + Update set supports different DynamoDB types but some operations are not supported. For example summing up 2 strings + raises an exception. It results in ClientError with code ValidationException: + Saying An operand in the update expression has an incorrect data type + """ + dynamodb = create_simple_table_and_return_client() + + try: + dynamodb.update_item( + TableName="moto-test", + Key={"id": {"S": "1"}}, + UpdateExpression="SET MyStr = MyStr + MyStr", + ) + assert False, "Validation exception not thrown" + except dynamodb.exceptions.ClientError as e: + assert_correct_client_error( + e, + "ValidationException", + "An operand in the update expression has an incorrect data type", + ) + + +# https://github.com/spulec/moto/issues/2806 +@mock_dynamodb +def test_update_item_with_attribute_in_right_hand_side(): + """ + After tokenization and building expression make sure referenced attributes are replaced with their current value + """ + dynamodb = create_simple_table_and_return_client() + + # Make sure there are 2 values + dynamodb.put_item( + TableName="moto-test", + Item={"id": {"S": "1"}, "myVal1": {"S": "Value1"}, "myVal2": {"S": "Value2"}}, + ) + + dynamodb.update_item( + TableName="moto-test", + Key={"id": {"S": "1"}}, + UpdateExpression="SET myVal1 = myVal2", + ) + + result = dynamodb.get_item(TableName="moto-test", Key={"id": {"S": "1"}}) + assert result["Item"]["myVal1"]["S"] == result["Item"]["myVal2"]["S"] == "Value2" + + +@mock_dynamodb +def test_multiple_updates(): + dynamodb = create_simple_table_and_return_client() + dynamodb.put_item( + TableName="moto-test", + Item={"id": {"S": "1"}, "myNum": {"N": "1"}, "path": {"N": "6"}}, + ) + dynamodb.update_item( + TableName="moto-test", + Key={"id": {"S": "1"}}, + UpdateExpression="SET myNum = #p + :val, newAttr = myNum", + ExpressionAttributeValues={":val": {"N": "1"}}, + ExpressionAttributeNames={"#p": "path"}, + ) + result = dynamodb.get_item(TableName="moto-test", Key={"id": {"S": "1"}})["Item"] + expected_result = { + "myNum": {"N": "7"}, + "newAttr": {"N": "1"}, + "path": {"N": "6"}, + "id": {"S": "1"}, + } + assert result == expected_result + + +@mock_dynamodb +def test_update_item_atomic_counter(): + table = "table_t" + ddb_mock = boto3.client("dynamodb", region_name="eu-west-3") + ddb_mock.create_table( + TableName=table, + KeySchema=[{"AttributeName": "t_id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "t_id", "AttributeType": "S"}], + BillingMode="PAY_PER_REQUEST", + ) + + key = {"t_id": {"S": "item1"}} + + ddb_mock.put_item( + TableName=table, + Item={"t_id": {"S": "item1"}, "n_i": {"N": "5"}, "n_f": {"N": "5.3"}}, + ) + + ddb_mock.update_item( + TableName=table, + Key=key, + UpdateExpression="set n_i = n_i + :inc1, n_f = n_f + :inc2", + ExpressionAttributeValues={":inc1": {"N": "1.2"}, ":inc2": {"N": "0.05"}}, + ) + updated_item = ddb_mock.get_item(TableName=table, Key=key)["Item"] + updated_item["n_i"]["N"].should.equal("6.2") + updated_item["n_f"]["N"].should.equal("5.35") + + +@mock_dynamodb +def test_update_item_atomic_counter_return_values(): + table = "table_t" + ddb_mock = boto3.client("dynamodb", region_name="eu-west-3") + ddb_mock.create_table( + TableName=table, + KeySchema=[{"AttributeName": "t_id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "t_id", "AttributeType": "S"}], + BillingMode="PAY_PER_REQUEST", + ) + + key = {"t_id": {"S": "item1"}} + + ddb_mock.put_item(TableName=table, Item={"t_id": {"S": "item1"}, "v": {"N": "5"}}) + + response = ddb_mock.update_item( + TableName=table, + Key=key, + UpdateExpression="set v = v + :inc", + ExpressionAttributeValues={":inc": {"N": "1"}}, + ReturnValues="UPDATED_OLD", + ) + assert ( + "v" in response["Attributes"] + ), "v has been updated, and should be returned here" + response["Attributes"]["v"]["N"].should.equal("5") + + # second update + response = ddb_mock.update_item( + TableName=table, + Key=key, + UpdateExpression="set v = v + :inc", + ExpressionAttributeValues={":inc": {"N": "1"}}, + ReturnValues="UPDATED_OLD", + ) + assert ( + "v" in response["Attributes"] + ), "v has been updated, and should be returned here" + response["Attributes"]["v"]["N"].should.equal("6") + + # third update + response = ddb_mock.update_item( + TableName=table, + Key=key, + UpdateExpression="set v = v + :inc", + ExpressionAttributeValues={":inc": {"N": "1"}}, + ReturnValues="UPDATED_NEW", + ) + assert ( + "v" in response["Attributes"] + ), "v has been updated, and should be returned here" + response["Attributes"]["v"]["N"].should.equal("8") + + +@mock_dynamodb +def test_update_item_atomic_counter_from_zero(): + table = "table_t" + ddb_mock = boto3.client("dynamodb", region_name="eu-west-1") + ddb_mock.create_table( + TableName=table, + KeySchema=[{"AttributeName": "t_id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "t_id", "AttributeType": "S"}], + BillingMode="PAY_PER_REQUEST", + ) + + key = {"t_id": {"S": "item1"}} + + ddb_mock.put_item( + TableName=table, Item=key, + ) + + ddb_mock.update_item( + TableName=table, + Key=key, + UpdateExpression="add n_i :inc1, n_f :inc2", + ExpressionAttributeValues={":inc1": {"N": "1.2"}, ":inc2": {"N": "-0.5"}}, + ) + updated_item = ddb_mock.get_item(TableName=table, Key=key)["Item"] + assert updated_item["n_i"]["N"] == "1.2" + assert updated_item["n_f"]["N"] == "-0.5" + + +@mock_dynamodb +def test_update_item_add_to_non_existent_set(): + table = "table_t" + ddb_mock = boto3.client("dynamodb", region_name="eu-west-1") + ddb_mock.create_table( + TableName=table, + KeySchema=[{"AttributeName": "t_id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "t_id", "AttributeType": "S"}], + BillingMode="PAY_PER_REQUEST", + ) + key = {"t_id": {"S": "item1"}} + ddb_mock.put_item( + TableName=table, Item=key, + ) + + ddb_mock.update_item( + TableName=table, + Key=key, + UpdateExpression="add s_i :s1", + ExpressionAttributeValues={":s1": {"SS": ["hello"]}}, + ) + updated_item = ddb_mock.get_item(TableName=table, Key=key)["Item"] + assert updated_item["s_i"]["SS"] == ["hello"] + + +@mock_dynamodb +def test_update_item_add_to_non_existent_number_set(): + table = "table_t" + ddb_mock = boto3.client("dynamodb", region_name="eu-west-1") + ddb_mock.create_table( + TableName=table, + KeySchema=[{"AttributeName": "t_id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "t_id", "AttributeType": "S"}], + BillingMode="PAY_PER_REQUEST", + ) + key = {"t_id": {"S": "item1"}} + ddb_mock.put_item( + TableName=table, Item=key, + ) + + ddb_mock.update_item( + TableName=table, + Key=key, + UpdateExpression="add s_i :s1", + ExpressionAttributeValues={":s1": {"NS": ["3"]}}, + ) + updated_item = ddb_mock.get_item(TableName=table, Key=key)["Item"] + assert updated_item["s_i"]["NS"] == ["3"] + + +@mock_dynamodb +def test_transact_write_items_fails_with_transaction_canceled_exception(): + table_schema = { + "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], + } + dynamodb = boto3.client("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema + ) + # Insert one item + dynamodb.put_item(TableName="test-table", Item={"id": {"S": "foo"}}) + # Update two items, the one that exists and another that doesn't + with pytest.raises(ClientError) as ex: + dynamodb.transact_write_items( + TransactItems=[ + { + "Update": { + "Key": {"id": {"S": "foo"}}, + "TableName": "test-table", + "UpdateExpression": "SET #k = :v", + "ConditionExpression": "attribute_exists(id)", + "ExpressionAttributeNames": {"#k": "key"}, + "ExpressionAttributeValues": {":v": {"S": "value"}}, + } + }, + { + "Update": { + "Key": {"id": {"S": "doesnotexist"}}, + "TableName": "test-table", + "UpdateExpression": "SET #e = :v", + "ConditionExpression": "attribute_exists(id)", + "ExpressionAttributeNames": {"#e": "key"}, + "ExpressionAttributeValues": {":v": {"S": "value"}}, + } + }, + ] + ) + ex.value.response["Error"]["Code"].should.equal("TransactionCanceledException") + ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.value.response["Error"]["Message"].should.equal( + "Transaction cancelled, please refer cancellation reasons for specific reasons [None, ConditionalCheckFailed]" + ) + + +@mock_dynamodb +def test_gsi_projection_type_keys_only(): + table_schema = { + "KeySchema": [{"AttributeName": "partitionKey", "KeyType": "HASH"}], + "GlobalSecondaryIndexes": [ + { + "IndexName": "GSI-K1", + "KeySchema": [ + {"AttributeName": "gsiK1PartitionKey", "KeyType": "HASH"}, + {"AttributeName": "gsiK1SortKey", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "KEYS_ONLY",}, + } + ], + "AttributeDefinitions": [ + {"AttributeName": "partitionKey", "AttributeType": "S"}, + {"AttributeName": "gsiK1PartitionKey", "AttributeType": "S"}, + {"AttributeName": "gsiK1SortKey", "AttributeType": "S"}, + ], + } + + item = { + "partitionKey": "pk-1", + "gsiK1PartitionKey": "gsi-pk", + "gsiK1SortKey": "gsi-sk", + "someAttribute": "lore ipsum", + } + + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema + ) + table = dynamodb.Table("test-table") + table.put_item(Item=item) + + items = table.query( + KeyConditionExpression=Key("gsiK1PartitionKey").eq("gsi-pk"), + IndexName="GSI-K1", + )["Items"] + items.should.have.length_of(1) + # Item should only include GSI Keys and Table Keys, as per the ProjectionType + items[0].should.equal( + { + "gsiK1PartitionKey": "gsi-pk", + "gsiK1SortKey": "gsi-sk", + "partitionKey": "pk-1", + } + ) + + +@mock_dynamodb +def test_gsi_projection_type_include(): + table_schema = { + "KeySchema": [{"AttributeName": "partitionKey", "KeyType": "HASH"}], + "GlobalSecondaryIndexes": [ + { + "IndexName": "GSI-INC", + "KeySchema": [ + {"AttributeName": "gsiK1PartitionKey", "KeyType": "HASH"}, + {"AttributeName": "gsiK1SortKey", "KeyType": "RANGE"}, + ], + "Projection": { + "ProjectionType": "INCLUDE", + "NonKeyAttributes": ["projectedAttribute"], + }, + } + ], + "AttributeDefinitions": [ + {"AttributeName": "partitionKey", "AttributeType": "S"}, + {"AttributeName": "gsiK1PartitionKey", "AttributeType": "S"}, + {"AttributeName": "gsiK1SortKey", "AttributeType": "S"}, + ], + } + + item = { + "partitionKey": "pk-1", + "gsiK1PartitionKey": "gsi-pk", + "gsiK1SortKey": "gsi-sk", + "projectedAttribute": "lore ipsum", + "nonProjectedAttribute": "dolor sit amet", + } + + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema + ) + table = dynamodb.Table("test-table") + table.put_item(Item=item) + + items = table.query( + KeyConditionExpression=Key("gsiK1PartitionKey").eq("gsi-pk"), + IndexName="GSI-INC", + )["Items"] + items.should.have.length_of(1) + # Item should only include keys and additionally projected attributes only + items[0].should.equal( + { + "gsiK1PartitionKey": "gsi-pk", + "gsiK1SortKey": "gsi-sk", + "partitionKey": "pk-1", + "projectedAttribute": "lore ipsum", + } + ) + + +@mock_dynamodb +def test_lsi_projection_type_keys_only(): + table_schema = { + "KeySchema": [ + {"AttributeName": "partitionKey", "KeyType": "HASH"}, + {"AttributeName": "sortKey", "KeyType": "RANGE"}, + ], + "LocalSecondaryIndexes": [ + { + "IndexName": "LSI", + "KeySchema": [ + {"AttributeName": "partitionKey", "KeyType": "HASH"}, + {"AttributeName": "lsiK1SortKey", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "KEYS_ONLY",}, + } + ], + "AttributeDefinitions": [ + {"AttributeName": "partitionKey", "AttributeType": "S"}, + {"AttributeName": "sortKey", "AttributeType": "S"}, + {"AttributeName": "lsiK1SortKey", "AttributeType": "S"}, + ], + } + + item = { + "partitionKey": "pk-1", + "sortKey": "sk-1", + "lsiK1SortKey": "lsi-sk", + "someAttribute": "lore ipsum", + } + + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema + ) + table = dynamodb.Table("test-table") + table.put_item(Item=item) + + items = table.query( + KeyConditionExpression=Key("partitionKey").eq("pk-1"), IndexName="LSI", + )["Items"] + items.should.have.length_of(1) + # Item should only include GSI Keys and Table Keys, as per the ProjectionType + items[0].should.equal( + {"partitionKey": "pk-1", "sortKey": "sk-1", "lsiK1SortKey": "lsi-sk"} + ) + + +@mock_dynamodb +@pytest.mark.parametrize( + "attr_name", + ["orders", "#placeholder"], + ids=["use attribute name", "use expression attribute name"], +) +def test_set_attribute_is_dropped_if_empty_after_update_expression(attr_name): + table_name, item_key, set_item = "test-table", "test-id", "test-data" + expression_attribute_names = {"#placeholder": "orders"} + client = boto3.client("dynamodb", region_name="us-east-1") + client.create_table( + TableName=table_name, + KeySchema=[{"AttributeName": "customer", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "customer", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + client.update_item( + TableName=table_name, + Key={"customer": {"S": item_key}}, + UpdateExpression="ADD {} :order".format(attr_name), + ExpressionAttributeNames=expression_attribute_names, + ExpressionAttributeValues={":order": {"SS": [set_item]}}, + ) + resp = client.scan(TableName=table_name, ProjectionExpression="customer, orders") + item = resp["Items"][0] + item.should.have.key("customer") + item.should.have.key("orders") + + client.update_item( + TableName=table_name, + Key={"customer": {"S": item_key}}, + UpdateExpression="DELETE {} :order".format(attr_name), + ExpressionAttributeNames=expression_attribute_names, + ExpressionAttributeValues={":order": {"SS": [set_item]}}, + ) + resp = client.scan(TableName=table_name, ProjectionExpression="customer, orders") + item = resp["Items"][0] + item.should.have.key("customer") + item.should_not.have.key("orders") + + +@mock_dynamodb +def test_transact_get_items_should_return_empty_map_for_non_existent_item(): + client = boto3.client("dynamodb", region_name="us-west-2") + table_name = "test-table" + key_schema = [{"AttributeName": "id", "KeyType": "HASH"}] + attribute_definitions = [{"AttributeName": "id", "AttributeType": "S"}] + client.create_table( + TableName=table_name, + KeySchema=key_schema, + AttributeDefinitions=attribute_definitions, + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + item = {"id": {"S": "1"}} + client.put_item(TableName=table_name, Item=item) + items = client.transact_get_items( + TransactItems=[ + {"Get": {"Key": {"id": {"S": "1"}}, "TableName": table_name}}, + {"Get": {"Key": {"id": {"S": "2"}}, "TableName": table_name}}, + ] + ).get("Responses", []) + items.should.have.length_of(2) + items[0].should.equal({"Item": item}) + items[1].should.equal({}) + + +@mock_dynamodb +def test_dynamodb_update_item_fails_on_string_sets(): + dynamodb = boto3.resource("dynamodb", region_name="eu-west-1") + client = boto3.client("dynamodb", region_name="eu-west-1") + + table = dynamodb.create_table( + TableName="test", + KeySchema=[{"AttributeName": "record_id", "KeyType": "HASH"},], + AttributeDefinitions=[{"AttributeName": "record_id", "AttributeType": "S"},], + BillingMode="PAY_PER_REQUEST", + ) + table.meta.client.get_waiter("table_exists").wait(TableName="test") + attribute = {"test_field": {"Value": {"SS": ["test1", "test2"],}, "Action": "PUT",}} + + client.update_item( + TableName="test", + Key={"record_id": {"S": "testrecord"}}, + AttributeUpdates=attribute, + ) + + +@mock_dynamodb +def test_update_item_add_to_list_using_legacy_attribute_updates(): + resource = boto3.resource("dynamodb", region_name="us-west-2") + resource.create_table( + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + TableName="TestTable", + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + table = resource.Table("TestTable") + table.wait_until_exists() + table.put_item(Item={"id": "list_add", "attr": ["a", "b", "c"]},) + + table.update_item( + TableName="TestTable", + Key={"id": "list_add"}, + AttributeUpdates={"attr": {"Action": "ADD", "Value": ["d", "e"]}}, + ) + + resp = table.get_item(Key={"id": "list_add"}) + resp["Item"]["attr"].should.equal(["a", "b", "c", "d", "e"]) + + +@mock_dynamodb +def test_get_item_for_non_existent_table_raises_error(): + client = boto3.client("dynamodb", "us-east-1") + with pytest.raises(ClientError) as ex: + client.get_item(TableName="non-existent", Key={"site-id": {"S": "foo"}}) + ex.value.response["Error"]["Code"].should.equal("ResourceNotFoundException") + ex.value.response["Error"]["Message"].should.equal("Requested resource not found") + + +@mock_dynamodb +def test_error_when_providing_expression_and_nonexpression_params(): + client = boto3.client("dynamodb", "eu-central-1") + table_name = "testtable" + client.create_table( + TableName=table_name, + KeySchema=[{"AttributeName": "pkey", "KeyType": "HASH"},], + AttributeDefinitions=[{"AttributeName": "pkey", "AttributeType": "S"},], + BillingMode="PAY_PER_REQUEST", + ) + + with pytest.raises(ClientError) as ex: + client.update_item( + TableName=table_name, + Key={"pkey": {"S": "testrecord"}}, + AttributeUpdates={ + "test_field": {"Value": {"SS": ["test1", "test2"],}, "Action": "PUT"} + }, + UpdateExpression="DELETE orders :order", + ExpressionAttributeValues={":order": {"SS": ["item"]}}, + ) + err = ex.value.response["Error"] + err["Code"].should.equal("ValidationException") + err["Message"].should.equal( + "Can not use both expression and non-expression parameters in the same request: Non-expression parameters: {AttributeUpdates} Expression parameters: {UpdateExpression}" + ) + + +@mock_dynamodb +def test_attribute_item_delete(): + name = "TestTable" + conn = boto3.client("dynamodb", region_name="eu-west-1") + conn.create_table( + TableName=name, + AttributeDefinitions=[{"AttributeName": "name", "AttributeType": "S"}], + KeySchema=[{"AttributeName": "name", "KeyType": "HASH"}], + BillingMode="PAY_PER_REQUEST", + ) + + item_name = "foo" + conn.put_item( + TableName=name, Item={"name": {"S": item_name}, "extra": {"S": "bar"}} + ) + + conn.update_item( + TableName=name, + Key={"name": {"S": item_name}}, + AttributeUpdates={"extra": {"Action": "DELETE"}}, + ) + items = conn.scan(TableName=name)["Items"] + items.should.equal([{"name": {"S": "foo"}}]) + + +@mock_dynamodb +def test_gsi_key_can_be_updated(): + name = "TestTable" + conn = boto3.client("dynamodb", region_name="eu-west-2") + conn.create_table( + TableName=name, + KeySchema=[{"AttributeName": "main_key", "KeyType": "HASH"}], + AttributeDefinitions=[ + {"AttributeName": "main_key", "AttributeType": "S"}, + {"AttributeName": "index_key", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + GlobalSecondaryIndexes=[ + { + "IndexName": "test_index", + "KeySchema": [{"AttributeName": "index_key", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "ALL",}, + "ProvisionedThroughput": { + "ReadCapacityUnits": 1, + "WriteCapacityUnits": 1, + }, + } + ], + ) + + conn.put_item( + TableName=name, + Item={ + "main_key": {"S": "testkey1"}, + "extra_data": {"S": "testdata"}, + "index_key": {"S": "indexkey1"}, + }, + ) + + conn.update_item( + TableName=name, + Key={"main_key": {"S": "testkey1"}}, + UpdateExpression="set index_key=:new_index_key", + ExpressionAttributeValues={":new_index_key": {"S": "new_value"}}, + ) + + item = conn.scan(TableName=name)["Items"][0] + item["index_key"].should.equal({"S": "new_value"}) + item["main_key"].should.equal({"S": "testkey1"}) + + +@mock_dynamodb +def test_gsi_key_cannot_be_empty(): + name = "TestTable" + conn = boto3.client("dynamodb", region_name="eu-west-2") + conn.create_table( + TableName=name, + KeySchema=[{"AttributeName": "main_key", "KeyType": "HASH"}], + AttributeDefinitions=[ + {"AttributeName": "main_key", "AttributeType": "S"}, + {"AttributeName": "index_key", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + GlobalSecondaryIndexes=[ + { + "IndexName": "test_index", + "KeySchema": [{"AttributeName": "index_key", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "ALL",}, + "ProvisionedThroughput": { + "ReadCapacityUnits": 1, + "WriteCapacityUnits": 1, + }, + } + ], + ) + + conn.put_item( + TableName=name, + Item={ + "main_key": {"S": "testkey1"}, + "extra_data": {"S": "testdata"}, + "index_key": {"S": "indexkey1"}, + }, + ) + + with pytest.raises(ClientError) as ex: + conn.update_item( + TableName=name, + Key={"main_key": {"S": "testkey1"}}, + UpdateExpression="set index_key=:new_index_key", + ExpressionAttributeValues={":new_index_key": {"S": ""}}, + ) + err = ex.value.response["Error"] + err["Code"].should.equal("ValidationException") + err["Message"].should.equal( + "One or more parameter values are not valid. The update expression attempted to update a secondary index key to a value that is not supported. The AttributeValue for a key attribute cannot contain an empty string value." + ) + + +@mock_dynamodb +def test_create_backup_for_non_existent_table_raises_error(): + client = boto3.client("dynamodb", "us-east-1") + with pytest.raises(ClientError) as ex: + client.create_backup(TableName="non-existent", BackupName="backup") + error = ex.value.response["Error"] + error["Code"].should.equal("TableNotFoundException") + error["Message"].should.equal("Table not found: non-existent") + + +@mock_dynamodb +def test_create_backup(): + client = boto3.client("dynamodb", "us-east-1") + table_name = "test-table" + client.create_table( + TableName=table_name, + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + backup_name = "backup-test-table" + resp = client.create_backup(TableName=table_name, BackupName=backup_name) + details = resp.get("BackupDetails") + details.should.have.key("BackupArn").should.contain(table_name) + details.should.have.key("BackupName").should.equal(backup_name) + details.should.have.key("BackupSizeBytes").should.be.a(int) + details.should.have.key("BackupStatus") + details.should.have.key("BackupType").should.equal("USER") + details.should.have.key("BackupCreationDateTime").should.be.a(datetime) + + +@mock_dynamodb +def test_create_multiple_backups_with_same_name(): + client = boto3.client("dynamodb", "us-east-1") + table_name = "test-table" + client.create_table( + TableName=table_name, + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + backup_name = "backup-test-table" + backup_arns = [] + for _ in range(4): + backup = client.create_backup(TableName=table_name, BackupName=backup_name).get( + "BackupDetails" + ) + backup["BackupName"].should.equal(backup_name) + backup_arns.should_not.contain(backup["BackupArn"]) + backup_arns.append(backup["BackupArn"]) + + +@mock_dynamodb +def test_describe_backup_for_non_existent_backup_raises_error(): + client = boto3.client("dynamodb", "us-east-1") + non_existent_arn = "arn:aws:dynamodb:us-east-1:123456789012:table/table-name/backup/01623095754481-2cfcd6f9" + with pytest.raises(ClientError) as ex: + client.describe_backup(BackupArn=non_existent_arn) + error = ex.value.response["Error"] + error["Code"].should.equal("BackupNotFoundException") + error["Message"].should.equal("Backup not found: {}".format(non_existent_arn)) + + +@mock_dynamodb +def test_describe_backup(): + client = boto3.client("dynamodb", "us-east-1") + table_name = "test-table" + table = client.create_table( + TableName=table_name, + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ).get("TableDescription") + backup_name = "backup-test-table" + backup_arn = ( + client.create_backup(TableName=table_name, BackupName=backup_name) + .get("BackupDetails") + .get("BackupArn") + ) + resp = client.describe_backup(BackupArn=backup_arn) + description = resp.get("BackupDescription") + details = description.get("BackupDetails") + details.should.have.key("BackupArn").should.contain(table_name) + details.should.have.key("BackupName").should.equal(backup_name) + details.should.have.key("BackupSizeBytes").should.be.a(int) + details.should.have.key("BackupStatus") + details.should.have.key("BackupType").should.equal("USER") + details.should.have.key("BackupCreationDateTime").should.be.a(datetime) + source = description.get("SourceTableDetails") + source.should.have.key("TableName").should.equal(table_name) + source.should.have.key("TableArn").should.equal(table["TableArn"]) + source.should.have.key("TableSizeBytes").should.be.a(int) + source.should.have.key("KeySchema").should.equal(table["KeySchema"]) + source.should.have.key("TableCreationDateTime").should.equal( + table["CreationDateTime"] + ) + source.should.have.key("ProvisionedThroughput").should.be.a(dict) + source.should.have.key("ItemCount").should.equal(table["ItemCount"]) + + +@mock_dynamodb +def test_list_backups_for_non_existent_table(): + client = boto3.client("dynamodb", "us-east-1") + resp = client.list_backups(TableName="non-existent") + resp["BackupSummaries"].should.have.length_of(0) + + +@mock_dynamodb +def test_list_backups(): + client = boto3.client("dynamodb", "us-east-1") + table_names = ["test-table-1", "test-table-2"] + backup_names = ["backup-1", "backup-2"] + for table_name in table_names: + client.create_table( + TableName=table_name, + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + for backup_name in backup_names: + client.create_backup(TableName=table_name, BackupName=backup_name) + resp = client.list_backups(BackupType="USER") + resp["BackupSummaries"].should.have.length_of(4) + for table_name in table_names: + resp = client.list_backups(TableName=table_name) + resp["BackupSummaries"].should.have.length_of(2) + for summary in resp["BackupSummaries"]: + summary.should.have.key("TableName").should.equal(table_name) + summary.should.have.key("TableArn").should.contain(table_name) + summary.should.have.key("BackupName").should.be.within(backup_names) + summary.should.have.key("BackupArn") + summary.should.have.key("BackupCreationDateTime").should.be.a(datetime) + summary.should.have.key("BackupStatus") + summary.should.have.key("BackupType").should.be.within(["USER", "SYSTEM"]) + summary.should.have.key("BackupSizeBytes").should.be.a(int) + + +@mock_dynamodb +def test_restore_table_from_non_existent_backup_raises_error(): + client = boto3.client("dynamodb", "us-east-1") + non_existent_arn = "arn:aws:dynamodb:us-east-1:123456789012:table/table-name/backup/01623095754481-2cfcd6f9" + with pytest.raises(ClientError) as ex: + client.restore_table_from_backup( + TargetTableName="from-backup", BackupArn=non_existent_arn + ) + error = ex.value.response["Error"] + error["Code"].should.equal("BackupNotFoundException") + error["Message"].should.equal("Backup not found: {}".format(non_existent_arn)) + + +@mock_dynamodb +def test_restore_table_from_backup_raises_error_when_table_already_exists(): + client = boto3.client("dynamodb", "us-east-1") + table_name = "test-table" + client.create_table( + TableName=table_name, + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + resp = client.create_backup(TableName=table_name, BackupName="backup") + backup = resp.get("BackupDetails") + with pytest.raises(ClientError) as ex: + client.restore_table_from_backup( + TargetTableName=table_name, BackupArn=backup["BackupArn"] + ) + error = ex.value.response["Error"] + error["Code"].should.equal("TableAlreadyExistsException") + error["Message"].should.equal("Table already exists: {}".format(table_name)) + + +@mock_dynamodb +def test_restore_table_from_backup(): + client = boto3.client("dynamodb", "us-east-1") + table_name = "test-table" + resp = client.create_table( + TableName=table_name, + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + table = resp.get("TableDescription") + for i in range(5): + client.put_item(TableName=table_name, Item={"id": {"S": "item %d" % i}}) + + backup_arn = ( + client.create_backup(TableName=table_name, BackupName="backup") + .get("BackupDetails") + .get("BackupArn") + ) + + restored_table_name = "restored-from-backup" + restored = client.restore_table_from_backup( + TargetTableName=restored_table_name, BackupArn=backup_arn + ).get("TableDescription") + restored.should.have.key("AttributeDefinitions").should.equal( + table["AttributeDefinitions"] + ) + restored.should.have.key("TableName").should.equal(restored_table_name) + restored.should.have.key("KeySchema").should.equal(table["KeySchema"]) + restored.should.have.key("TableStatus") + restored.should.have.key("ItemCount").should.equal(5) + restored.should.have.key("TableArn").should.contain(restored_table_name) + restored.should.have.key("RestoreSummary").should.be.a(dict) + summary = restored.get("RestoreSummary") + summary.should.have.key("SourceBackupArn").should.equal(backup_arn) + summary.should.have.key("SourceTableArn").should.equal(table["TableArn"]) + summary.should.have.key("RestoreDateTime").should.be.a(datetime) + summary.should.have.key("RestoreInProgress").should.equal(False) + + +@mock_dynamodb +def test_restore_table_to_point_in_time(): + client = boto3.client("dynamodb", "us-east-1") + table_name = "test-table" + resp = client.create_table( + TableName=table_name, + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + table = resp.get("TableDescription") + for i in range(5): + client.put_item(TableName=table_name, Item={"id": {"S": "item %d" % i}}) + + restored_table_name = "restored-from-pit" + restored = client.restore_table_to_point_in_time( + TargetTableName=restored_table_name, SourceTableName=table_name + ).get("TableDescription") + restored.should.have.key("TableName").should.equal(restored_table_name) + restored.should.have.key("KeySchema").should.equal(table["KeySchema"]) + restored.should.have.key("TableStatus") + restored.should.have.key("ItemCount").should.equal(5) + restored.should.have.key("TableArn").should.contain(restored_table_name) + restored.should.have.key("RestoreSummary").should.be.a(dict) + summary = restored.get("RestoreSummary") + summary.should.have.key("SourceTableArn").should.equal(table["TableArn"]) + summary.should.have.key("RestoreDateTime").should.be.a(datetime) + summary.should.have.key("RestoreInProgress").should.equal(False) + + +@mock_dynamodb +def test_restore_table_to_point_in_time_raises_error_when_source_not_exist(): + client = boto3.client("dynamodb", "us-east-1") + table_name = "test-table" + restored_table_name = "restored-from-pit" + with pytest.raises(ClientError) as ex: + client.restore_table_to_point_in_time( + TargetTableName=restored_table_name, SourceTableName=table_name + ) + error = ex.value.response["Error"] + error["Code"].should.equal("SourceTableNotFoundException") + error["Message"].should.equal("Source table not found: %s" % table_name) + + +@mock_dynamodb +def test_restore_table_to_point_in_time_raises_error_when_dest_exist(): + client = boto3.client("dynamodb", "us-east-1") + table_name = "test-table" + restored_table_name = "restored-from-pit" + client.create_table( + TableName=table_name, + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + client.create_table( + TableName=restored_table_name, + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + with pytest.raises(ClientError) as ex: + client.restore_table_to_point_in_time( + TargetTableName=restored_table_name, SourceTableName=table_name + ) + error = ex.value.response["Error"] + error["Code"].should.equal("TableAlreadyExistsException") + error["Message"].should.equal("Table already exists: %s" % restored_table_name) + + +@mock_dynamodb +def test_delete_non_existent_backup_raises_error(): + client = boto3.client("dynamodb", "us-east-1") + non_existent_arn = "arn:aws:dynamodb:us-east-1:123456789012:table/table-name/backup/01623095754481-2cfcd6f9" + with pytest.raises(ClientError) as ex: + client.delete_backup(BackupArn=non_existent_arn) + error = ex.value.response["Error"] + error["Code"].should.equal("BackupNotFoundException") + error["Message"].should.equal("Backup not found: {}".format(non_existent_arn)) + + +@mock_dynamodb +def test_delete_backup(): + client = boto3.client("dynamodb", "us-east-1") + table_name = "test-table-1" + backup_names = ["backup-1", "backup-2"] + client.create_table( + TableName=table_name, + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + for backup_name in backup_names: + client.create_backup(TableName=table_name, BackupName=backup_name) + resp = client.list_backups(TableName=table_name, BackupType="USER") + resp["BackupSummaries"].should.have.length_of(2) + backup_to_delete = resp["BackupSummaries"][0]["BackupArn"] + backup_deleted = client.delete_backup(BackupArn=backup_to_delete).get( + "BackupDescription" + ) + backup_deleted.should.have.key("SourceTableDetails") + backup_deleted.should.have.key("BackupDetails") + details = backup_deleted["BackupDetails"] + details.should.have.key("BackupArn").should.equal(backup_to_delete) + details.should.have.key("BackupName").should.be.within(backup_names) + details.should.have.key("BackupStatus").should.equal("DELETED") + resp = client.list_backups(TableName=table_name, BackupType="USER") + resp["BackupSummaries"].should.have.length_of(1) + + +@mock_dynamodb +def test_source_and_restored_table_items_are_not_linked(): + client = boto3.client("dynamodb", "us-east-1") + + def add_guids_to_table(table, num_items): + guids = [] + for _ in range(num_items): + guid = str(uuid.uuid4()) + client.put_item(TableName=table, Item={"id": {"S": guid}}) + guids.append(guid) + return guids + + source_table_name = "source-table" + client.create_table( + TableName=source_table_name, + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + guids_original = add_guids_to_table(source_table_name, 5) + + backup_arn = ( + client.create_backup(TableName=source_table_name, BackupName="backup") + .get("BackupDetails") + .get("BackupArn") + ) + guids_added_after_backup = add_guids_to_table(source_table_name, 5) + + restored_table_name = "restored-from-backup" + client.restore_table_from_backup( + TargetTableName=restored_table_name, BackupArn=backup_arn + ) + guids_added_after_restore = add_guids_to_table(restored_table_name, 5) + + source_table_items = client.scan(TableName=source_table_name) + source_table_items.should.have.key("Count").should.equal(10) + source_table_guids = [x["id"]["S"] for x in source_table_items["Items"]] + set(source_table_guids).should.equal( + set(guids_original) | set(guids_added_after_backup) + ) + + restored_table_items = client.scan(TableName=restored_table_name) + restored_table_items.should.have.key("Count").should.equal(10) + restored_table_guids = [x["id"]["S"] for x in restored_table_items["Items"]] + set(restored_table_guids).should.equal( + set(guids_original) | set(guids_added_after_restore) + ) + + +@mock_dynamodb +@pytest.mark.parametrize("region", ["eu-central-1", "ap-south-1"]) +def test_describe_endpoints(region): + client = boto3.client("dynamodb", region) + res = client.describe_endpoints()["Endpoints"] + res.should.equal( + [ + { + "Address": "dynamodb.{}.amazonaws.com".format(region), + "CachePeriodInMinutes": 1440, + }, + ] + ) + + +@mock_dynamodb +def test_update_non_existing_item_raises_error_and_does_not_contain_item_afterwards(): + """ + https://github.com/spulec/moto/issues/3729 + Exception is raised, but item was persisted anyway + Happened because we would create a placeholder, before validating/executing the UpdateExpression + :return: + """ + name = "TestTable" + conn = boto3.client("dynamodb", region_name="us-west-2") + hkey = "primary_partition_key" + conn.create_table( + TableName=name, + KeySchema=[{"AttributeName": hkey, "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": hkey, "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + update_expression = { + "Key": {hkey: "some_identification_string"}, + "UpdateExpression": "set #AA.#AB = :aa", + "ExpressionAttributeValues": {":aa": "abc"}, + "ExpressionAttributeNames": {"#AA": "some_dict", "#AB": "key1"}, + "ConditionExpression": "attribute_not_exists(#AA.#AB)", + } + table = boto3.resource("dynamodb", region_name="us-west-2").Table(name) + with pytest.raises(ClientError) as err: + table.update_item(**update_expression) + err.value.response["Error"]["Code"].should.equal("ValidationException") + + conn.scan(TableName=name)["Items"].should.have.length_of(0) + + +@mock_dynamodb +def test_batch_write_item(): + conn = boto3.resource("dynamodb", region_name="us-west-2") + tables = [f"table-{i}" for i in range(3)] + for name in tables: + conn.create_table( + TableName=name, + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + BillingMode="PAY_PER_REQUEST", + ) + + conn.batch_write_item( + RequestItems={ + tables[0]: [{"PutRequest": {"Item": {"id": "0"}}}], + tables[1]: [{"PutRequest": {"Item": {"id": "1"}}}], + tables[2]: [{"PutRequest": {"Item": {"id": "2"}}}], + } + ) + + for idx, name in enumerate(tables): + table = conn.Table(f"table-{idx}") + res = table.get_item(Key={"id": str(idx)}) + assert res["Item"].should.equal({"id": str(idx)}) + scan = table.scan() + assert scan["Count"].should.equal(1) + + conn.batch_write_item( + RequestItems={ + tables[0]: [{"DeleteRequest": {"Key": {"id": "0"}}}], + tables[1]: [{"DeleteRequest": {"Key": {"id": "1"}}}], + tables[2]: [{"DeleteRequest": {"Key": {"id": "2"}}}], + } + ) + + for idx, name in enumerate(tables): + table = conn.Table(f"table-{idx}") + scan = table.scan() + assert scan["Count"].should.equal(0) + + +@mock_dynamodb +def test_gsi_lastevaluatedkey(): + # github.com/spulec/moto/issues/3968 + conn = boto3.resource("dynamodb", region_name="us-west-2") + name = "test-table" + table = conn.Table(name) + + conn.create_table( + TableName=name, + KeySchema=[{"AttributeName": "main_key", "KeyType": "HASH"}], + AttributeDefinitions=[ + {"AttributeName": "main_key", "AttributeType": "S"}, + {"AttributeName": "index_key", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + GlobalSecondaryIndexes=[ + { + "IndexName": "test_index", + "KeySchema": [{"AttributeName": "index_key", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "ALL",}, + "ProvisionedThroughput": { + "ReadCapacityUnits": 1, + "WriteCapacityUnits": 1, + }, + } + ], + ) + + table.put_item( + Item={ + "main_key": "testkey1", + "extra_data": "testdata", + "index_key": "indexkey", + }, + ) + table.put_item( + Item={ + "main_key": "testkey2", + "extra_data": "testdata", + "index_key": "indexkey", + }, + ) + + response = table.query( + Limit=1, + KeyConditionExpression=Key("index_key").eq("indexkey"), + IndexName="test_index", + ) + + items = response["Items"] + items.should.have.length_of(1) + items[0].should.equal( + {"main_key": "testkey1", "extra_data": "testdata", "index_key": "indexkey"} + ) + + last_evaluated_key = response["LastEvaluatedKey"] + last_evaluated_key.should.have.length_of(2) + last_evaluated_key.should.equal({"main_key": "testkey1", "index_key": "indexkey"}) + + +@mock_dynamodb +def test_filter_expression_execution_order(): + # As mentioned here: https://github.com/spulec/moto/issues/3909 + # and documented here: https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Query.html#Query.FilterExpression + # the filter expression should be evaluated after the query. + # The same applies to scan operations: + # https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Scan.html#Scan.FilterExpression + + # If we set limit=1 and apply a filter expression whixh excludes the first result + # then we should get no items in response. + + conn = boto3.resource("dynamodb", region_name="us-west-2") + name = "test-filter-expression-table" + table = conn.Table(name) + + conn.create_table( + TableName=name, + KeySchema=[ + {"AttributeName": "hash_key", "KeyType": "HASH"}, + {"AttributeName": "range_key", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "hash_key", "AttributeType": "S"}, + {"AttributeName": "range_key", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + table.put_item( + Item={"hash_key": "keyvalue", "range_key": "A", "filtered_attribute": "Y"}, + ) + table.put_item( + Item={"hash_key": "keyvalue", "range_key": "B", "filtered_attribute": "Z"}, + ) + + # test query + + query_response_1 = table.query( + Limit=1, + KeyConditionExpression=Key("hash_key").eq("keyvalue"), + FilterExpression=Attr("filtered_attribute").eq("Z"), + ) + + query_items_1 = query_response_1["Items"] + query_items_1.should.have.length_of(0) + + query_last_evaluated_key = query_response_1["LastEvaluatedKey"] + query_last_evaluated_key.should.have.length_of(2) + query_last_evaluated_key.should.equal({"hash_key": "keyvalue", "range_key": "A"}) + + query_response_2 = table.query( + Limit=1, + KeyConditionExpression=Key("hash_key").eq("keyvalue"), + FilterExpression=Attr("filtered_attribute").eq("Z"), + ExclusiveStartKey=query_last_evaluated_key, + ) + + query_items_2 = query_response_2["Items"] + query_items_2.should.have.length_of(1) + query_items_2[0].should.equal( + {"hash_key": "keyvalue", "filtered_attribute": "Z", "range_key": "B"} + ) + + # test scan + + scan_response_1 = table.scan( + Limit=1, FilterExpression=Attr("filtered_attribute").eq("Z"), + ) + + scan_items_1 = scan_response_1["Items"] + scan_items_1.should.have.length_of(0) + + scan_last_evaluated_key = scan_response_1["LastEvaluatedKey"] + scan_last_evaluated_key.should.have.length_of(2) + scan_last_evaluated_key.should.equal({"hash_key": "keyvalue", "range_key": "A"}) + + scan_response_2 = table.scan( + Limit=1, + FilterExpression=Attr("filtered_attribute").eq("Z"), + ExclusiveStartKey=query_last_evaluated_key, + ) + + scan_items_2 = scan_response_2["Items"] + scan_items_2.should.have.length_of(1) + scan_items_2[0].should.equal( + {"hash_key": "keyvalue", "filtered_attribute": "Z", "range_key": "B"} + ) + + +@mock_dynamodb +def test_projection_expression_execution_order(): + # projection expression needs to be applied after calculation of + # LastEvaluatedKey as it is possible for LastEvaluatedKey to + # include attributes which are not projected. + + conn = boto3.resource("dynamodb", region_name="us-west-2") + name = "test-projection-expression-with-gsi" + table = conn.Table(name) + + conn.create_table( + TableName=name, + KeySchema=[ + {"AttributeName": "hash_key", "KeyType": "HASH"}, + {"AttributeName": "range_key", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "hash_key", "AttributeType": "S"}, + {"AttributeName": "range_key", "AttributeType": "S"}, + {"AttributeName": "index_key", "AttributeType": "S"}, + ], + GlobalSecondaryIndexes=[ + { + "IndexName": "test_index", + "KeySchema": [{"AttributeName": "index_key", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "ALL",}, + "ProvisionedThroughput": { + "ReadCapacityUnits": 1, + "WriteCapacityUnits": 1, + }, + } + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + table.put_item(Item={"hash_key": "keyvalue", "range_key": "A", "index_key": "Z"},) + table.put_item(Item={"hash_key": "keyvalue", "range_key": "B", "index_key": "Z"},) + + # test query + + # if projection expression is applied before LastEvaluatedKey is computed + # then this raises an exception. + table.query( + Limit=1, + IndexName="test_index", + KeyConditionExpression=Key("index_key").eq("Z"), + ProjectionExpression="#a", + ExpressionAttributeNames={"#a": "hashKey"}, + ) + # if projection expression is applied before LastEvaluatedKey is computed + # then this raises an exception. + table.scan( + Limit=1, + IndexName="test_index", + ProjectionExpression="#a", + ExpressionAttributeNames={"#a": "hashKey"}, ) diff --git a/tests/test_dynamodb2/test_dynamodb_cloudformation.py b/tests/test_dynamodb/test_dynamodb_cloudformation.py similarity index 95% rename from tests/test_dynamodb2/test_dynamodb_cloudformation.py rename to tests/test_dynamodb/test_dynamodb_cloudformation.py index a953151a9d01..bddc931a6618 100644 --- a/tests/test_dynamodb2/test_dynamodb_cloudformation.py +++ b/tests/test_dynamodb/test_dynamodb_cloudformation.py @@ -2,7 +2,7 @@ import json import sure # noqa # pylint: disable=unused-import -from moto import mock_cloudformation, mock_dynamodb2 +from moto import mock_cloudformation, mock_dynamodb template_create_table = { @@ -30,7 +30,7 @@ } -@mock_dynamodb2 +@mock_dynamodb @mock_cloudformation def test_delete_stack_dynamo_template_boto3(): conn = boto3.client("cloudformation", region_name="us-east-1") diff --git a/tests/test_dynamodb2/test_dynamodb_condition_expressions.py b/tests/test_dynamodb/test_dynamodb_condition_expressions.py similarity index 99% rename from tests/test_dynamodb2/test_dynamodb_condition_expressions.py rename to tests/test_dynamodb/test_dynamodb_condition_expressions.py index e4637ce8018c..e468aec194aa 100644 --- a/tests/test_dynamodb2/test_dynamodb_condition_expressions.py +++ b/tests/test_dynamodb/test_dynamodb_condition_expressions.py @@ -4,10 +4,10 @@ import boto3 import pytest import sure # noqa # pylint: disable=unused-import -from moto import mock_dynamodb2 +from moto import mock_dynamodb -@mock_dynamodb2 +@mock_dynamodb def test_condition_expression_with_dot_in_attr_name(): dynamodb = boto3.resource("dynamodb", region_name="us-east-2") table_name = "Test" @@ -46,7 +46,7 @@ def test_condition_expression_with_dot_in_attr_name(): ) -@mock_dynamodb2 +@mock_dynamodb def test_condition_expressions(): client = boto3.client("dynamodb", region_name="us-east-1") @@ -241,7 +241,7 @@ def _assert_conditional_check_failed_exception(exc): err["Message"].should.equal("The conditional request failed") -@mock_dynamodb2 +@mock_dynamodb def test_condition_expression_numerical_attribute(): dynamodb = boto3.resource("dynamodb", region_name="us-east-1") dynamodb.create_table( @@ -284,7 +284,7 @@ def update_numerical_con_expr(key, con_expr, res, table): ) -@mock_dynamodb2 +@mock_dynamodb def test_condition_expression__attr_doesnt_exist(): client = boto3.client("dynamodb", region_name="us-east-1") @@ -322,7 +322,7 @@ def update_if_attr_doesnt_exist(): _assert_conditional_check_failed_exception(exc) -@mock_dynamodb2 +@mock_dynamodb def test_condition_expression__or_order(): client = boto3.client("dynamodb", region_name="us-east-1") @@ -345,7 +345,7 @@ def test_condition_expression__or_order(): ) -@mock_dynamodb2 +@mock_dynamodb def test_condition_expression__and_order(): client = boto3.client("dynamodb", region_name="us-east-1") @@ -370,7 +370,7 @@ def test_condition_expression__and_order(): _assert_conditional_check_failed_exception(exc) -@mock_dynamodb2 +@mock_dynamodb def test_condition_expression_with_reserved_keyword_as_attr_name(): dynamodb = boto3.resource("dynamodb", region_name="us-east-2") table_name = "Test" diff --git a/tests/test_dynamodb2/test_dynamodb_consumedcapacity.py b/tests/test_dynamodb/test_dynamodb_consumedcapacity.py similarity index 98% rename from tests/test_dynamodb2/test_dynamodb_consumedcapacity.py rename to tests/test_dynamodb/test_dynamodb_consumedcapacity.py index 055e24d3fa1e..0d367c75adf1 100644 --- a/tests/test_dynamodb2/test_dynamodb_consumedcapacity.py +++ b/tests/test_dynamodb/test_dynamodb_consumedcapacity.py @@ -3,10 +3,10 @@ import sure # noqa # pylint: disable=unused-import from botocore.exceptions import ClientError -from moto import mock_dynamodb2 +from moto import mock_dynamodb -@mock_dynamodb2 +@mock_dynamodb def test_error_on_wrong_value_for_consumed_capacity(): resource = boto3.resource("dynamodb", region_name="ap-northeast-3") client = boto3.client("dynamodb", region_name="ap-northeast-3") @@ -30,7 +30,7 @@ def test_error_on_wrong_value_for_consumed_capacity(): ) -@mock_dynamodb2 +@mock_dynamodb def test_consumed_capacity_get_unknown_item(): conn = boto3.client("dynamodb", region_name="us-east-1") conn.create_table( @@ -52,7 +52,7 @@ def test_consumed_capacity_get_unknown_item(): ) -@mock_dynamodb2 +@mock_dynamodb @pytest.mark.parametrize( "capacity,should_have_capacity,should_have_table", [ diff --git a/tests/test_dynamodb2/test_dynamodb_create_table.py b/tests/test_dynamodb/test_dynamodb_create_table.py similarity index 98% rename from tests/test_dynamodb2/test_dynamodb_create_table.py rename to tests/test_dynamodb/test_dynamodb_create_table.py index 999c5a06557d..7bdb5ed71023 100644 --- a/tests/test_dynamodb2/test_dynamodb_create_table.py +++ b/tests/test_dynamodb/test_dynamodb_create_table.py @@ -4,11 +4,11 @@ from datetime import datetime import pytest -from moto import mock_dynamodb2 +from moto import mock_dynamodb from moto.core import ACCOUNT_ID -@mock_dynamodb2 +@mock_dynamodb def test_create_table_standard(): client = boto3.client("dynamodb", region_name="us-east-1") client.create_table( @@ -52,7 +52,7 @@ def test_create_table_standard(): actual.should.have.key("ItemCount").equal(0) -@mock_dynamodb2 +@mock_dynamodb def test_create_table_with_local_index(): client = boto3.client("dynamodb", region_name="us-east-1") client.create_table( @@ -119,7 +119,7 @@ def test_create_table_with_local_index(): actual.should.have.key("ItemCount").equal(0) -@mock_dynamodb2 +@mock_dynamodb def test_create_table_with_gsi(): dynamodb = boto3.client("dynamodb", region_name="us-east-1") @@ -196,7 +196,7 @@ def test_create_table_with_gsi(): ) -@mock_dynamodb2 +@mock_dynamodb def test_create_table_with_stream_specification(): conn = boto3.client("dynamodb", region_name="us-east-1") @@ -223,7 +223,7 @@ def test_create_table_with_stream_specification(): resp["TableDescription"].should.contain("StreamSpecification") -@mock_dynamodb2 +@mock_dynamodb def test_create_table_with_tags(): client = boto3.client("dynamodb", region_name="us-east-1") @@ -241,7 +241,7 @@ def test_create_table_with_tags(): resp.should.have.key("Tags").equals([{"Key": "tk", "Value": "tv"}]) -@mock_dynamodb2 +@mock_dynamodb def test_create_table_pay_per_request(): client = boto3.client("dynamodb", region_name="us-east-1") client.create_table( @@ -266,7 +266,7 @@ def test_create_table_pay_per_request(): ) -@mock_dynamodb2 +@mock_dynamodb def test_create_table__provisioned_throughput(): client = boto3.client("dynamodb", region_name="us-east-1") client.create_table( @@ -289,7 +289,7 @@ def test_create_table__provisioned_throughput(): ) -@mock_dynamodb2 +@mock_dynamodb def test_create_table_without_specifying_throughput(): dynamodb_client = boto3.client("dynamodb", region_name="us-east-1") @@ -310,7 +310,7 @@ def test_create_table_without_specifying_throughput(): ) -@mock_dynamodb2 +@mock_dynamodb def test_create_table_error_pay_per_request_with_provisioned_param(): client = boto3.client("dynamodb", region_name="us-east-1") @@ -335,7 +335,7 @@ def test_create_table_error_pay_per_request_with_provisioned_param(): ) -@mock_dynamodb2 +@mock_dynamodb def test_create_table_with_ssespecification__false(): client = boto3.client("dynamodb", region_name="us-east-1") client.create_table( @@ -356,7 +356,7 @@ def test_create_table_with_ssespecification__false(): actual.shouldnt.have.key("SSEDescription") -@mock_dynamodb2 +@mock_dynamodb def test_create_table_with_ssespecification__true(): client = boto3.client("dynamodb", region_name="us-east-1") client.create_table( @@ -382,7 +382,7 @@ def test_create_table_with_ssespecification__true(): ) # Default KMS key for DynamoDB -@mock_dynamodb2 +@mock_dynamodb def test_create_table_with_ssespecification__custom_kms_key(): client = boto3.client("dynamodb", region_name="us-east-1") client.create_table( diff --git a/tests/test_dynamodb2/test_dynamodb_executor.py b/tests/test_dynamodb/test_dynamodb_executor.py similarity index 97% rename from tests/test_dynamodb2/test_dynamodb_executor.py rename to tests/test_dynamodb/test_dynamodb_executor.py index 159c90523969..8df9ef7bb694 100644 --- a/tests/test_dynamodb2/test_dynamodb_executor.py +++ b/tests/test_dynamodb/test_dynamodb_executor.py @@ -1,10 +1,10 @@ import pytest -from moto.dynamodb2.exceptions import IncorrectOperandType, IncorrectDataType -from moto.dynamodb2.models import Item, DynamoType -from moto.dynamodb2.parsing.executors import UpdateExpressionExecutor -from moto.dynamodb2.parsing.expressions import UpdateExpressionParser -from moto.dynamodb2.parsing.validators import UpdateExpressionValidator +from moto.dynamodb.exceptions import IncorrectOperandType, IncorrectDataType +from moto.dynamodb.models import Item, DynamoType +from moto.dynamodb.parsing.executors import UpdateExpressionExecutor +from moto.dynamodb.parsing.expressions import UpdateExpressionParser +from moto.dynamodb.parsing.validators import UpdateExpressionValidator def test_execution_of_if_not_exists_not_existing_value(table): diff --git a/tests/test_dynamodb2/test_dynamodb_expression_tokenizer.py b/tests/test_dynamodb/test_dynamodb_expression_tokenizer.py similarity index 98% rename from tests/test_dynamodb2/test_dynamodb_expression_tokenizer.py rename to tests/test_dynamodb/test_dynamodb_expression_tokenizer.py index ddfb81d1a258..bc4342dbba1f 100644 --- a/tests/test_dynamodb2/test_dynamodb_expression_tokenizer.py +++ b/tests/test_dynamodb/test_dynamodb_expression_tokenizer.py @@ -1,8 +1,8 @@ -from moto.dynamodb2.exceptions import ( +from moto.dynamodb.exceptions import ( InvalidTokenException, InvalidExpressionAttributeNameKey, ) -from moto.dynamodb2.parsing.tokens import ExpressionTokenizer, Token +from moto.dynamodb.parsing.tokens import ExpressionTokenizer, Token def test_expression_tokenizer_single_set_action(): diff --git a/tests/test_dynamodb2/test_dynamodb_expressions.py b/tests/test_dynamodb/test_dynamodb_expressions.py similarity index 98% rename from tests/test_dynamodb2/test_dynamodb_expressions.py rename to tests/test_dynamodb/test_dynamodb_expressions.py index 2c82d8bc4873..eeaa2abfc6d6 100644 --- a/tests/test_dynamodb2/test_dynamodb_expressions.py +++ b/tests/test_dynamodb/test_dynamodb_expressions.py @@ -1,6 +1,6 @@ -from moto.dynamodb2.exceptions import InvalidTokenException -from moto.dynamodb2.parsing.expressions import UpdateExpressionParser -from moto.dynamodb2.parsing.reserved_keywords import ReservedKeywords +from moto.dynamodb.exceptions import InvalidTokenException +from moto.dynamodb.parsing.expressions import UpdateExpressionParser +from moto.dynamodb.parsing.reserved_keywords import ReservedKeywords def test_get_reserved_keywords(): diff --git a/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py b/tests/test_dynamodb/test_dynamodb_table_with_range_key.py similarity index 98% rename from tests/test_dynamodb2/test_dynamodb_table_with_range_key.py rename to tests/test_dynamodb/test_dynamodb_table_with_range_key.py index 43b34981bf35..aeed5c29979f 100644 --- a/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py +++ b/tests/test_dynamodb/test_dynamodb_table_with_range_key.py @@ -6,11 +6,11 @@ import sure # noqa # pylint: disable=unused-import import pytest -from moto import mock_dynamodb2 +from moto import mock_dynamodb from uuid import uuid4 -@mock_dynamodb2 +@mock_dynamodb def test_get_item_without_range_key_boto3(): client = boto3.resource("dynamodb", region_name="us-east-1") table = client.create_table( @@ -37,7 +37,7 @@ def test_get_item_without_range_key_boto3(): ex.value.response["Error"]["Message"].should.equal("Validation Exception") -@mock_dynamodb2 +@mock_dynamodb def test_query_filter_boto3(): table_schema = { "KeySchema": [ @@ -80,7 +80,7 @@ def test_query_filter_boto3(): res["Items"].should.equal([{"pk": "pk", "sk": "sk-1"}, {"pk": "pk", "sk": "sk-2"}]) -@mock_dynamodb2 +@mock_dynamodb def test_boto3_conditions(): dynamodb = boto3.resource("dynamodb", region_name="us-east-1") @@ -154,7 +154,7 @@ def test_boto3_conditions(): results["Count"].should.equal(1) -@mock_dynamodb2 +@mock_dynamodb def test_boto3_conditions_ignorecase(): dynamodb = boto3.client("dynamodb", region_name="us-east-1") @@ -220,7 +220,7 @@ def test_boto3_conditions_ignorecase(): ) -@mock_dynamodb2 +@mock_dynamodb def test_boto3_put_item_with_conditions(): import botocore @@ -294,7 +294,7 @@ def _create_table_with_range_key(): return dynamodb.Table("users") -@mock_dynamodb2 +@mock_dynamodb def test_update_item_range_key_set(): table = _create_table_with_range_key() table.put_item( @@ -331,7 +331,7 @@ def test_update_item_range_key_set(): ) -@mock_dynamodb2 +@mock_dynamodb def test_update_item_does_not_exist_is_created(): table = _create_table_with_range_key() @@ -363,7 +363,7 @@ def test_update_item_does_not_exist_is_created(): ) -@mock_dynamodb2 +@mock_dynamodb def test_update_item_add_value(): table = _create_table_with_range_key() @@ -386,7 +386,7 @@ def test_update_item_add_value(): ) -@mock_dynamodb2 +@mock_dynamodb def test_update_item_add_value_string_set(): table = _create_table_with_range_key() @@ -417,7 +417,7 @@ def test_update_item_add_value_string_set(): ) -@mock_dynamodb2 +@mock_dynamodb def test_update_item_delete_value_string_set(): table = _create_table_with_range_key() @@ -444,7 +444,7 @@ def test_update_item_delete_value_string_set(): ) -@mock_dynamodb2 +@mock_dynamodb def test_update_item_add_value_does_not_exist_is_created(): table = _create_table_with_range_key() @@ -463,7 +463,7 @@ def test_update_item_add_value_does_not_exist_is_created(): ) -@mock_dynamodb2 +@mock_dynamodb def test_update_item_with_expression(): table = _create_table_with_range_key() @@ -499,7 +499,7 @@ def assert_failure_due_to_key_not_in_schema(func, **kwargs): ) -@mock_dynamodb2 +@mock_dynamodb def test_update_item_add_with_expression(): table = _create_table_with_range_key() @@ -580,7 +580,7 @@ def test_update_item_add_with_expression(): ).should.have.raised(ClientError) -@mock_dynamodb2 +@mock_dynamodb def test_update_item_add_with_nested_sets(): table = _create_table_with_range_key() @@ -617,7 +617,7 @@ def test_update_item_add_with_nested_sets(): assert dict(table.get_item(Key=item_key)["Item"]) == current_item -@mock_dynamodb2 +@mock_dynamodb def test_update_item_delete_with_nested_sets(): table = _create_table_with_range_key() @@ -643,7 +643,7 @@ def test_update_item_delete_with_nested_sets(): dict(table.get_item(Key=item_key)["Item"]).should.equal(current_item) -@mock_dynamodb2 +@mock_dynamodb def test_update_item_delete_with_expression(): table = _create_table_with_range_key() @@ -699,7 +699,7 @@ def test_update_item_delete_with_expression(): ).should.have.raised(ClientError) -@mock_dynamodb2 +@mock_dynamodb def test_boto3_query_gsi_range_comparison(): table = _create_table_with_range_key() @@ -795,7 +795,7 @@ def test_boto3_query_gsi_range_comparison(): item["created"].should.equal(expected[index]) -@mock_dynamodb2 +@mock_dynamodb def test_boto3_update_table_throughput(): dynamodb = boto3.resource("dynamodb", region_name="us-east-1") @@ -827,7 +827,7 @@ def test_boto3_update_table_throughput(): table.provisioned_throughput["WriteCapacityUnits"].should.equal(11) -@mock_dynamodb2 +@mock_dynamodb def test_boto3_update_table_gsi_throughput(): dynamodb = boto3.resource("dynamodb", region_name="us-east-1") @@ -894,7 +894,7 @@ def test_boto3_update_table_gsi_throughput(): gsi_throughput["WriteCapacityUnits"].should.equal(11) -@mock_dynamodb2 +@mock_dynamodb def test_update_table_gsi_create(): dynamodb = boto3.resource("dynamodb", region_name="us-east-1") @@ -976,7 +976,7 @@ def test_update_table_gsi_create(): table.global_secondary_indexes.should.have.length_of(0) -@mock_dynamodb2 +@mock_dynamodb def test_update_table_gsi_throughput(): dynamodb = boto3.resource("dynamodb", region_name="us-east-1") @@ -1018,7 +1018,7 @@ def test_update_table_gsi_throughput(): table.global_secondary_indexes.should.have.length_of(0) -@mock_dynamodb2 +@mock_dynamodb def test_query_pagination(): table = _create_table_with_range_key() for i in range(10): @@ -1050,7 +1050,7 @@ def test_query_pagination(): subjects.should.equal(set(range(10))) -@mock_dynamodb2 +@mock_dynamodb def test_scan_by_index(): dynamodb = boto3.client("dynamodb", region_name="us-east-1") @@ -1152,7 +1152,7 @@ def test_scan_by_index(): assert last_eval_key["lsi_range_key"]["S"] == "1" -@mock_dynamodb2 +@mock_dynamodb @pytest.mark.parametrize("create_item_first", [False, True]) @pytest.mark.parametrize( "expression", ["set h=:New", "set r=:New", "set x=:New, r=:New"] diff --git a/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py b/tests/test_dynamodb/test_dynamodb_table_without_range_key.py similarity index 97% rename from tests/test_dynamodb2/test_dynamodb_table_without_range_key.py rename to tests/test_dynamodb/test_dynamodb_table_without_range_key.py index 264e256450b0..c7e635634b16 100644 --- a/tests/test_dynamodb2/test_dynamodb_table_without_range_key.py +++ b/tests/test_dynamodb/test_dynamodb_table_without_range_key.py @@ -4,12 +4,12 @@ import pytest from datetime import datetime from botocore.exceptions import ClientError -from moto import mock_dynamodb2 +from moto import mock_dynamodb from moto.core import ACCOUNT_ID import botocore -@mock_dynamodb2 +@mock_dynamodb def test_create_table_boto3(): client = boto3.client("dynamodb", region_name="us-east-1") client.create_table( @@ -72,7 +72,7 @@ def test_create_table_boto3(): actual.should.have.key("ItemCount").equal(0) -@mock_dynamodb2 +@mock_dynamodb def test_delete_table_boto3(): conn = boto3.client("dynamodb", region_name="us-west-2") conn.create_table( @@ -94,7 +94,7 @@ def test_delete_table_boto3(): ex.value.response["Error"]["Message"].should.equal("Requested resource not found") -@mock_dynamodb2 +@mock_dynamodb def test_item_add_and_describe_and_update_boto3(): conn = boto3.resource("dynamodb", region_name="us-west-2") table = conn.create_table( @@ -130,7 +130,7 @@ def test_item_add_and_describe_and_update_boto3(): ) -@mock_dynamodb2 +@mock_dynamodb def test_item_put_without_table_boto3(): conn = boto3.client("dynamodb", region_name="us-west-2") @@ -149,7 +149,7 @@ def test_item_put_without_table_boto3(): ex.value.response["Error"]["Message"].should.equal("Requested resource not found") -@mock_dynamodb2 +@mock_dynamodb def test_get_item_with_undeclared_table_boto3(): conn = boto3.client("dynamodb", region_name="us-west-2") @@ -161,7 +161,7 @@ def test_get_item_with_undeclared_table_boto3(): ex.value.response["Error"]["Message"].should.equal("Requested resource not found") -@mock_dynamodb2 +@mock_dynamodb def test_delete_item_boto3(): conn = boto3.resource("dynamodb", region_name="us-west-2") table = conn.create_table( @@ -188,7 +188,7 @@ def test_delete_item_boto3(): table.delete_item(Key={"id": "LOLCat Forum"}) -@mock_dynamodb2 +@mock_dynamodb def test_delete_item_with_undeclared_table_boto3(): conn = boto3.client("dynamodb", region_name="us-west-2") @@ -204,7 +204,7 @@ def test_delete_item_with_undeclared_table_boto3(): ) -@mock_dynamodb2 +@mock_dynamodb def test_scan_with_undeclared_table_boto3(): conn = boto3.client("dynamodb", region_name="us-west-2") @@ -216,7 +216,7 @@ def test_scan_with_undeclared_table_boto3(): ex.value.response["Error"]["Message"].should.equal("Requested resource not found") -@mock_dynamodb2 +@mock_dynamodb def test_get_key_schema(): conn = boto3.resource("dynamodb", region_name="us-west-2") table = conn.create_table( @@ -229,7 +229,7 @@ def test_get_key_schema(): table.key_schema.should.equal([{"AttributeName": "id", "KeyType": "HASH"}]) -@mock_dynamodb2 +@mock_dynamodb def test_update_item_double_nested_remove(): conn = boto3.client("dynamodb", region_name="us-east-1") conn.create_table( @@ -264,7 +264,7 @@ def test_update_item_double_nested_remove(): dict(returned_item["Item"]).should.equal(expected_item) -@mock_dynamodb2 +@mock_dynamodb def test_update_item_set_boto3(): conn = boto3.resource("dynamodb", region_name="us-east-1") table = conn.create_table( @@ -288,7 +288,7 @@ def test_update_item_set_boto3(): dict(returned_item).should.equal({"username": "steve", "foo": "bar", "blah": "baz"}) -@mock_dynamodb2 +@mock_dynamodb def test_boto3_create_table(): dynamodb = boto3.resource("dynamodb", region_name="us-east-1") @@ -313,7 +313,7 @@ def _create_user_table(): return dynamodb.Table("users") -@mock_dynamodb2 +@mock_dynamodb def test_boto3_conditions(): table = _create_user_table() @@ -326,7 +326,7 @@ def test_boto3_conditions(): response["Items"][0].should.equal({"username": "johndoe"}) -@mock_dynamodb2 +@mock_dynamodb def test_boto3_put_item_conditions_pass(): table = _create_user_table() table.put_item(Item={"username": "johndoe", "foo": "bar"}) @@ -338,7 +338,7 @@ def test_boto3_put_item_conditions_pass(): assert dict(final_item)["Item"]["foo"].should.equal("baz") -@mock_dynamodb2 +@mock_dynamodb def test_boto3_put_item_conditions_pass_because_expect_not_exists_by_compare_to_null(): table = _create_user_table() table.put_item(Item={"username": "johndoe", "foo": "bar"}) @@ -350,7 +350,7 @@ def test_boto3_put_item_conditions_pass_because_expect_not_exists_by_compare_to_ assert dict(final_item)["Item"]["foo"].should.equal("baz") -@mock_dynamodb2 +@mock_dynamodb def test_boto3_put_item_conditions_pass_because_expect_exists_by_compare_to_not_null(): table = _create_user_table() table.put_item(Item={"username": "johndoe", "foo": "bar"}) @@ -362,7 +362,7 @@ def test_boto3_put_item_conditions_pass_because_expect_exists_by_compare_to_not_ assert dict(final_item)["Item"]["foo"].should.equal("baz") -@mock_dynamodb2 +@mock_dynamodb def test_boto3_put_item_conditions_fail(): table = _create_user_table() table.put_item(Item={"username": "johndoe", "foo": "bar"}) @@ -372,7 +372,7 @@ def test_boto3_put_item_conditions_fail(): ).should.throw(botocore.client.ClientError) -@mock_dynamodb2 +@mock_dynamodb def test_boto3_update_item_conditions_fail(): table = _create_user_table() table.put_item(Item={"username": "johndoe", "foo": "baz"}) @@ -384,7 +384,7 @@ def test_boto3_update_item_conditions_fail(): ).should.throw(botocore.client.ClientError) -@mock_dynamodb2 +@mock_dynamodb def test_boto3_update_item_conditions_fail_because_expect_not_exists(): table = _create_user_table() table.put_item(Item={"username": "johndoe", "foo": "baz"}) @@ -396,7 +396,7 @@ def test_boto3_update_item_conditions_fail_because_expect_not_exists(): ).should.throw(botocore.client.ClientError) -@mock_dynamodb2 +@mock_dynamodb def test_boto3_update_item_conditions_fail_because_expect_not_exists_by_compare_to_null(): table = _create_user_table() table.put_item(Item={"username": "johndoe", "foo": "baz"}) @@ -408,7 +408,7 @@ def test_boto3_update_item_conditions_fail_because_expect_not_exists_by_compare_ ).should.throw(botocore.client.ClientError) -@mock_dynamodb2 +@mock_dynamodb def test_boto3_update_item_conditions_pass(): table = _create_user_table() table.put_item(Item={"username": "johndoe", "foo": "bar"}) @@ -422,7 +422,7 @@ def test_boto3_update_item_conditions_pass(): assert dict(returned_item)["Item"]["foo"].should.equal("baz") -@mock_dynamodb2 +@mock_dynamodb def test_boto3_update_item_conditions_pass_because_expect_not_exists(): table = _create_user_table() table.put_item(Item={"username": "johndoe", "foo": "bar"}) @@ -436,7 +436,7 @@ def test_boto3_update_item_conditions_pass_because_expect_not_exists(): assert dict(returned_item)["Item"]["foo"].should.equal("baz") -@mock_dynamodb2 +@mock_dynamodb def test_boto3_update_item_conditions_pass_because_expect_not_exists_by_compare_to_null(): table = _create_user_table() table.put_item(Item={"username": "johndoe", "foo": "bar"}) @@ -450,7 +450,7 @@ def test_boto3_update_item_conditions_pass_because_expect_not_exists_by_compare_ assert dict(returned_item)["Item"]["foo"].should.equal("baz") -@mock_dynamodb2 +@mock_dynamodb def test_boto3_update_item_conditions_pass_because_expect_exists_by_compare_to_not_null(): table = _create_user_table() table.put_item(Item={"username": "johndoe", "foo": "bar"}) @@ -464,7 +464,7 @@ def test_boto3_update_item_conditions_pass_because_expect_exists_by_compare_to_n assert dict(returned_item)["Item"]["foo"].should.equal("baz") -@mock_dynamodb2 +@mock_dynamodb def test_boto3_update_settype_item_with_conditions(): class OrderedSet(set): """A set with predictable iteration order""" @@ -501,7 +501,7 @@ def __iter__(self): assert dict(returned_item)["Item"]["foo"].should.equal(set(["baz"])) -@mock_dynamodb2 +@mock_dynamodb def test_scan_pagination(): table = _create_user_table() @@ -524,7 +524,7 @@ def test_scan_pagination(): usernames.should.equal(set(expected_usernames)) -@mock_dynamodb2 +@mock_dynamodb def test_scan_by_index(): dynamodb = boto3.client("dynamodb", region_name="us-east-1") diff --git a/tests/test_dynamodb2/test_dynamodb_update_table.py b/tests/test_dynamodb/test_dynamodb_update_table.py similarity index 97% rename from tests/test_dynamodb2/test_dynamodb_update_table.py rename to tests/test_dynamodb/test_dynamodb_update_table.py index dbae7b552b22..52af798d90d9 100644 --- a/tests/test_dynamodb2/test_dynamodb_update_table.py +++ b/tests/test_dynamodb/test_dynamodb_update_table.py @@ -1,10 +1,10 @@ import boto3 import sure # noqa # pylint: disable=unused-import -from moto import mock_dynamodb2 +from moto import mock_dynamodb -@mock_dynamodb2 +@mock_dynamodb def test_update_table__billing_mode(): client = boto3.client("dynamodb", region_name="us-east-1") client.create_table( @@ -33,7 +33,7 @@ def test_update_table__billing_mode(): ) -@mock_dynamodb2 +@mock_dynamodb def test_update_table_throughput(): conn = boto3.resource("dynamodb", region_name="us-west-2") table = conn.create_table( @@ -53,7 +53,7 @@ def test_update_table_throughput(): table.provisioned_throughput["WriteCapacityUnits"].should.equal(6) -@mock_dynamodb2 +@mock_dynamodb def test_update_table__enable_stream(): conn = boto3.client("dynamodb", region_name="us-east-1") diff --git a/tests/test_dynamodb2/test_dynamodb_validation.py b/tests/test_dynamodb/test_dynamodb_validation.py similarity index 98% rename from tests/test_dynamodb2/test_dynamodb_validation.py rename to tests/test_dynamodb/test_dynamodb_validation.py index 3420c37d648a..b3b0b6a915a5 100644 --- a/tests/test_dynamodb2/test_dynamodb_validation.py +++ b/tests/test_dynamodb/test_dynamodb_validation.py @@ -1,6 +1,6 @@ import pytest -from moto.dynamodb2.exceptions import ( +from moto.dynamodb.exceptions import ( AttributeIsReservedKeyword, ExpressionAttributeValueNotDefined, AttributeDoesNotExist, @@ -8,14 +8,14 @@ IncorrectOperandType, InvalidUpdateExpressionInvalidDocumentPath, ) -from moto.dynamodb2.models import Item, DynamoType -from moto.dynamodb2.parsing.ast_nodes import ( +from moto.dynamodb.models import Item, DynamoType +from moto.dynamodb.parsing.ast_nodes import ( NodeDepthLeftTypeFetcher, UpdateExpressionSetAction, DDBTypedValue, ) -from moto.dynamodb2.parsing.expressions import UpdateExpressionParser -from moto.dynamodb2.parsing.validators import UpdateExpressionValidator +from moto.dynamodb.parsing.expressions import UpdateExpressionParser +from moto.dynamodb.parsing.validators import UpdateExpressionValidator def test_valid_update_expression(table): diff --git a/tests/test_dynamodb/test_server.py b/tests/test_dynamodb/test_server.py index a0b5f1654c05..9afa08076f0f 100644 --- a/tests/test_dynamodb/test_server.py +++ b/tests/test_dynamodb/test_server.py @@ -1,1342 +1,19 @@ -import json import sure # noqa # pylint: disable=unused-import -import pytest -from moto import mock_dynamodb import moto.server as server """ Test the different server responses -Docs: -https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Appendix.APIv20111205.html """ -TABLE_NAME = "my_table_name" -TABLE_WITH_RANGE_NAME = "my_table_with_range_name" - - -@pytest.fixture(autouse=True) -def test_client(): - with mock_dynamodb(): - backend = server.create_backend_app("dynamodb") - test_client = backend.test_client() - - yield test_client - - -def test_404(test_client): +def test_table_list(): + backend = server.create_backend_app("dynamodb") + test_client = backend.test_client() res = test_client.get("/") res.status_code.should.equal(404) - -def test_table_list(test_client): headers = {"X-Amz-Target": "TestTable.ListTables"} res = test_client.get("/", headers=headers) - json.loads(res.data).should.equal({"TableNames": []}) - - -def test_create_table(test_client): - res = create_table(test_client) - res = json.loads(res.data)["Table"] - res.should.have.key("CreationDateTime") - del res["CreationDateTime"] - res.should.equal( - { - "KeySchema": { - "HashKeyElement": {"AttributeName": "hkey", "AttributeType": "S"}, - "RangeKeyElement": {"AttributeName": "rkey", "AttributeType": "N"}, - }, - "ProvisionedThroughput": {"ReadCapacityUnits": 5, "WriteCapacityUnits": 10}, - "TableName": TABLE_WITH_RANGE_NAME, - "TableStatus": "ACTIVE", - "ItemCount": 0, - "TableSizeBytes": 0, - } - ) - - headers = {"X-Amz-Target": "TestTable.ListTables"} - res = test_client.get("/", headers=headers) - res = json.loads(res.data) - res.should.equal({"TableNames": [TABLE_WITH_RANGE_NAME]}) - - -def test_create_table_without_range_key(test_client): - res = create_table(test_client, use_range_key=False) - res = json.loads(res.data)["Table"] - res.should.have.key("CreationDateTime") - del res["CreationDateTime"] - res.should.equal( - { - "KeySchema": { - "HashKeyElement": {"AttributeName": "hkey", "AttributeType": "S"} - }, - "ProvisionedThroughput": {"ReadCapacityUnits": 5, "WriteCapacityUnits": 10}, - "TableName": TABLE_NAME, - "TableStatus": "ACTIVE", - "ItemCount": 0, - "TableSizeBytes": 0, - } - ) - - headers = {"X-Amz-Target": "TestTable.ListTables"} - res = test_client.get("/", headers=headers) - res = json.loads(res.data) - res.should.equal({"TableNames": [TABLE_NAME]}) - - -# This test is pointless, as we treat DynamoDB as a global resource -def test_create_table_in_different_regions(test_client): - create_table(test_client) - create_table(test_client, name="Table2", region="us-west-2") - - headers = {"X-Amz-Target": "TestTable.ListTables"} - res = test_client.get("/", headers=headers) - res = json.loads(res.data) - res.should.equal({"TableNames": [TABLE_WITH_RANGE_NAME, "Table2"]}) - - -def test_update_item(): - backend = server.create_backend_app("dynamodb") - test_client = backend.test_client() - - create_table(test_client) - - headers, res = put_item(test_client) - - # UpdateItem - headers["X-Amz-Target"] = "DynamoDB_20111205.UpdateItem" - request_body = { - "TableName": TABLE_WITH_RANGE_NAME, - "Key": { - "HashKeyElement": {"S": "customer"}, - "RangeKeyElement": {"N": "12341234"}, - }, - "AttributeUpdates": {"new_att": {"Value": {"SS": ["val"]}, "Action": "PUT"}}, - } - res = test_client.post("/", headers=headers, json=request_body) - - # UpdateItem - headers["X-Amz-Target"] = "DynamoDB_20111205.UpdateItem" - request_body = { - "TableName": TABLE_WITH_RANGE_NAME, - "Key": { - "HashKeyElement": {"S": "customer"}, - "RangeKeyElement": {"N": "12341234"}, - }, - "AttributeUpdates": {"new_n": {"Value": {"N": "42"}, "Action": "PUT"}}, - } - res = test_client.post("/", headers=headers, json=request_body) - res = json.loads(res.data) - - res["ConsumedCapacityUnits"].should.equal(0.5) - res["Attributes"].should.equal( - { - "hkey": "customer", - "name": "myname", - "rkey": "12341234", - "new_att": ["val"], - "new_n": "42", - } - ) - - # UpdateItem - multiples - headers["X-Amz-Target"] = "DynamoDB_20111205.UpdateItem" - request_body = { - "TableName": TABLE_WITH_RANGE_NAME, - "Key": { - "HashKeyElement": {"S": "customer"}, - "RangeKeyElement": {"N": "12341234"}, - }, - "AttributeUpdates": { - "new_n": {"Value": {"N": 7}, "Action": "ADD"}, - "new_att": {"Value": {"S": "val2"}, "Action": "ADD"}, - "name": {"Action": "DELETE"}, - }, - } - res = test_client.post("/", headers=headers, json=request_body) - res = json.loads(res.data) - - res["ConsumedCapacityUnits"].should.equal(0.5) - res["Attributes"].should.equal( - { - "hkey": "customer", - "rkey": "12341234", - "new_att": ["val", "val2"], - "new_n": "49", - } - ) - - # GetItem - headers["X-Amz-Target"] = "DynamoDB_20111205.GetItem" - request_body = { - "TableName": TABLE_WITH_RANGE_NAME, - "Key": { - "HashKeyElement": {"S": "customer"}, - "RangeKeyElement": {"N": "12341234"}, - }, - } - res = test_client.post("/", headers=headers, json=request_body) - res = json.loads(res.data) - res["Item"].should.have.key("new_att").equal({"SS": ["val", "val2"]}) - res["Item"].should.have.key("new_n").equal({"N": "49"}) - res["Item"].shouldnt.have.key("name") - - -@pytest.mark.parametrize( - "use_range_key", [True, False], ids=["using range key", "using hash key only"] -) -def test_delete_table(use_range_key, test_client): - create_table(test_client, use_range_key=use_range_key) - - headers = {"X-Amz-Target": "DynamoDB_20111205.DeleteTable"} - name = TABLE_WITH_RANGE_NAME if use_range_key else TABLE_NAME - test_client.post("/", headers=headers, json={"TableName": name}) - - headers = {"X-Amz-Target": "DynamoDB_20111205.ListTables"} - res = test_client.post("/", headers=headers) - res = json.loads(res.data) - res.should.equal({"TableNames": []}) - - -def test_delete_unknown_table(test_client): - headers = {"X-Amz-Target": "DynamoDB_20111205.DeleteTable"} - res = test_client.post("/", headers=headers, json={"TableName": "unknown_table"}) - res.status_code.should.equal(400) - - json.loads(res.data).should.equal( - {"__type": "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException"} - ) - - -def test_describe_table(test_client): - create_table(test_client) - - headers = { - "X-Amz-Target": "DynamoDB_20111205.DescribeTable", - "Content-Type": "application/x-amz-json-1.0", - } - res = test_client.post( - "/", headers=headers, json={"TableName": TABLE_WITH_RANGE_NAME} - ) - res = json.loads(res.data)["Table"] - res.should.have.key("CreationDateTime") - del res["CreationDateTime"] - res.should.equal( - { - "KeySchema": { - "HashKeyElement": {"AttributeName": "hkey", "AttributeType": "S"}, - "RangeKeyElement": {"AttributeName": "rkey", "AttributeType": "N"}, - }, - "ProvisionedThroughput": {"ReadCapacityUnits": 5, "WriteCapacityUnits": 10}, - "TableName": TABLE_WITH_RANGE_NAME, - "TableStatus": "ACTIVE", - "ItemCount": 0, - "TableSizeBytes": 0, - } - ) - - -def test_describe_missing_table(test_client): - headers = { - "X-Amz-Target": "DynamoDB_20111205.DescribeTable", - "Content-Type": "application/x-amz-json-1.0", - } - res = test_client.post("/", headers=headers, json={"TableName": "unknown_table"}) - res.status_code.should.equal(400) - json.loads(res.data).should.equal( - {"__type": "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException"} - ) - - -@pytest.mark.parametrize( - "use_range_key", [True, False], ids=["using range key", "using hash key only"] -) -def test_update_table(test_client, use_range_key): - table_name = TABLE_WITH_RANGE_NAME if use_range_key else TABLE_NAME - create_table(test_client, use_range_key=use_range_key) - - headers = { - "X-Amz-Target": "DynamoDB_20111205.UpdateTable", - "Content-Type": "application/x-amz-json-1.0", - } - request_data = { - "TableName": table_name, - "ProvisionedThroughput": {"ReadCapacityUnits": 5, "WriteCapacityUnits": 15}, - } - test_client.post("/", headers=headers, json=request_data) - - # DescribeTable - verify the throughput is persisted - headers = { - "X-Amz-Target": "DynamoDB_20111205.DescribeTable", - "Content-Type": "application/x-amz-json-1.0", - } - res = test_client.post("/", headers=headers, json={"TableName": table_name}) - throughput = json.loads(res.data)["Table"]["ProvisionedThroughput"] - - throughput.should.equal({"ReadCapacityUnits": 5, "WriteCapacityUnits": 15}) - - -def test_put_return_none(test_client): - create_table(test_client) - - headers = { - "X-Amz-Target": "DynamoDB_20111205.PutItem", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": TABLE_WITH_RANGE_NAME, - "Item": { - "hkey": {"S": "customer"}, - "rkey": {"N": "12341234"}, - "name": {"S": "myname"}, - }, - "ReturnValues": "NONE", - } - res = test_client.post("/", headers=headers, json=request_body) - res = json.loads(res.data) - # This seems wrong - it should return nothing, considering return_values is set to none - res["Attributes"].should.equal( - {"hkey": "customer", "name": "myname", "rkey": "12341234"} - ) - - -def test_put_return_none_without_range_key(test_client): - create_table(test_client, use_range_key=False) - - headers = { - "X-Amz-Target": "DynamoDB_20111205.PutItem", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": TABLE_NAME, - "Item": {"hkey": {"S": "customer"}, "name": {"S": "myname"}}, - "ReturnValues": "NONE", - } - res = test_client.post("/", headers=headers, json=request_body) - res = json.loads(res.data) - print(res) - # This seems wrong - it should return nothing, considering return_values is set to none - res["Attributes"].should.equal({"hkey": "customer", "name": "myname"}) - - -def test_put_item_from_unknown_table(test_client): - headers = { - "X-Amz-Target": "DynamoDB_20111205.PutItem", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": "unknown_table", - "Item": { - "hkey": {"S": "customer"}, - "rkey": {"N": "12341234"}, - "name": {"S": "myname"}, - }, - "ReturnValues": "NONE", - } - res = test_client.post("/", headers=headers, json=request_body) - - res.status_code.should.equal(400) - json.loads(res.data).should.equal( - {"__type": "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException"} - ) - - -def test_get_item_from_unknown_table(test_client): - headers = { - "X-Amz-Target": "DynamoDB_20111205.GetItem", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": "unknown_table", - "Key": { - "HashKeyElement": {"S": "customer"}, - "RangeKeyElement": {"N": "12341234"}, - }, - } - res = test_client.post("/", headers=headers, json=request_body) - - res.status_code.should.equal(404) - json.loads(res.data).should.equal( - {"__type": "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException"} - ) - - -@pytest.mark.parametrize( - "use_range_key", [True, False], ids=["using range key", "using hash key only"] -) -def test_get_unknown_item_from_table(use_range_key, test_client): - create_table(test_client, use_range_key=use_range_key) - - headers = { - "X-Amz-Target": "DynamoDB_20111205.GetItem", - "Content-Type": "application/x-amz-json-1.0", - } - table_name = TABLE_WITH_RANGE_NAME if use_range_key else TABLE_NAME - request_body = { - "TableName": table_name, - "Key": {"HashKeyElement": {"S": "customer"}}, - } - if use_range_key: - request_body["Key"]["RangeKeyElement"] = {"N": "12341234"} - res = test_client.post("/", headers=headers, json=request_body) - - res.status_code.should.equal(404) - json.loads(res.data).should.equal( - {"__type": "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException"} - ) - - -def test_get_item_without_range_key(test_client): - create_table(test_client) - - headers = { - "X-Amz-Target": "DynamoDB_20111205.GetItem", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": TABLE_WITH_RANGE_NAME, - "Key": {"HashKeyElement": {"S": "customer"}}, - } - res = test_client.post("/", headers=headers, json=request_body) - - res.status_code.should.equal(400) - json.loads(res.data).should.equal( - {"__type": "com.amazon.coral.validate#ValidationException"} - ) - - -def test_put_and_get_item(test_client): - create_table(test_client) - - headers, res = put_item(test_client) - - res["ConsumedCapacityUnits"].should.equal(1) - res["Attributes"].should.equal( - {"hkey": "customer", "name": "myname", "rkey": "12341234"} - ) - - # GetItem - headers["X-Amz-Target"] = "DynamoDB_20111205.GetItem" - request_body = { - "TableName": TABLE_WITH_RANGE_NAME, - "Key": { - "HashKeyElement": {"S": "customer"}, - "RangeKeyElement": {"N": "12341234"}, - }, - } - res = test_client.post("/", headers=headers, json=request_body) - res = json.loads(res.data) - res["ConsumedCapacityUnits"].should.equal(0.5) - res["Item"].should.equal( - {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "12341234"}} - ) - - # GetItem - return single attribute - headers["X-Amz-Target"] = "DynamoDB_20111205.GetItem" - request_body = { - "TableName": TABLE_WITH_RANGE_NAME, - "Key": { - "HashKeyElement": {"S": "customer"}, - "RangeKeyElement": {"N": "12341234"}, - }, - "AttributesToGet": ["name"], - } - res = test_client.post("/", headers=headers, json=request_body) - res = json.loads(res.data) - res["ConsumedCapacityUnits"].should.equal(0.5) - res["Item"].should.equal({"name": {"S": "myname"}}) - - -def test_put_and_get_item_without_range_key(test_client): - create_table(test_client, use_range_key=False) - - headers, res = put_item(test_client, use_range_key=False) - - res["ConsumedCapacityUnits"].should.equal(1) - res["Attributes"].should.equal({"hkey": "customer", "name": "myname"}) - - # GetItem - headers["X-Amz-Target"] = "DynamoDB_20111205.GetItem" - request_body = { - "TableName": TABLE_NAME, - "Key": {"HashKeyElement": {"S": "customer"}}, - } - res = test_client.post("/", headers=headers, json=request_body) - res = json.loads(res.data) - res["ConsumedCapacityUnits"].should.equal(0.5) - res["Item"].should.equal({"hkey": {"S": "customer"}, "name": {"S": "myname"}}) - - # GetItem - return single attribute - headers["X-Amz-Target"] = "DynamoDB_20111205.GetItem" - request_body = { - "TableName": TABLE_NAME, - "Key": {"HashKeyElement": {"S": "customer"}}, - "AttributesToGet": ["name"], - } - res = test_client.post("/", headers=headers, json=request_body) - res = json.loads(res.data) - res["ConsumedCapacityUnits"].should.equal(0.5) - res["Item"].should.equal({"name": {"S": "myname"}}) - - -def test_scan_simple(test_client): - create_table(test_client) - - put_item(test_client) - put_item(test_client, rkey="12341235") - put_item(test_client, rkey="12341236") - - headers = { - "X-Amz-Target": "DynamoDB_20111205.Scan", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = {"TableName": TABLE_WITH_RANGE_NAME} - res = test_client.post("/", headers=headers, json=request_body) - res = json.loads(res.data) - - res.should.have.key("Count").equal(3) - res.should.have.key("ScannedCount").equal(3) - res.should.have.key("ConsumedCapacityUnits").equal(1) - res.should.have.key("Items").length_of(3) - - items = res["Items"] - items.should.contain( - {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "12341234"}} - ) - items.should.contain( - {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "12341235"}} - ) - items.should.contain( - {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "12341236"}} - ) - - -def test_scan_with_filter(test_client): - create_table(test_client) - - put_item(test_client, rkey="1230", name="somename") - put_item(test_client, rkey="1234", name=None) - put_item(test_client, rkey="1246") - - # SCAN specific item - headers = { - "X-Amz-Target": "DynamoDB_20111205.Scan", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": TABLE_WITH_RANGE_NAME, - "ScanFilter": { - "rkey": {"AttributeValueList": [{"S": "1234"}], "ComparisonOperator": "EQ"} - }, - } - res = test_client.post("/", headers=headers, json=request_body) - res = json.loads(res.data) - - res.should.have.key("Count").equal(1) - res.should.have.key("ScannedCount").equal(3) - res.should.have.key("ConsumedCapacityUnits").equal(1) - res.should.have.key("Items").length_of(1) - - items = res["Items"] - items.should.contain({"hkey": {"S": "customer"}, "rkey": {"N": "1234"}}) - - # SCAN begins_with - headers = { - "X-Amz-Target": "DynamoDB_20111205.Scan", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": TABLE_WITH_RANGE_NAME, - "ScanFilter": { - "rkey": { - "AttributeValueList": [{"S": "124"}], - "ComparisonOperator": "BEGINS_WITH", - } - }, - } - res = test_client.post("/", headers=headers, json=request_body) - items = json.loads(res.data)["Items"] - - items.should.contain( - {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "1246"}} - ) - - # SCAN contains - headers = { - "X-Amz-Target": "DynamoDB_20111205.Scan", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": TABLE_WITH_RANGE_NAME, - "ScanFilter": { - "name": { - "AttributeValueList": [{"S": "mena"}], - "ComparisonOperator": "CONTAINS", - } - }, - } - res = test_client.post("/", headers=headers, json=request_body) - items = json.loads(res.data)["Items"] - - items.should.contain( - {"hkey": {"S": "customer"}, "name": {"S": "somename"}, "rkey": {"N": "1230"}} - ) - - # SCAN null - headers = { - "X-Amz-Target": "DynamoDB_20111205.Scan", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": TABLE_WITH_RANGE_NAME, - "ScanFilter": {"name": {"ComparisonOperator": "NULL"}}, - } - res = test_client.post("/", headers=headers, json=request_body) - items = json.loads(res.data)["Items"] - - items.should.contain({"hkey": {"S": "customer"}, "rkey": {"N": "1234"}}) - - # SCAN NOT NULL - headers = { - "X-Amz-Target": "DynamoDB_20111205.Scan", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": TABLE_WITH_RANGE_NAME, - "ScanFilter": {"name": {"ComparisonOperator": "NOT_NULL"}}, - } - res = test_client.post("/", headers=headers, json=request_body) - items = json.loads(res.data)["Items"] - - items.should.equal( - [ - { - "hkey": {"S": "customer"}, - "name": {"S": "somename"}, - "rkey": {"N": "1230"}, - }, - {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "1246"}}, - ] - ) - - # SCAN between - headers = { - "X-Amz-Target": "DynamoDB_20111205.Scan", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": TABLE_WITH_RANGE_NAME, - "ScanFilter": { - "rkey": { - "AttributeValueList": [{"S": "1230"}, {"S": "1240"}], - "ComparisonOperator": "BETWEEN", - } - }, - } - res = test_client.post("/", headers=headers, json=request_body) - items = json.loads(res.data)["Items"] - - items.should.contain( - {"hkey": {"S": "customer"}, "name": {"S": "somename"}, "rkey": {"N": "1230"}} - ) - - -def test_scan_with_filter_in_table_without_range_key(test_client): - create_table(test_client, use_range_key=False) - - put_item(test_client, use_range_key=False, hkey="customer1", name=None) - put_item(test_client, use_range_key=False, hkey="customer2") - put_item(test_client, use_range_key=False, hkey="customer3", name="special") - - # SCAN specific item - headers = { - "X-Amz-Target": "DynamoDB_20111205.Scan", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": TABLE_NAME, - "ScanFilter": { - "name": { - "AttributeValueList": [{"S": "special"}], - "ComparisonOperator": "EQ", - } - }, - } - res = test_client.post("/", headers=headers, json=request_body) - res = json.loads(res.data) - - res.should.have.key("Count").equal(1) - res.should.have.key("ScannedCount").equal(3) - res.should.have.key("ConsumedCapacityUnits").equal(1) - res.should.have.key("Items").length_of(1) - - items = res["Items"] - items.should.contain({"hkey": {"S": "customer3"}, "name": {"S": "special"}}) - - # SCAN begins_with - headers = { - "X-Amz-Target": "DynamoDB_20111205.Scan", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": TABLE_NAME, - "ScanFilter": { - "hkey": { - "AttributeValueList": [{"S": "cust"}], - "ComparisonOperator": "BEGINS_WITH", - } - }, - } - res = test_client.post("/", headers=headers, json=request_body) - items = json.loads(res.data)["Items"] - - items.should.have.length_of(3) # all customers start with cust - - # SCAN contains - headers = { - "X-Amz-Target": "DynamoDB_20111205.Scan", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": TABLE_NAME, - "ScanFilter": { - "name": { - "AttributeValueList": [{"S": "yna"}], - "ComparisonOperator": "CONTAINS", - } - }, - } - res = test_client.post("/", headers=headers, json=request_body) - items = json.loads(res.data)["Items"] - - items.should.have.equal([{"hkey": {"S": "customer2"}, "name": {"S": "myname"}}]) - - # SCAN null - headers = { - "X-Amz-Target": "DynamoDB_20111205.Scan", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": TABLE_NAME, - "ScanFilter": {"name": {"ComparisonOperator": "NULL"}}, - } - res = test_client.post("/", headers=headers, json=request_body) - items = json.loads(res.data)["Items"] - - items.should.equal([{"hkey": {"S": "customer1"}}]) - - # SCAN NOT NULL - headers = { - "X-Amz-Target": "DynamoDB_20111205.Scan", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": TABLE_NAME, - "ScanFilter": {"name": {"ComparisonOperator": "NOT_NULL"}}, - } - res = test_client.post("/", headers=headers, json=request_body) - items = json.loads(res.data)["Items"] - - items.should.have.length_of(2) - items.should.contain({"hkey": {"S": "customer2"}, "name": {"S": "myname"}}) - items.should.contain({"hkey": {"S": "customer3"}, "name": {"S": "special"}}) - - -def test_scan_with_undeclared_table(test_client): - headers = { - "X-Amz-Target": "DynamoDB_20111205.Scan", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = {"TableName": "unknown_table"} - res = test_client.post("/", headers=headers, json=request_body) - res.status_code.should.equal(400) - json.loads(res.data).should.equal( - {"__type": "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException"} - ) - - -def test_query_in_table_without_range_key(test_client): - create_table(test_client, use_range_key=False) - - put_item(test_client, use_range_key=False) - - headers = { - "X-Amz-Target": "DynamoDB_20111205.Query", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = {"TableName": TABLE_NAME, "HashKeyValue": {"S": "customer"}} - res = test_client.post("/", headers=headers, json=request_body) - res = json.loads(res.data) - - res.should.have.key("Count").equal(1) - res.should.have.key("ConsumedCapacityUnits").equal(1) - res.should.have.key("Items").length_of(1) - - items = res["Items"] - items.should.contain({"hkey": {"S": "customer"}, "name": {"S": "myname"}}) - - # QUERY for unknown value - headers = { - "X-Amz-Target": "DynamoDB_20111205.Query", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = {"TableName": TABLE_NAME, "HashKeyValue": {"S": "unknown-value"}} - res = test_client.post("/", headers=headers, json=request_body) - res = json.loads(res.data) - - # TODO: We should not get any results here - # res.should.have.key("Count").equal(0) - # res.should.have.key("Items").length_of(0) - - -def test_query_item_by_hash_only(test_client): - create_table(test_client) - - put_item(test_client) - put_item(test_client, rkey="12341235") - put_item(test_client, rkey="12341236") - - headers = { - "X-Amz-Target": "DynamoDB_20111205.Query", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": TABLE_WITH_RANGE_NAME, - "HashKeyValue": {"S": "customer"}, - } - res = test_client.post("/", headers=headers, json=request_body) - res = json.loads(res.data) - - res.should.have.key("Count").equal(3) - res.should.have.key("ConsumedCapacityUnits").equal(1) - res.should.have.key("Items").length_of(3) - - items = res["Items"] - items.should.contain( - {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "12341234"}} - ) - items.should.contain( - {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "12341235"}} - ) - items.should.contain( - {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "12341236"}} - ) - - -def test_query_item_by_range_key(test_client): - create_table(test_client, use_range_key=True) - - put_item(test_client, rkey="1234") - put_item(test_client, rkey="1235") - put_item(test_client, rkey="1247") - - # GT some - headers = { - "X-Amz-Target": "DynamoDB_20111205.Query", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": TABLE_WITH_RANGE_NAME, - "HashKeyValue": {"S": "customer"}, - "RangeKeyCondition": { - "AttributeValueList": [{"N": "1235"}], - "ComparisonOperator": "GT", - }, - } - res = test_client.post("/", headers=headers, json=request_body) - res = json.loads(res.data) - - res.should.have.key("Count").equal(1) - res.should.have.key("ConsumedCapacityUnits").equal(1) - res.should.have.key("Items").length_of(1) - - items = res["Items"] - items.should.contain( - {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "1247"}} - ) - - # GT all - headers = { - "X-Amz-Target": "DynamoDB_20111205.Query", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": TABLE_WITH_RANGE_NAME, - "HashKeyValue": {"S": "customer"}, - "RangeKeyCondition": { - "AttributeValueList": [{"N": "0"}], - "ComparisonOperator": "GT", - }, - } - res = test_client.post("/", headers=headers, json=request_body) - res = json.loads(res.data) - - res.should.have.key("Count").equal(3) - res.should.have.key("Items").length_of(3) - - # GT none - headers = { - "X-Amz-Target": "DynamoDB_20111205.Query", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": TABLE_WITH_RANGE_NAME, - "HashKeyValue": {"S": "customer"}, - "RangeKeyCondition": { - "AttributeValueList": [{"N": "9999"}], - "ComparisonOperator": "GT", - }, - } - res = test_client.post("/", headers=headers, json=request_body) - res = json.loads(res.data) - - res["ConsumedCapacityUnits"].should.equal(1) - res["Items"].should.equal([]) - res["Count"].should.equal(0) - - # CONTAINS some - headers = { - "X-Amz-Target": "DynamoDB_20111205.Query", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": TABLE_WITH_RANGE_NAME, - "HashKeyValue": {"S": "customer"}, - "RangeKeyCondition": { - "AttributeValueList": [{"N": "24"}], - "ComparisonOperator": "CONTAINS", - }, - } - res = test_client.post("/", headers=headers, json=request_body) - items = json.loads(res.data)["Items"] - - items.should.equal( - [{"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "1247"}}] - ) - - # BEGINS_WITH - headers = { - "X-Amz-Target": "DynamoDB_20111205.Query", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": TABLE_WITH_RANGE_NAME, - "HashKeyValue": {"S": "customer"}, - "RangeKeyCondition": { - "AttributeValueList": [{"N": "123"}], - "ComparisonOperator": "BEGINS_WITH", - }, - } - res = test_client.post("/", headers=headers, json=request_body) - items = json.loads(res.data)["Items"] - - items.should.equal( - [ - {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "1234"}}, - {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "1235"}}, - ] - ) - - # CONTAINS - headers = { - "X-Amz-Target": "DynamoDB_20111205.Query", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": TABLE_WITH_RANGE_NAME, - "HashKeyValue": {"S": "customer"}, - "RangeKeyCondition": { - "AttributeValueList": [{"N": "0"}, {"N": "1240"}], - "ComparisonOperator": "BETWEEN", - }, - } - res = test_client.post("/", headers=headers, json=request_body) - items = json.loads(res.data)["Items"] - - items.should.equal( - [ - {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "1234"}}, - {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "1235"}}, - ] - ) - - -def test_query_item_with_undeclared_table(test_client): - headers = { - "X-Amz-Target": "DynamoDB_20111205.Query", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": "unknown_table", - "HashKeyValue": {"S": "customer"}, - "RangeKeyCondition": { - "AttributeValueList": [{"N": "1235"}], - "ComparisonOperator": "GT", - }, - } - res = test_client.post("/", headers=headers, json=request_body) - res.status_code.should.equal(400) - json.loads(res.data).should.equal( - {"__type": "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException"} - ) - - -def test_delete_item(test_client): - create_table(test_client) - - put_item(test_client) - put_item(test_client, rkey="12341235") - put_item(test_client, rkey="12341236") - - headers = { - "X-Amz-Target": "DynamoDB_20111205.DeleteItem", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": TABLE_WITH_RANGE_NAME, - "Key": { - "HashKeyElement": {"S": "customer"}, - "RangeKeyElement": {"N": "12341236"}, - }, - } - res = test_client.post("/", headers=headers, json=request_body) - res = json.loads(res.data) - res.should.equal({"Attributes": [], "ConsumedCapacityUnits": 0.5}) - - # GetItem - headers["X-Amz-Target"] = "DynamoDB_20111205.GetItem" - request_body = { - "TableName": TABLE_WITH_RANGE_NAME, - "Key": { - "HashKeyElement": {"S": "customer"}, - "RangeKeyElement": {"N": "12341234"}, - }, - } - res = test_client.post("/", headers=headers, json=request_body) - res = json.loads(res.data) - - res["Item"].should.have.key("hkey").equal({"S": "customer"}) - res["Item"].should.have.key("rkey").equal({"N": "12341234"}) - res["Item"].should.have.key("name").equal({"S": "myname"}) - - -def test_update_item_that_doesnt_exist(): - backend = server.create_backend_app("dynamodb") - test_client = backend.test_client() - - create_table(test_client) - - # UpdateItem - headers = {"X-Amz-Target": "DynamoDB_20111205.UpdateItem"} - request_body = { - "TableName": "Table1", - "Key": { - "HashKeyElement": {"S": "customer"}, - "RangeKeyElement": {"N": "12341234"}, - }, - "AttributeUpdates": {"new_att": {"Value": {"SS": ["val"]}, "Action": "PUT"}}, - } - res = test_client.post("/", headers=headers, json=request_body) - res = json.loads(res.data) - res.should.equal( - {"__type": "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException"} - ) - - -def test_delete_item_without_range_key(test_client): - create_table(test_client, use_range_key=False) - - put_item(test_client, use_range_key=False) - - headers = { - "X-Amz-Target": "DynamoDB_20111205.DeleteItem", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": TABLE_NAME, - "Key": {"HashKeyElement": {"S": "customer"}}, - } - res = test_client.post("/", headers=headers, json=request_body) - res = json.loads(res.data) - res.should.equal({"Attributes": [], "ConsumedCapacityUnits": 0.5}) - - -def test_delete_item_with_return_values(test_client): - create_table(test_client) - - put_item(test_client) - put_item(test_client, rkey="12341235") - put_item(test_client, rkey="12341236") - - headers = { - "X-Amz-Target": "DynamoDB_20111205.DeleteItem", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": TABLE_WITH_RANGE_NAME, - "Key": { - "HashKeyElement": {"S": "customer"}, - "RangeKeyElement": {"N": "12341236"}, - }, - "ReturnValues": "ALL_OLD", - } - res = test_client.post("/", headers=headers, json=request_body) - res = json.loads(res.data) - res.should.equal( - { - "Attributes": {"hkey": "customer", "name": "myname", "rkey": "12341236"}, - "ConsumedCapacityUnits": 0.5, - } - ) - - -def test_delete_unknown_item(test_client): - create_table(test_client) - - headers = { - "X-Amz-Target": "DynamoDB_20111205.DeleteItem", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": TABLE_WITH_RANGE_NAME, - "Key": { - "HashKeyElement": {"S": "customer"}, - "RangeKeyElement": {"N": "12341236"}, - }, - "ReturnValues": "ALL_OLD", - } - res = test_client.post("/", headers=headers, json=request_body) - res.status_code.should.equal(400) - json.loads(res.data).should.equal( - {"__type": "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException"} - ) - - -def test_update_item_in_nonexisting_table(): - backend = server.create_backend_app("dynamodb") - test_client = backend.test_client() - - # UpdateItem - headers = {"X-Amz-Target": "DynamoDB_20111205.UpdateItem"} - request_body = { - "TableName": "nonexistent", - "Key": { - "HashKeyElement": {"S": "customer"}, - "RangeKeyElement": {"N": "12341234"}, - }, - "AttributeUpdates": {"new_att": {"Value": {"SS": ["val"]}, "Action": "PUT"}}, - } - res = test_client.post("/", headers=headers, json=request_body) - res.status_code.should.equal(400) - json.loads(res.data).should.equal( - {"__type": "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException"} - ) - - -def test_delete_from_unknown_table(test_client): - headers = { - "X-Amz-Target": "DynamoDB_20111205.DeleteItem", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": "unknown_table", - "Key": { - "HashKeyElement": {"S": "customer"}, - "RangeKeyElement": {"N": "12341236"}, - }, - "ReturnValues": "ALL_OLD", - } - res = test_client.post("/", headers=headers, json=request_body) - res.status_code.should.equal(400) - json.loads(res.data).should.equal( - {"__type": "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException"} - ) - - -def test_batch_get_item(test_client): - create_table(test_client) - - put_item(test_client) - put_item(test_client, rkey="12341235") - put_item(test_client, rkey="12341236") - - headers = { - "X-Amz-Target": "DynamoDB_20111205.BatchGetItem", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "RequestItems": { - TABLE_WITH_RANGE_NAME: { - "Keys": [ - { - "HashKeyElement": {"S": "customer"}, - "RangeKeyElement": {"N": "12341235"}, - }, - { - "HashKeyElement": {"S": "customer"}, - "RangeKeyElement": {"N": "12341236"}, - }, - ], - } - } - } - res = test_client.post("/", headers=headers, json=request_body) - res = json.loads(res.data)["Responses"] - - res.should.have.key("UnprocessedKeys").equal({}) - table_items = [i["Item"] for i in res[TABLE_WITH_RANGE_NAME]["Items"]] - table_items.should.have.length_of(2) - - table_items.should.contain( - {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "12341235"}} - ) - table_items.should.contain( - {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "12341236"}} - ) - - -def test_batch_get_item_without_range_key(test_client): - create_table(test_client, use_range_key=False) - - put_item(test_client, use_range_key=False, hkey="customer1") - put_item(test_client, use_range_key=False, hkey="customer2") - put_item(test_client, use_range_key=False, hkey="customer3") - - headers = { - "X-Amz-Target": "DynamoDB_20111205.BatchGetItem", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "RequestItems": { - TABLE_NAME: { - "Keys": [ - {"HashKeyElement": {"S": "customer1"}}, - {"HashKeyElement": {"S": "customer3"}}, - ], - } - } - } - res = test_client.post("/", headers=headers, json=request_body) - res = json.loads(res.data)["Responses"] - - res.should.have.key("UnprocessedKeys").equal({}) - table_items = [i["Item"] for i in res[TABLE_NAME]["Items"]] - table_items.should.have.length_of(2) - - table_items.should.contain({"hkey": {"S": "customer1"}, "name": {"S": "myname"}}) - table_items.should.contain({"hkey": {"S": "customer3"}, "name": {"S": "myname"}}) - - -def test_batch_write_item(test_client): - create_table(test_client) - - # BATCH-WRITE - headers = { - "X-Amz-Target": "DynamoDB_20111205.BatchWriteItem", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "RequestItems": { - TABLE_WITH_RANGE_NAME: [ - { - "PutRequest": { - "Item": {"hkey": {"S": "customer"}, "rkey": {"S": "1234"}} - } - }, - { - "PutRequest": { - "Item": {"hkey": {"S": "customer"}, "rkey": {"S": "1235"}} - } - }, - ], - } - } - test_client.post("/", headers=headers, json=request_body) - - # SCAN - verify all items are present - headers = { - "X-Amz-Target": "DynamoDB_20111205.Scan", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = {"TableName": TABLE_WITH_RANGE_NAME} - res = test_client.post("/", headers=headers, json=request_body) - res = json.loads(res.data) - - res.should.have.key("Count").equal(2) - - -def test_batch_write_item_without_range_key(test_client): - create_table(test_client, use_range_key=False) - - # BATCH-WRITE - headers = { - "X-Amz-Target": "DynamoDB_20111205.BatchWriteItem", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "RequestItems": { - TABLE_NAME: [ - {"PutRequest": {"Item": {"hkey": {"S": "customer"}}}}, - {"PutRequest": {"Item": {"hkey": {"S": "customer2"}}}}, - ], - } - } - test_client.post("/", headers=headers, json=request_body) - - # SCAN - verify all items are present - headers = { - "X-Amz-Target": "DynamoDB_20111205.Scan", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = {"TableName": TABLE_NAME} - res = test_client.post("/", headers=headers, json=request_body) - res = json.loads(res.data) - - res.should.have.key("Count").equal(2) - - -def put_item( - test_client, hkey="customer", rkey="12341234", name="myname", use_range_key=True -): - table_name = TABLE_WITH_RANGE_NAME if use_range_key else TABLE_NAME - headers = { - "X-Amz-Target": "DynamoDB_20111205.PutItem", - "Content-Type": "application/x-amz-json-1.0", - } - request_body = { - "TableName": table_name, - "Item": {"hkey": {"S": hkey}}, - "ReturnValues": "ALL_OLD", - } - if name: - request_body["Item"]["name"] = {"S": name} - if rkey and use_range_key: - request_body["Item"]["rkey"] = {"N": rkey} - res = test_client.post("/", headers=headers, json=request_body) - res = json.loads(res.data) - return headers, res - - -def create_table(test_client, name=None, region=None, use_range_key=True): - if not name: - name = TABLE_WITH_RANGE_NAME if use_range_key else TABLE_NAME - headers = { - "X-Amz-Target": "DynamoDB_20111205.CreateTable", - "Content-Type": "application/x-amz-json-1.0", - } - if region: - headers["Host"] = "dynamodb.{}.amazonaws.com".format(region) - request_body = { - "TableName": name, - "KeySchema": { - "HashKeyElement": {"AttributeName": "hkey", "AttributeType": "S"} - }, - "ProvisionedThroughput": {"ReadCapacityUnits": 5, "WriteCapacityUnits": 10}, - } - if use_range_key: - request_body["KeySchema"]["RangeKeyElement"] = { - "AttributeName": "rkey", - "AttributeType": "N", - } - return test_client.post("/", headers=headers, json=request_body) + res.data.should.contain(b"TableNames") + res.headers.should.have.key("X-Amz-Crc32") diff --git a/tests/test_dynamodb2/__init__.py b/tests/test_dynamodb2/__init__.py index 08a1c1568c9c..e69de29bb2d1 100644 --- a/tests/test_dynamodb2/__init__.py +++ b/tests/test_dynamodb2/__init__.py @@ -1 +0,0 @@ -# This file is intentionally left blank. diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 7eef4b13f84b..46d14bae004b 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -1,412 +1,23 @@ -import uuid -from datetime import datetime -from decimal import Decimal - import boto3 -from boto3.dynamodb.conditions import Attr, Key -import re import sure # noqa # pylint: disable=unused-import -from moto import mock_dynamodb2 -from moto.dynamodb2 import dynamodb_backends2 -from botocore.exceptions import ClientError - -import moto.dynamodb2.comparisons -import moto.dynamodb2.models - import pytest +from boto3.dynamodb.conditions import Key +from moto import mock_dynamodb2 -@mock_dynamodb2 -@pytest.mark.parametrize( - "names", - [[], ["TestTable"], ["TestTable1", "TestTable2"]], - ids=["no-table", "one-table", "multiple-tables"], -) -def test_list_tables_boto3(names): - conn = boto3.client("dynamodb", region_name="us-west-2") - for name in names: - conn.create_table( - TableName=name, - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - conn.list_tables()["TableNames"].should.equal(names) - - -@mock_dynamodb2 -def test_list_tables_paginated(): - conn = boto3.client("dynamodb", region_name="us-west-2") - for name in ["name1", "name2", "name3"]: - conn.create_table( - TableName=name, - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - - res = conn.list_tables(Limit=2) - res.should.have.key("TableNames").equal(["name1", "name2"]) - res.should.have.key("LastEvaluatedTableName").equal("name2") - - res = conn.list_tables(Limit=1, ExclusiveStartTableName="name1") - res.should.have.key("TableNames").equal(["name2"]) - res.should.have.key("LastEvaluatedTableName").equal("name2") - - res = conn.list_tables(ExclusiveStartTableName="name1") - res.should.have.key("TableNames").equal(["name2", "name3"]) - res.shouldnt.have.key("LastEvaluatedTableName") - - -@mock_dynamodb2 -def test_describe_missing_table_boto3(): - conn = boto3.client("dynamodb", region_name="us-west-2") - with pytest.raises(ClientError) as ex: - conn.describe_table(TableName="messages") - ex.value.response["Error"]["Code"].should.equal("ResourceNotFoundException") - ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) - ex.value.response["Error"]["Message"].should.equal("Requested resource not found") - - -@mock_dynamodb2 -def test_list_table_tags(): - name = "TestTable" - conn = boto3.client( - "dynamodb", - region_name="us-west-2", - aws_access_key_id="ak", - aws_secret_access_key="sk", - ) - conn.create_table( - TableName=name, - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - table_description = conn.describe_table(TableName=name) - arn = table_description["Table"]["TableArn"] - - # Tag table - tags = [ - {"Key": "TestTag", "Value": "TestValue"}, - {"Key": "TestTag2", "Value": "TestValue2"}, - ] - conn.tag_resource(ResourceArn=arn, Tags=tags) - - # Check tags - resp = conn.list_tags_of_resource(ResourceArn=arn) - assert resp["Tags"] == tags - - # Remove 1 tag - conn.untag_resource(ResourceArn=arn, TagKeys=["TestTag"]) - - # Check tags - resp = conn.list_tags_of_resource(ResourceArn=arn) - assert resp["Tags"] == [{"Key": "TestTag2", "Value": "TestValue2"}] - - -@mock_dynamodb2 -def test_list_table_tags_empty(): - name = "TestTable" - conn = boto3.client( - "dynamodb", - region_name="us-west-2", - aws_access_key_id="ak", - aws_secret_access_key="sk", - ) - conn.create_table( - TableName=name, - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - table_description = conn.describe_table(TableName=name) - arn = table_description["Table"]["TableArn"] - resp = conn.list_tags_of_resource(ResourceArn=arn) - assert resp["Tags"] == [] - - -@mock_dynamodb2 -def test_list_table_tags_paginated(): - name = "TestTable" - conn = boto3.client( - "dynamodb", - region_name="us-west-2", - aws_access_key_id="ak", - aws_secret_access_key="sk", - ) - conn.create_table( - TableName=name, - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - table_description = conn.describe_table(TableName=name) - arn = table_description["Table"]["TableArn"] - for i in range(11): - tags = [{"Key": "TestTag%d" % i, "Value": "TestValue"}] - conn.tag_resource(ResourceArn=arn, Tags=tags) - resp = conn.list_tags_of_resource(ResourceArn=arn) - assert len(resp["Tags"]) == 10 - assert "NextToken" in resp.keys() - resp2 = conn.list_tags_of_resource(ResourceArn=arn, NextToken=resp["NextToken"]) - assert len(resp2["Tags"]) == 1 - assert "NextToken" not in resp2.keys() - - -@mock_dynamodb2 -def test_list_not_found_table_tags(): - conn = boto3.client( - "dynamodb", - region_name="us-west-2", - aws_access_key_id="ak", - aws_secret_access_key="sk", - ) - arn = "DymmyArn" - try: - conn.list_tags_of_resource(ResourceArn=arn) - except ClientError as exception: - assert exception.response["Error"]["Code"] == "ResourceNotFoundException" - - -@mock_dynamodb2 -def test_item_add_empty_string_hash_key_exception(): - name = "TestTable" - conn = boto3.client( - "dynamodb", - region_name="us-west-2", - aws_access_key_id="ak", - aws_secret_access_key="sk", - ) - conn.create_table( - TableName=name, - KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - - with pytest.raises(ClientError) as ex: - conn.put_item( - TableName=name, - Item={ - "forum_name": {"S": ""}, - "subject": {"S": "Check this out!"}, - "Body": {"S": "http://url_to_lolcat.gif"}, - "SentBy": {"S": "someone@somewhere.edu"}, - "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"}, - }, - ) - - ex.value.response["Error"]["Code"].should.equal("ValidationException") - ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) - ex.value.response["Error"]["Message"].should.equal( - "One or more parameter values were invalid: An AttributeValue may not contain an empty string" - ) - - -@mock_dynamodb2 -def test_item_add_empty_string_range_key_exception(): - name = "TestTable" - conn = boto3.client( - "dynamodb", - region_name="us-west-2", - aws_access_key_id="ak", - aws_secret_access_key="sk", - ) - conn.create_table( - TableName=name, - KeySchema=[ - {"AttributeName": "forum_name", "KeyType": "HASH"}, - {"AttributeName": "ReceivedTime", "KeyType": "RANGE"}, - ], - AttributeDefinitions=[ - {"AttributeName": "forum_name", "AttributeType": "S"}, - {"AttributeName": "ReceivedTime", "AttributeType": "S"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - - with pytest.raises(ClientError) as ex: - conn.put_item( - TableName=name, - Item={ - "forum_name": {"S": "LOLCat Forum"}, - "subject": {"S": "Check this out!"}, - "Body": {"S": "http://url_to_lolcat.gif"}, - "SentBy": {"S": "someone@somewhere.edu"}, - "ReceivedTime": {"S": ""}, - }, - ) - - ex.value.response["Error"]["Code"].should.equal("ValidationException") - ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) - ex.value.response["Error"]["Message"].should.equal( - "One or more parameter values were invalid: An AttributeValue may not contain an empty string" - ) - - -@mock_dynamodb2 -def test_item_add_empty_string_attr_no_exception(): - name = "TestTable" - conn = boto3.client( - "dynamodb", - region_name="us-west-2", - aws_access_key_id="ak", - aws_secret_access_key="sk", - ) - conn.create_table( - TableName=name, - KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - - conn.put_item( - TableName=name, - Item={ - "forum_name": {"S": "LOLCat Forum"}, - "subject": {"S": "Check this out!"}, - "Body": {"S": "http://url_to_lolcat.gif"}, - "SentBy": {"S": ""}, - "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"}, - }, - ) - - -@mock_dynamodb2 -def test_update_item_with_empty_string_attr_no_exception(): - name = "TestTable" - conn = boto3.client( - "dynamodb", - region_name="us-west-2", - aws_access_key_id="ak", - aws_secret_access_key="sk", - ) - conn.create_table( - TableName=name, - KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - - conn.put_item( - TableName=name, - Item={ - "forum_name": {"S": "LOLCat Forum"}, - "subject": {"S": "Check this out!"}, - "Body": {"S": "http://url_to_lolcat.gif"}, - "SentBy": {"S": "test"}, - "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"}, - }, - ) - - conn.update_item( - TableName=name, - Key={"forum_name": {"S": "LOLCat Forum"}}, - UpdateExpression="set Body=:Body", - ExpressionAttributeValues={":Body": {"S": ""}}, - ) - - -@mock_dynamodb2 -def test_query_invalid_table(): - conn = boto3.client( - "dynamodb", - region_name="us-west-2", - aws_access_key_id="ak", - aws_secret_access_key="sk", - ) - try: - conn.query( - TableName="invalid_table", - KeyConditionExpression="index1 = :partitionkeyval", - ExpressionAttributeValues={":partitionkeyval": {"S": "test"}}, - ) - except ClientError as exception: - assert exception.response["Error"]["Code"] == "ResourceNotFoundException" - - -@mock_dynamodb2 -def test_put_item_with_special_chars(): - name = "TestTable" - conn = boto3.client( - "dynamodb", - region_name="us-west-2", - aws_access_key_id="ak", - aws_secret_access_key="sk", - ) - - conn.create_table( - TableName=name, - KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - - conn.put_item( - TableName=name, - Item={ - "forum_name": {"S": "LOLCat Forum"}, - "subject": {"S": "Check this out!"}, - "Body": {"S": "http://url_to_lolcat.gif"}, - "SentBy": {"S": "test"}, - "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"}, - '"': {"S": "foo"}, - }, - ) - - -@mock_dynamodb2 -def test_put_item_with_streams(): - name = "TestTable" - conn = boto3.client( - "dynamodb", - region_name="us-west-2", - aws_access_key_id="ak", - aws_secret_access_key="sk", - ) - - conn.create_table( - TableName=name, - KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], - StreamSpecification={ - "StreamEnabled": True, - "StreamViewType": "NEW_AND_OLD_IMAGES", - }, - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - conn.put_item( - TableName=name, - Item={ - "forum_name": {"S": "LOLCat Forum"}, - "subject": {"S": "Check this out!"}, - "Body": {"S": "http://url_to_lolcat.gif"}, - "SentBy": {"S": "test"}, - "Data": {"M": {"Key1": {"S": "Value1"}, "Key2": {"S": "Value2"}}}, - }, +def test_deprecation_warning(): + with pytest.warns(None) as record: + mock_dynamodb2() + str(record[0].message).should.contain( + "Module mock_dynamodb2 has been deprecated, and will be removed in a later release" ) - result = conn.get_item(TableName=name, Key={"forum_name": {"S": "LOLCat Forum"}}) - result["Item"].should.be.equal( - { - "forum_name": {"S": "LOLCat Forum"}, - "subject": {"S": "Check this out!"}, - "Body": {"S": "http://url_to_lolcat.gif"}, - "SentBy": {"S": "test"}, - "Data": {"M": {"Key1": {"S": "Value1"}, "Key2": {"S": "Value2"}}}, - } - ) - table = dynamodb_backends2["us-west-2"].get_table(name) - if not table: - # There is no way to access stream data over the API, so this part can't run in server-tests mode. - return - len(table.stream_shard.items).should.be.equal(1) - stream_record = table.stream_shard.items[0].record - stream_record["eventName"].should.be.equal("INSERT") - stream_record["dynamodb"]["SizeBytes"].should.be.equal(447) +""" +Copy some basics test from DynamoDB +Verify that the behaviour still works using the 'mock_dynamodb2' decorator +""" @mock_dynamodb2 @@ -414,7 +25,7 @@ def test_basic_projection_expression_using_get_item(): dynamodb = boto3.resource("dynamodb", region_name="us-east-1") # Create the DynamoDB table. - table = dynamodb.create_table( + dynamodb.create_table( TableName="users", KeySchema=[ {"AttributeName": "forum_name", "KeyType": "HASH"}, @@ -455,5354 +66,82 @@ def test_basic_projection_expression_using_get_item(): @mock_dynamodb2 -def test_basic_projection_expressions_using_scan(): - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - - # Create the DynamoDB table. +def test_condition_expression_with_dot_in_attr_name(): + dynamodb = boto3.resource("dynamodb", region_name="us-east-2") + table_name = "Test" dynamodb.create_table( - TableName="users", - KeySchema=[ - {"AttributeName": "forum_name", "KeyType": "HASH"}, - {"AttributeName": "subject", "KeyType": "RANGE"}, - ], - AttributeDefinitions=[ - {"AttributeName": "forum_name", "AttributeType": "S"}, - {"AttributeName": "subject", "AttributeType": "S"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - table = dynamodb.Table("users") - - table.put_item( - Item={"forum_name": "the-key", "subject": "123", "body": "some test message"} - ) - - table.put_item( - Item={ - "forum_name": "not-the-key", - "subject": "123", - "body": "some other test message", - } - ) - # Test a scan returning all items - results = table.scan( - FilterExpression=Key("forum_name").eq("the-key"), - ProjectionExpression="body, subject", - ) - - assert "body" in results["Items"][0] - assert results["Items"][0]["body"] == "some test message" - assert "subject" in results["Items"][0] - - table.put_item( - Item={ - "forum_name": "the-key", - "subject": "1234", - "body": "yet another test message", - } - ) - - results = table.scan( - FilterExpression=Key("forum_name").eq("the-key"), ProjectionExpression="body" + TableName=table_name, + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + BillingMode="PAY_PER_REQUEST", ) + table = dynamodb.Table(table_name) - bodies = [item["body"] for item in results["Items"]] - bodies.should.contain("some test message") - bodies.should.contain("yet another test message") - assert "subject" not in results["Items"][0] - assert "forum_name" not in results["Items"][0] - assert "subject" not in results["Items"][1] - assert "forum_name" not in results["Items"][1] - - # The projection expression should not remove data from storage - results = table.query(KeyConditionExpression=Key("forum_name").eq("the-key")) - assert "subject" in results["Items"][0] - assert "body" in results["Items"][1] - assert "forum_name" in results["Items"][1] - - -@mock_dynamodb2 -def test_nested_projection_expression_using_get_item(): - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + email_like_str = "test@foo.com" + record = { + "id": "key-0", + "first": {email_like_str: {"third": {"VALUE"}},}, + } + table.put_item(Item=record) - # Create the DynamoDB table. - dynamodb.create_table( - TableName="users", - KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - table = dynamodb.Table("users") - table.put_item( - Item={ - "forum_name": "key1", - "nested": { - "level1": {"id": "id1", "att": "irrelevant"}, - "level2": {"id": "id2", "include": "all"}, - "level3": {"id": "irrelevant"}, - }, - "foo": "bar", - } - ) - table.put_item( - Item={ - "forum_name": "key2", - "nested": {"id": "id2", "incode": "code2"}, - "foo": "bar", - } + table.update_item( + Key={"id": "key-0"}, + UpdateExpression="REMOVE #first.#second, #other", + ExpressionAttributeNames={ + "#first": "first", + "#second": email_like_str, + "#third": "third", + "#other": "other", + }, + ExpressionAttributeValues={":value": "VALUE", ":one": 1}, + ConditionExpression="size(#first.#second.#third) = :one AND contains(#first.#second.#third, :value)", + ReturnValues="ALL_NEW", ) - # Test a get_item returning all items - result = table.get_item( - Key={"forum_name": "key1"}, - ProjectionExpression="nested.level1.id, nested.level2", - )["Item"] - result.should.equal( - {"nested": {"level1": {"id": "id1"}, "level2": {"id": "id2", "include": "all"}}} - ) - # Assert actual data has not been deleted - result = table.get_item(Key={"forum_name": "key1"})["Item"] - result.should.equal( - { - "foo": "bar", - "forum_name": "key1", - "nested": { - "level1": {"id": "id1", "att": "irrelevant"}, - "level2": {"id": "id2", "include": "all"}, - "level3": {"id": "irrelevant"}, - }, - } + item = table.get_item(Key={"id": "key-0"})["Item"] + item.should.equal( + {"id": "key-0", "first": {},} ) @mock_dynamodb2 -def test_basic_projection_expressions_using_query(): - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - - # Create the DynamoDB table. - dynamodb.create_table( - TableName="users", - KeySchema=[ - {"AttributeName": "forum_name", "KeyType": "HASH"}, - {"AttributeName": "subject", "KeyType": "RANGE"}, +def test_query_filter_boto3(): + table_schema = { + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, ], - AttributeDefinitions=[ - {"AttributeName": "forum_name", "AttributeType": "S"}, - {"AttributeName": "subject", "AttributeType": "S"}, + "AttributeDefinitions": [ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, ], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - table = dynamodb.Table("users") - table.put_item( - Item={"forum_name": "the-key", "subject": "123", "body": "some test message"} - ) - table.put_item( - Item={ - "forum_name": "not-the-key", - "subject": "123", - "body": "some other test message", - } - ) - - # Test a query returning all items - result = table.query( - KeyConditionExpression=Key("forum_name").eq("the-key"), - ProjectionExpression="body, subject", - )["Items"][0] - - assert "body" in result - assert result["body"] == "some test message" - assert "subject" in result - assert "forum_name" not in result + } - table.put_item( - Item={ - "forum_name": "the-key", - "subject": "1234", - "body": "yet another test message", - } + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + table = dynamodb.create_table( + TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema ) - items = table.query( - KeyConditionExpression=Key("forum_name").eq("the-key"), - ProjectionExpression="body", - )["Items"] - - assert "body" in items[0] - assert "subject" not in items[0] - assert items[0]["body"] == "some test message" - assert "body" in items[1] - assert "subject" not in items[1] - assert items[1]["body"] == "yet another test message" - - # The projection expression should not remove data from storage - items = table.query(KeyConditionExpression=Key("forum_name").eq("the-key"))["Items"] - assert "subject" in items[0] - assert "body" in items[1] - assert "forum_name" in items[1] - - -@mock_dynamodb2 -def test_nested_projection_expression_using_query(): - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + for i in range(0, 3): + table.put_item( + Item={"pk": "pk".format(i), "sk": "sk-{}".format(i),} + ) - # Create the DynamoDB table. - dynamodb.create_table( - TableName="users", - KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - table = dynamodb.Table("users") - table.put_item( - Item={ - "forum_name": "key1", - "nested": { - "level1": {"id": "id1", "att": "irrelevant"}, - "level2": {"id": "id2", "include": "all"}, - "level3": {"id": "irrelevant"}, - }, - "foo": "bar", - } - ) - table.put_item( - Item={ - "forum_name": "key2", - "nested": {"id": "id2", "incode": "code2"}, - "foo": "bar", - } - ) + res = table.query(KeyConditionExpression=Key("pk").eq("pk")) + res["Items"].should.have.length_of(3) - # Test a query returning all items - result = table.query( - KeyConditionExpression=Key("forum_name").eq("key1"), - ProjectionExpression="nested.level1.id, nested.level2", - )["Items"][0] + res = table.query(KeyConditionExpression=Key("pk").eq("pk") & Key("sk").lt("sk-1")) + res["Items"].should.have.length_of(1) + res["Items"].should.equal([{"pk": "pk", "sk": "sk-0"}]) - assert "nested" in result - result["nested"].should.equal( - {"level1": {"id": "id1"}, "level2": {"id": "id2", "include": "all"}} - ) - assert "foo" not in result - # Assert actual data has not been deleted - result = table.query(KeyConditionExpression=Key("forum_name").eq("key1"))["Items"][ - 0 - ] - result.should.equal( - { - "foo": "bar", - "forum_name": "key1", - "nested": { - "level1": {"id": "id1", "att": "irrelevant"}, - "level2": {"id": "id2", "include": "all"}, - "level3": {"id": "irrelevant"}, - }, - } - ) + res = table.query(KeyConditionExpression=Key("pk").eq("pk") & Key("sk").lte("sk-1")) + res["Items"].should.have.length_of(2) + res["Items"].should.equal([{"pk": "pk", "sk": "sk-0"}, {"pk": "pk", "sk": "sk-1"}]) + res = table.query(KeyConditionExpression=Key("pk").eq("pk") & Key("sk").gt("sk-1")) + res["Items"].should.have.length_of(1) + res["Items"].should.equal([{"pk": "pk", "sk": "sk-2"}]) -@mock_dynamodb2 -def test_nested_projection_expression_using_scan(): - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - - # Create the DynamoDB table. - dynamodb.create_table( - TableName="users", - KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - table = dynamodb.Table("users") - table.put_item( - Item={ - "forum_name": "key1", - "nested": { - "level1": {"id": "id1", "att": "irrelevant"}, - "level2": {"id": "id2", "include": "all"}, - "level3": {"id": "irrelevant"}, - }, - "foo": "bar", - } - ) - table.put_item( - Item={ - "forum_name": "key2", - "nested": {"id": "id2", "incode": "code2"}, - "foo": "bar", - } - ) - - # Test a scan - results = table.scan( - FilterExpression=Key("forum_name").eq("key1"), - ProjectionExpression="nested.level1.id, nested.level2", - )["Items"] - results.should.equal( - [ - { - "nested": { - "level1": {"id": "id1"}, - "level2": {"include": "all", "id": "id2"}, - } - } - ] - ) - # Assert original data is still there - results = table.scan(FilterExpression=Key("forum_name").eq("key1"))["Items"] - results.should.equal( - [ - { - "forum_name": "key1", - "foo": "bar", - "nested": { - "level1": {"att": "irrelevant", "id": "id1"}, - "level2": {"include": "all", "id": "id2"}, - "level3": {"id": "irrelevant"}, - }, - } - ] - ) - - -@mock_dynamodb2 -def test_basic_projection_expression_using_get_item_with_attr_expression_names(): - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - - # Create the DynamoDB table. - table = dynamodb.create_table( - TableName="users", - KeySchema=[ - {"AttributeName": "forum_name", "KeyType": "HASH"}, - {"AttributeName": "subject", "KeyType": "RANGE"}, - ], - AttributeDefinitions=[ - {"AttributeName": "forum_name", "AttributeType": "S"}, - {"AttributeName": "subject", "AttributeType": "S"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - table = dynamodb.Table("users") - - table.put_item( - Item={ - "forum_name": "the-key", - "subject": "123", - "body": "some test message", - "attachment": "something", - } - ) - - table.put_item( - Item={ - "forum_name": "not-the-key", - "subject": "123", - "body": "some other test message", - "attachment": "something", - } - ) - result = table.get_item( - Key={"forum_name": "the-key", "subject": "123"}, - ProjectionExpression="#rl, #rt, subject", - ExpressionAttributeNames={"#rl": "body", "#rt": "attachment"}, - ) - - result["Item"].should.be.equal( - {"subject": "123", "body": "some test message", "attachment": "something"} - ) - - -@mock_dynamodb2 -def test_basic_projection_expressions_using_query_with_attr_expression_names(): - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - - # Create the DynamoDB table. - table = dynamodb.create_table( - TableName="users", - KeySchema=[ - {"AttributeName": "forum_name", "KeyType": "HASH"}, - {"AttributeName": "subject", "KeyType": "RANGE"}, - ], - AttributeDefinitions=[ - {"AttributeName": "forum_name", "AttributeType": "S"}, - {"AttributeName": "subject", "AttributeType": "S"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - table = dynamodb.Table("users") - - table.put_item( - Item={ - "forum_name": "the-key", - "subject": "123", - "body": "some test message", - "attachment": "something", - } - ) - - table.put_item( - Item={ - "forum_name": "not-the-key", - "subject": "123", - "body": "some other test message", - "attachment": "something", - } - ) - # Test a query returning all items - - results = table.query( - KeyConditionExpression=Key("forum_name").eq("the-key"), - ProjectionExpression="#rl, #rt, subject", - ExpressionAttributeNames={"#rl": "body", "#rt": "attachment"}, - ) - - assert "body" in results["Items"][0] - assert results["Items"][0]["body"] == "some test message" - assert "subject" in results["Items"][0] - assert results["Items"][0]["subject"] == "123" - assert "attachment" in results["Items"][0] - assert results["Items"][0]["attachment"] == "something" - - -@mock_dynamodb2 -def test_nested_projection_expression_using_get_item_with_attr_expression(): - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - - # Create the DynamoDB table. - dynamodb.create_table( - TableName="users", - KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - table = dynamodb.Table("users") - table.put_item( - Item={ - "forum_name": "key1", - "nested": { - "level1": {"id": "id1", "att": "irrelevant"}, - "level2": {"id": "id2", "include": "all"}, - "level3": {"id": "irrelevant"}, - }, - "foo": "bar", - } - ) - table.put_item( - Item={ - "forum_name": "key2", - "nested": {"id": "id2", "incode": "code2"}, - "foo": "bar", - } - ) - - # Test a get_item returning all items - result = table.get_item( - Key={"forum_name": "key1"}, - ProjectionExpression="#nst.level1.id, #nst.#lvl2", - ExpressionAttributeNames={"#nst": "nested", "#lvl2": "level2"}, - )["Item"] - result.should.equal( - {"nested": {"level1": {"id": "id1"}, "level2": {"id": "id2", "include": "all"}}} - ) - # Assert actual data has not been deleted - result = table.get_item(Key={"forum_name": "key1"})["Item"] - result.should.equal( - { - "foo": "bar", - "forum_name": "key1", - "nested": { - "level1": {"id": "id1", "att": "irrelevant"}, - "level2": {"id": "id2", "include": "all"}, - "level3": {"id": "irrelevant"}, - }, - } - ) - - -@mock_dynamodb2 -def test_nested_projection_expression_using_query_with_attr_expression_names(): - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - - # Create the DynamoDB table. - dynamodb.create_table( - TableName="users", - KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - table = dynamodb.Table("users") - table.put_item( - Item={ - "forum_name": "key1", - "nested": { - "level1": {"id": "id1", "att": "irrelevant"}, - "level2": {"id": "id2", "include": "all"}, - "level3": {"id": "irrelevant"}, - }, - "foo": "bar", - } - ) - table.put_item( - Item={ - "forum_name": "key2", - "nested": {"id": "id2", "incode": "code2"}, - "foo": "bar", - } - ) - - # Test a query returning all items - result = table.query( - KeyConditionExpression=Key("forum_name").eq("key1"), - ProjectionExpression="#nst.level1.id, #nst.#lvl2", - ExpressionAttributeNames={"#nst": "nested", "#lvl2": "level2"}, - )["Items"][0] - - assert "nested" in result - result["nested"].should.equal( - {"level1": {"id": "id1"}, "level2": {"id": "id2", "include": "all"}} - ) - assert "foo" not in result - # Assert actual data has not been deleted - result = table.query(KeyConditionExpression=Key("forum_name").eq("key1"))["Items"][ - 0 - ] - result.should.equal( - { - "foo": "bar", - "forum_name": "key1", - "nested": { - "level1": {"id": "id1", "att": "irrelevant"}, - "level2": {"id": "id2", "include": "all"}, - "level3": {"id": "irrelevant"}, - }, - } - ) - - -@mock_dynamodb2 -def test_basic_projection_expressions_using_scan_with_attr_expression_names(): - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - - # Create the DynamoDB table. - table = dynamodb.create_table( - TableName="users", - KeySchema=[ - {"AttributeName": "forum_name", "KeyType": "HASH"}, - {"AttributeName": "subject", "KeyType": "RANGE"}, - ], - AttributeDefinitions=[ - {"AttributeName": "forum_name", "AttributeType": "S"}, - {"AttributeName": "subject", "AttributeType": "S"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - table = dynamodb.Table("users") - - table.put_item( - Item={ - "forum_name": "the-key", - "subject": "123", - "body": "some test message", - "attachment": "something", - } - ) - - table.put_item( - Item={ - "forum_name": "not-the-key", - "subject": "123", - "body": "some other test message", - "attachment": "something", - } - ) - # Test a scan returning all items - - results = table.scan( - FilterExpression=Key("forum_name").eq("the-key"), - ProjectionExpression="#rl, #rt, subject", - ExpressionAttributeNames={"#rl": "body", "#rt": "attachment"}, - ) - - assert "body" in results["Items"][0] - assert "attachment" in results["Items"][0] - assert "subject" in results["Items"][0] - assert "form_name" not in results["Items"][0] - - # Test without a FilterExpression - results = table.scan( - ProjectionExpression="#rl, #rt, subject", - ExpressionAttributeNames={"#rl": "body", "#rt": "attachment"}, - ) - - assert "body" in results["Items"][0] - assert "attachment" in results["Items"][0] - assert "subject" in results["Items"][0] - assert "form_name" not in results["Items"][0] - - -@mock_dynamodb2 -def test_nested_projection_expression_using_scan_with_attr_expression_names(): - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - - # Create the DynamoDB table. - dynamodb.create_table( - TableName="users", - KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - table = dynamodb.Table("users") - table.put_item( - Item={ - "forum_name": "key1", - "nested": { - "level1": {"id": "id1", "att": "irrelevant"}, - "level2": {"id": "id2", "include": "all"}, - "level3": {"id": "irrelevant"}, - }, - "foo": "bar", - } - ) - table.put_item( - Item={ - "forum_name": "key2", - "nested": {"id": "id2", "incode": "code2"}, - "foo": "bar", - } - ) - - # Test a scan - results = table.scan( - FilterExpression=Key("forum_name").eq("key1"), - ProjectionExpression="nested.level1.id, nested.level2", - ExpressionAttributeNames={"#nst": "nested", "#lvl2": "level2"}, - )["Items"] - results.should.equal( - [ - { - "nested": { - "level1": {"id": "id1"}, - "level2": {"include": "all", "id": "id2"}, - } - } - ] - ) - # Assert original data is still there - results = table.scan(FilterExpression=Key("forum_name").eq("key1"))["Items"] - results.should.equal( - [ - { - "forum_name": "key1", - "foo": "bar", - "nested": { - "level1": {"att": "irrelevant", "id": "id1"}, - "level2": {"include": "all", "id": "id2"}, - "level3": {"id": "irrelevant"}, - }, - } - ] - ) - - -@mock_dynamodb2 -def test_put_empty_item(): - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - dynamodb.create_table( - AttributeDefinitions=[{"AttributeName": "structure_id", "AttributeType": "S"},], - TableName="test", - KeySchema=[{"AttributeName": "structure_id", "KeyType": "HASH"},], - ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, - ) - table = dynamodb.Table("test") - - with pytest.raises(ClientError) as ex: - table.put_item(Item={}) - ex.value.response["Error"]["Message"].should.equal( - "One or more parameter values were invalid: Missing the key structure_id in the item" - ) - ex.value.response["Error"]["Code"].should.equal("ValidationException") - - -@mock_dynamodb2 -def test_put_item_nonexisting_hash_key(): - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - dynamodb.create_table( - AttributeDefinitions=[{"AttributeName": "structure_id", "AttributeType": "S"},], - TableName="test", - KeySchema=[{"AttributeName": "structure_id", "KeyType": "HASH"},], - ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, - ) - table = dynamodb.Table("test") - - with pytest.raises(ClientError) as ex: - table.put_item(Item={"a_terribly_misguided_id_attribute": "abcdef"}) - ex.value.response["Error"]["Message"].should.equal( - "One or more parameter values were invalid: Missing the key structure_id in the item" - ) - ex.value.response["Error"]["Code"].should.equal("ValidationException") - - -@mock_dynamodb2 -def test_put_item_nonexisting_range_key(): - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - dynamodb.create_table( - AttributeDefinitions=[ - {"AttributeName": "structure_id", "AttributeType": "S"}, - {"AttributeName": "added_at", "AttributeType": "N"}, - ], - TableName="test", - KeySchema=[ - {"AttributeName": "structure_id", "KeyType": "HASH"}, - {"AttributeName": "added_at", "KeyType": "RANGE"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, - ) - table = dynamodb.Table("test") - - with pytest.raises(ClientError) as ex: - table.put_item(Item={"structure_id": "abcdef"}) - ex.value.response["Error"]["Message"].should.equal( - "One or more parameter values were invalid: Missing the key added_at in the item" - ) - ex.value.response["Error"]["Code"].should.equal("ValidationException") - - -def test_filter_expression(): - row1 = moto.dynamodb2.models.Item( - hash_key=None, - range_key=None, - attrs={ - "Id": {"N": "8"}, - "Subs": {"N": "5"}, - "Des": {"S": "Some description"}, - "KV": {"SS": ["test1", "test2"]}, - }, - ) - row2 = moto.dynamodb2.models.Item( - hash_key=None, - range_key=None, - attrs={ - "Id": {"N": "8"}, - "Subs": {"N": "10"}, - "Des": {"S": "A description"}, - "KV": {"SS": ["test3", "test4"]}, - }, - ) - - # NOT test 1 - filter_expr = moto.dynamodb2.comparisons.get_filter_expression( - "NOT attribute_not_exists(Id)", {}, {} - ) - filter_expr.expr(row1).should.be(True) - - # NOT test 2 - filter_expr = moto.dynamodb2.comparisons.get_filter_expression( - "NOT (Id = :v0)", {}, {":v0": {"N": "8"}} - ) - filter_expr.expr(row1).should.be(False) # Id = 8 so should be false - - # AND test - filter_expr = moto.dynamodb2.comparisons.get_filter_expression( - "Id > :v0 AND Subs < :v1", {}, {":v0": {"N": "5"}, ":v1": {"N": "7"}} - ) - filter_expr.expr(row1).should.be(True) - filter_expr.expr(row2).should.be(False) - - # lowercase AND test - filter_expr = moto.dynamodb2.comparisons.get_filter_expression( - "Id > :v0 and Subs < :v1", {}, {":v0": {"N": "5"}, ":v1": {"N": "7"}} - ) - filter_expr.expr(row1).should.be(True) - filter_expr.expr(row2).should.be(False) - - # OR test - filter_expr = moto.dynamodb2.comparisons.get_filter_expression( - "Id = :v0 OR Id=:v1", {}, {":v0": {"N": "5"}, ":v1": {"N": "8"}} - ) - filter_expr.expr(row1).should.be(True) - - # BETWEEN test - filter_expr = moto.dynamodb2.comparisons.get_filter_expression( - "Id BETWEEN :v0 AND :v1", {}, {":v0": {"N": "5"}, ":v1": {"N": "10"}} - ) - filter_expr.expr(row1).should.be(True) - - # BETWEEN integer test - filter_expr = moto.dynamodb2.comparisons.get_filter_expression( - "Id BETWEEN :v0 AND :v1", {}, {":v0": {"N": "0"}, ":v1": {"N": "10"}} - ) - filter_expr.expr(row1).should.be(True) - - # PAREN test - filter_expr = moto.dynamodb2.comparisons.get_filter_expression( - "Id = :v0 AND (Subs = :v0 OR Subs = :v1)", - {}, - {":v0": {"N": "8"}, ":v1": {"N": "5"}}, - ) - filter_expr.expr(row1).should.be(True) - - # IN test - filter_expr = moto.dynamodb2.comparisons.get_filter_expression( - "Id IN (:v0, :v1, :v2)", - {}, - {":v0": {"N": "7"}, ":v1": {"N": "8"}, ":v2": {"N": "9"}}, - ) - filter_expr.expr(row1).should.be(True) - - # attribute function tests (with extra spaces) - filter_expr = moto.dynamodb2.comparisons.get_filter_expression( - "attribute_exists(Id) AND attribute_not_exists (UnknownAttribute)", {}, {} - ) - filter_expr.expr(row1).should.be(True) - - filter_expr = moto.dynamodb2.comparisons.get_filter_expression( - "attribute_type(Id, :v0)", {}, {":v0": {"S": "N"}} - ) - filter_expr.expr(row1).should.be(True) - - # beginswith function test - filter_expr = moto.dynamodb2.comparisons.get_filter_expression( - "begins_with(Des, :v0)", {}, {":v0": {"S": "Some"}} - ) - filter_expr.expr(row1).should.be(True) - filter_expr.expr(row2).should.be(False) - - # contains function test - filter_expr = moto.dynamodb2.comparisons.get_filter_expression( - "contains(KV, :v0)", {}, {":v0": {"S": "test1"}} - ) - filter_expr.expr(row1).should.be(True) - filter_expr.expr(row2).should.be(False) - - # size function test - filter_expr = moto.dynamodb2.comparisons.get_filter_expression( - "size(Des) > size(KV)", {}, {} - ) - filter_expr.expr(row1).should.be(True) - - # Expression from @batkuip - filter_expr = moto.dynamodb2.comparisons.get_filter_expression( - "(#n0 < :v0 AND attribute_not_exists(#n1))", - {"#n0": "Subs", "#n1": "fanout_ts"}, - {":v0": {"N": "7"}}, - ) - filter_expr.expr(row1).should.be(True) - # Expression from to check contains on string value - filter_expr = moto.dynamodb2.comparisons.get_filter_expression( - "contains(#n0, :v0)", {"#n0": "Des"}, {":v0": {"S": "Some"}} - ) - filter_expr.expr(row1).should.be(True) - filter_expr.expr(row2).should.be(False) - - -@mock_dynamodb2 -def test_query_filter(): - client = boto3.client("dynamodb", region_name="us-east-1") - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - - # Create the DynamoDB table. - client.create_table( - TableName="test1", - AttributeDefinitions=[ - {"AttributeName": "client", "AttributeType": "S"}, - {"AttributeName": "app", "AttributeType": "S"}, - ], - KeySchema=[ - {"AttributeName": "client", "KeyType": "HASH"}, - {"AttributeName": "app", "KeyType": "RANGE"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, - ) - client.put_item( - TableName="test1", - Item={ - "client": {"S": "client1"}, - "app": {"S": "app1"}, - "nested": { - "M": { - "version": {"S": "version1"}, - "contents": {"L": [{"S": "value1"}, {"S": "value2"}]}, - } - }, - }, - ) - client.put_item( - TableName="test1", - Item={ - "client": {"S": "client1"}, - "app": {"S": "app2"}, - "nested": { - "M": { - "version": {"S": "version2"}, - "contents": {"L": [{"S": "value1"}, {"S": "value2"}]}, - } - }, - }, - ) - - table = dynamodb.Table("test1") - response = table.query(KeyConditionExpression=Key("client").eq("client1")) - assert response["Count"] == 2 - - response = table.query( - KeyConditionExpression=Key("client").eq("client1"), - FilterExpression=Attr("app").eq("app2"), - ) - assert response["Count"] == 1 - assert response["Items"][0]["app"] == "app2" - response = table.query( - KeyConditionExpression=Key("client").eq("client1"), - FilterExpression=Attr("app").contains("app"), - ) - assert response["Count"] == 2 - - response = table.query( - KeyConditionExpression=Key("client").eq("client1"), - FilterExpression=Attr("nested.version").contains("version"), - ) - assert response["Count"] == 2 - - response = table.query( - KeyConditionExpression=Key("client").eq("client1"), - FilterExpression=Attr("nested.contents[0]").eq("value1"), - ) - assert response["Count"] == 2 - - -@mock_dynamodb2 -def test_query_filter_overlapping_expression_prefixes(): - client = boto3.client("dynamodb", region_name="us-east-1") - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - - # Create the DynamoDB table. - client.create_table( - TableName="test1", - AttributeDefinitions=[ - {"AttributeName": "client", "AttributeType": "S"}, - {"AttributeName": "app", "AttributeType": "S"}, - ], - KeySchema=[ - {"AttributeName": "client", "KeyType": "HASH"}, - {"AttributeName": "app", "KeyType": "RANGE"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, - ) - - client.put_item( - TableName="test1", - Item={ - "client": {"S": "client1"}, - "app": {"S": "app1"}, - "nested": { - "M": { - "version": {"S": "version1"}, - "contents": {"L": [{"S": "value1"}, {"S": "value2"}]}, - } - }, - }, - ) - - table = dynamodb.Table("test1") - response = table.query( - KeyConditionExpression=Key("client").eq("client1") & Key("app").eq("app1"), - ProjectionExpression="#1, #10, nested", - ExpressionAttributeNames={"#1": "client", "#10": "app"}, - ) - - assert response["Count"] == 1 - assert response["Items"][0] == { - "client": "client1", - "app": "app1", - "nested": {"version": "version1", "contents": ["value1", "value2"]}, - } - - -@mock_dynamodb2 -def test_scan_filter(): - client = boto3.client("dynamodb", region_name="us-east-1") - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - - # Create the DynamoDB table. - client.create_table( - TableName="test1", - AttributeDefinitions=[ - {"AttributeName": "client", "AttributeType": "S"}, - {"AttributeName": "app", "AttributeType": "S"}, - ], - KeySchema=[ - {"AttributeName": "client", "KeyType": "HASH"}, - {"AttributeName": "app", "KeyType": "RANGE"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, - ) - client.put_item( - TableName="test1", Item={"client": {"S": "client1"}, "app": {"S": "app1"}} - ) - - table = dynamodb.Table("test1") - response = table.scan(FilterExpression=Attr("app").eq("app2")) - assert response["Count"] == 0 - - response = table.scan(FilterExpression=Attr("app").eq("app1")) - assert response["Count"] == 1 - - response = table.scan(FilterExpression=Attr("app").ne("app2")) - assert response["Count"] == 1 - - response = table.scan(FilterExpression=Attr("app").ne("app1")) - assert response["Count"] == 0 - - -@mock_dynamodb2 -def test_scan_filter2(): - client = boto3.client("dynamodb", region_name="us-east-1") - - # Create the DynamoDB table. - client.create_table( - TableName="test1", - AttributeDefinitions=[ - {"AttributeName": "client", "AttributeType": "S"}, - {"AttributeName": "app", "AttributeType": "N"}, - ], - KeySchema=[ - {"AttributeName": "client", "KeyType": "HASH"}, - {"AttributeName": "app", "KeyType": "RANGE"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, - ) - client.put_item( - TableName="test1", Item={"client": {"S": "client1"}, "app": {"N": "1"}} - ) - - response = client.scan( - TableName="test1", - Select="ALL_ATTRIBUTES", - FilterExpression="#tb >= :dt", - ExpressionAttributeNames={"#tb": "app"}, - ExpressionAttributeValues={":dt": {"N": str(1)}}, - ) - assert response["Count"] == 1 - - -@mock_dynamodb2 -def test_scan_filter3(): - client = boto3.client("dynamodb", region_name="us-east-1") - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - - # Create the DynamoDB table. - client.create_table( - TableName="test1", - AttributeDefinitions=[ - {"AttributeName": "client", "AttributeType": "S"}, - {"AttributeName": "app", "AttributeType": "N"}, - ], - KeySchema=[ - {"AttributeName": "client", "KeyType": "HASH"}, - {"AttributeName": "app", "KeyType": "RANGE"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, - ) - client.put_item( - TableName="test1", - Item={"client": {"S": "client1"}, "app": {"N": "1"}, "active": {"BOOL": True}}, - ) - - table = dynamodb.Table("test1") - response = table.scan(FilterExpression=Attr("active").eq(True)) - assert response["Count"] == 1 - - response = table.scan(FilterExpression=Attr("active").ne(True)) - assert response["Count"] == 0 - - response = table.scan(FilterExpression=Attr("active").ne(False)) - assert response["Count"] == 1 - - response = table.scan(FilterExpression=Attr("app").ne(1)) - assert response["Count"] == 0 - - response = table.scan(FilterExpression=Attr("app").ne(2)) - assert response["Count"] == 1 - - -@mock_dynamodb2 -def test_scan_filter4(): - client = boto3.client("dynamodb", region_name="us-east-1") - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - - # Create the DynamoDB table. - client.create_table( - TableName="test1", - AttributeDefinitions=[ - {"AttributeName": "client", "AttributeType": "S"}, - {"AttributeName": "app", "AttributeType": "N"}, - ], - KeySchema=[ - {"AttributeName": "client", "KeyType": "HASH"}, - {"AttributeName": "app", "KeyType": "RANGE"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, - ) - - table = dynamodb.Table("test1") - response = table.scan( - FilterExpression=Attr("epoch_ts").lt(7) & Attr("fanout_ts").not_exists() - ) - # Just testing - assert response["Count"] == 0 - - -@mock_dynamodb2 -def test_scan_filter_should_not_return_non_existing_attributes(): - table_name = "my-table" - item = {"partitionKey": "pk-2", "my-attr": 42} - # Create table - res = boto3.resource("dynamodb", region_name="us-east-1") - res.create_table( - TableName=table_name, - KeySchema=[{"AttributeName": "partitionKey", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "partitionKey", "AttributeType": "S"}], - BillingMode="PAY_PER_REQUEST", - ) - table = res.Table(table_name) - # Insert items - table.put_item(Item={"partitionKey": "pk-1"}) - table.put_item(Item=item) - # Verify a few operations - # Assert we only find the item that has this attribute - table.scan(FilterExpression=Attr("my-attr").lt(43))["Items"].should.equal([item]) - table.scan(FilterExpression=Attr("my-attr").lte(42))["Items"].should.equal([item]) - table.scan(FilterExpression=Attr("my-attr").gte(42))["Items"].should.equal([item]) - table.scan(FilterExpression=Attr("my-attr").gt(41))["Items"].should.equal([item]) - # Sanity check that we can't find the item if the FE is wrong - table.scan(FilterExpression=Attr("my-attr").gt(43))["Items"].should.equal([]) - - -@mock_dynamodb2 -def test_bad_scan_filter(): - client = boto3.client("dynamodb", region_name="us-east-1") - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - - # Create the DynamoDB table. - client.create_table( - TableName="test1", - AttributeDefinitions=[ - {"AttributeName": "client", "AttributeType": "S"}, - {"AttributeName": "app", "AttributeType": "S"}, - ], - KeySchema=[ - {"AttributeName": "client", "KeyType": "HASH"}, - {"AttributeName": "app", "KeyType": "RANGE"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, - ) - table = dynamodb.Table("test1") - - # Bad expression - try: - table.scan(FilterExpression="client test") - except ClientError as err: - err.response["Error"]["Code"].should.equal("ValidationError") - else: - raise RuntimeError("Should have raised ResourceInUseException") - - -@mock_dynamodb2 -def test_duplicate_create(): - client = boto3.client("dynamodb", region_name="us-east-1") - - # Create the DynamoDB table. - client.create_table( - TableName="test1", - AttributeDefinitions=[ - {"AttributeName": "client", "AttributeType": "S"}, - {"AttributeName": "app", "AttributeType": "S"}, - ], - KeySchema=[ - {"AttributeName": "client", "KeyType": "HASH"}, - {"AttributeName": "app", "KeyType": "RANGE"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, - ) - - try: - client.create_table( - TableName="test1", - AttributeDefinitions=[ - {"AttributeName": "client", "AttributeType": "S"}, - {"AttributeName": "app", "AttributeType": "S"}, - ], - KeySchema=[ - {"AttributeName": "client", "KeyType": "HASH"}, - {"AttributeName": "app", "KeyType": "RANGE"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, - ) - except ClientError as err: - err.response["Error"]["Code"].should.equal("ResourceInUseException") - else: - raise RuntimeError("Should have raised ResourceInUseException") - - -@mock_dynamodb2 -def test_delete_table(): - client = boto3.client("dynamodb", region_name="us-east-1") - - # Create the DynamoDB table. - client.create_table( - TableName="test1", - AttributeDefinitions=[ - {"AttributeName": "client", "AttributeType": "S"}, - {"AttributeName": "app", "AttributeType": "S"}, - ], - KeySchema=[ - {"AttributeName": "client", "KeyType": "HASH"}, - {"AttributeName": "app", "KeyType": "RANGE"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, - ) - - client.delete_table(TableName="test1") - - resp = client.list_tables() - len(resp["TableNames"]).should.equal(0) - - try: - client.delete_table(TableName="test1") - except ClientError as err: - err.response["Error"]["Code"].should.equal("ResourceNotFoundException") - else: - raise RuntimeError("Should have raised ResourceNotFoundException") - - -@mock_dynamodb2 -def test_delete_item(): - client = boto3.client("dynamodb", region_name="us-east-1") - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - - # Create the DynamoDB table. - client.create_table( - TableName="test1", - AttributeDefinitions=[ - {"AttributeName": "client", "AttributeType": "S"}, - {"AttributeName": "app", "AttributeType": "S"}, - ], - KeySchema=[ - {"AttributeName": "client", "KeyType": "HASH"}, - {"AttributeName": "app", "KeyType": "RANGE"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, - ) - client.put_item( - TableName="test1", Item={"client": {"S": "client1"}, "app": {"S": "app1"}} - ) - client.put_item( - TableName="test1", Item={"client": {"S": "client1"}, "app": {"S": "app2"}} - ) - - table = dynamodb.Table("test1") - response = table.scan() - assert response["Count"] == 2 - - # Test ReturnValues validation - with pytest.raises(ClientError) as ex: - table.delete_item( - Key={"client": "client1", "app": "app1"}, ReturnValues="ALL_NEW" - ) - err = ex.value.response["Error"] - err["Code"].should.equal("ValidationException") - err["Message"].should.equal("Return values set to invalid value") - - # Test deletion and returning old value - response = table.delete_item( - Key={"client": "client1", "app": "app1"}, ReturnValues="ALL_OLD" - ) - response["Attributes"].should.contain("client") - response["Attributes"].should.contain("app") - - response = table.scan() - assert response["Count"] == 1 - - # Test deletion returning nothing - response = table.delete_item(Key={"client": "client1", "app": "app2"}) - len(response["Attributes"]).should.equal(0) - - response = table.scan() - assert response["Count"] == 0 - - -@mock_dynamodb2 -def test_describe_limits(): - client = boto3.client("dynamodb", region_name="eu-central-1") - resp = client.describe_limits() - - resp["AccountMaxReadCapacityUnits"].should.equal(20000) - resp["AccountMaxWriteCapacityUnits"].should.equal(20000) - resp["TableMaxWriteCapacityUnits"].should.equal(10000) - resp["TableMaxReadCapacityUnits"].should.equal(10000) - - -@mock_dynamodb2 -def test_set_ttl(): - client = boto3.client("dynamodb", region_name="us-east-1") - - # Create the DynamoDB table. - client.create_table( - TableName="test1", - AttributeDefinitions=[ - {"AttributeName": "client", "AttributeType": "S"}, - {"AttributeName": "app", "AttributeType": "S"}, - ], - KeySchema=[ - {"AttributeName": "client", "KeyType": "HASH"}, - {"AttributeName": "app", "KeyType": "RANGE"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, - ) - - client.update_time_to_live( - TableName="test1", - TimeToLiveSpecification={"Enabled": True, "AttributeName": "expire"}, - ) - - resp = client.describe_time_to_live(TableName="test1") - resp["TimeToLiveDescription"]["TimeToLiveStatus"].should.equal("ENABLED") - resp["TimeToLiveDescription"]["AttributeName"].should.equal("expire") - - client.update_time_to_live( - TableName="test1", - TimeToLiveSpecification={"Enabled": False, "AttributeName": "expire"}, - ) - - resp = client.describe_time_to_live(TableName="test1") - resp["TimeToLiveDescription"]["TimeToLiveStatus"].should.equal("DISABLED") - - -@mock_dynamodb2 -def test_describe_continuous_backups(): - # given - client = boto3.client("dynamodb", region_name="us-east-1") - table_name = client.create_table( - TableName="test", - AttributeDefinitions=[ - {"AttributeName": "client", "AttributeType": "S"}, - {"AttributeName": "app", "AttributeType": "S"}, - ], - KeySchema=[ - {"AttributeName": "client", "KeyType": "HASH"}, - {"AttributeName": "app", "KeyType": "RANGE"}, - ], - BillingMode="PAY_PER_REQUEST", - )["TableDescription"]["TableName"] - - # when - response = client.describe_continuous_backups(TableName=table_name) - - # then - response["ContinuousBackupsDescription"].should.equal( - { - "ContinuousBackupsStatus": "ENABLED", - "PointInTimeRecoveryDescription": {"PointInTimeRecoveryStatus": "DISABLED"}, - } - ) - - -@mock_dynamodb2 -def test_describe_continuous_backups_errors(): - # given - client = boto3.client("dynamodb", region_name="us-east-1") - - # when - with pytest.raises(Exception) as e: - client.describe_continuous_backups(TableName="not-existing-table") - - # then - ex = e.value - ex.operation_name.should.equal("DescribeContinuousBackups") - ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) - ex.response["Error"]["Code"].should.contain("TableNotFoundException") - ex.response["Error"]["Message"].should.equal("Table not found: not-existing-table") - - -@mock_dynamodb2 -def test_update_continuous_backups(): - # given - client = boto3.client("dynamodb", region_name="us-east-1") - table_name = client.create_table( - TableName="test", - AttributeDefinitions=[ - {"AttributeName": "client", "AttributeType": "S"}, - {"AttributeName": "app", "AttributeType": "S"}, - ], - KeySchema=[ - {"AttributeName": "client", "KeyType": "HASH"}, - {"AttributeName": "app", "KeyType": "RANGE"}, - ], - BillingMode="PAY_PER_REQUEST", - )["TableDescription"]["TableName"] - - # when - response = client.update_continuous_backups( - TableName=table_name, - PointInTimeRecoverySpecification={"PointInTimeRecoveryEnabled": True}, - ) - - # then - response["ContinuousBackupsDescription"]["ContinuousBackupsStatus"].should.equal( - "ENABLED" - ) - point_in_time = response["ContinuousBackupsDescription"][ - "PointInTimeRecoveryDescription" - ] - earliest_datetime = point_in_time["EarliestRestorableDateTime"] - earliest_datetime.should.be.a(datetime) - latest_datetime = point_in_time["LatestRestorableDateTime"] - latest_datetime.should.be.a(datetime) - point_in_time["PointInTimeRecoveryStatus"].should.equal("ENABLED") - - # when - # a second update should not change anything - response = client.update_continuous_backups( - TableName=table_name, - PointInTimeRecoverySpecification={"PointInTimeRecoveryEnabled": True}, - ) - - # then - response["ContinuousBackupsDescription"]["ContinuousBackupsStatus"].should.equal( - "ENABLED" - ) - point_in_time = response["ContinuousBackupsDescription"][ - "PointInTimeRecoveryDescription" - ] - point_in_time["EarliestRestorableDateTime"].should.equal(earliest_datetime) - point_in_time["LatestRestorableDateTime"].should.equal(latest_datetime) - point_in_time["PointInTimeRecoveryStatus"].should.equal("ENABLED") - - # when - response = client.update_continuous_backups( - TableName=table_name, - PointInTimeRecoverySpecification={"PointInTimeRecoveryEnabled": False}, - ) - - # then - response["ContinuousBackupsDescription"].should.equal( - { - "ContinuousBackupsStatus": "ENABLED", - "PointInTimeRecoveryDescription": {"PointInTimeRecoveryStatus": "DISABLED"}, - } - ) - - -@mock_dynamodb2 -def test_update_continuous_backups_errors(): - # given - client = boto3.client("dynamodb", region_name="us-east-1") - - # when - with pytest.raises(Exception) as e: - client.update_continuous_backups( - TableName="not-existing-table", - PointInTimeRecoverySpecification={"PointInTimeRecoveryEnabled": True}, - ) - - # then - ex = e.value - ex.operation_name.should.equal("UpdateContinuousBackups") - ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) - ex.response["Error"]["Code"].should.contain("TableNotFoundException") - ex.response["Error"]["Message"].should.equal("Table not found: not-existing-table") - - -# https://github.com/spulec/moto/issues/1043 -@mock_dynamodb2 -def test_query_missing_expr_names(): - client = boto3.client("dynamodb", region_name="us-east-1") - - # Create the DynamoDB table. - client.create_table( - TableName="test1", - AttributeDefinitions=[ - {"AttributeName": "client", "AttributeType": "S"}, - {"AttributeName": "app", "AttributeType": "S"}, - ], - KeySchema=[ - {"AttributeName": "client", "KeyType": "HASH"}, - {"AttributeName": "app", "KeyType": "RANGE"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 123, "WriteCapacityUnits": 123}, - ) - client.put_item( - TableName="test1", Item={"client": {"S": "test1"}, "app": {"S": "test1"}} - ) - client.put_item( - TableName="test1", Item={"client": {"S": "test2"}, "app": {"S": "test2"}} - ) - - resp = client.query( - TableName="test1", - KeyConditionExpression="client=:client", - ExpressionAttributeValues={":client": {"S": "test1"}}, - ) - - resp["Count"].should.equal(1) - resp["Items"][0]["client"]["S"].should.equal("test1") - - resp = client.query( - TableName="test1", - KeyConditionExpression=":name=test2", - ExpressionAttributeNames={":name": "client"}, - ) - - resp["Count"].should.equal(1) - resp["Items"][0]["client"]["S"].should.equal("test2") - - -# https://github.com/spulec/moto/issues/2328 -@mock_dynamodb2 -def test_update_item_with_list(): - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - - # Create the DynamoDB table. - dynamodb.create_table( - TableName="Table", - KeySchema=[{"AttributeName": "key", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "key", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, - ) - table = dynamodb.Table("Table") - table.update_item( - Key={"key": "the-key"}, - AttributeUpdates={"list": {"Value": [1, 2], "Action": "PUT"}}, - ) - - resp = table.get_item(Key={"key": "the-key"}) - resp["Item"].should.equal({"key": "the-key", "list": [1, 2]}) - - -# https://github.com/spulec/moto/issues/2328 -@mock_dynamodb2 -def test_update_item_with_no_action_passed_with_list(): - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - - # Create the DynamoDB table. - dynamodb.create_table( - TableName="Table", - KeySchema=[{"AttributeName": "key", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "key", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, - ) - table = dynamodb.Table("Table") - table.update_item( - Key={"key": "the-key"}, - # Do not pass 'Action' key, in order to check that the - # parameter's default value will be used. - AttributeUpdates={"list": {"Value": [1, 2]}}, - ) - - resp = table.get_item(Key={"key": "the-key"}) - resp["Item"].should.equal({"key": "the-key", "list": [1, 2]}) - - -# https://github.com/spulec/moto/issues/1342 -@mock_dynamodb2 -def test_update_item_on_map(): - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - client = boto3.client("dynamodb", region_name="us-east-1") - - # Create the DynamoDB table. - dynamodb.create_table( - TableName="users", - KeySchema=[ - {"AttributeName": "forum_name", "KeyType": "HASH"}, - {"AttributeName": "subject", "KeyType": "RANGE"}, - ], - AttributeDefinitions=[ - {"AttributeName": "forum_name", "AttributeType": "S"}, - {"AttributeName": "subject", "AttributeType": "S"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - table = dynamodb.Table("users") - - table.put_item( - Item={ - "forum_name": "the-key", - "subject": "123", - "body": {"nested": {"data": "test"}}, - } - ) - - resp = table.scan() - resp["Items"][0]["body"].should.equal({"nested": {"data": "test"}}) - - # Nonexistent nested attributes are supported for existing top-level attributes. - table.update_item( - Key={"forum_name": "the-key", "subject": "123"}, - UpdateExpression="SET body.#nested.#data = :tb", - ExpressionAttributeNames={"#nested": "nested", "#data": "data",}, - ExpressionAttributeValues={":tb": "new_value"}, - ) - # Running this against AWS DDB gives an exception so make sure it also fails.: - with pytest.raises(client.exceptions.ClientError): - # botocore.exceptions.ClientError: An error occurred (ValidationException) when calling the UpdateItem - # operation: The document path provided in the update expression is invalid for update - table.update_item( - Key={"forum_name": "the-key", "subject": "123"}, - UpdateExpression="SET body.#nested.#nonexistentnested.#data = :tb2", - ExpressionAttributeNames={ - "#nested": "nested", - "#nonexistentnested": "nonexistentnested", - "#data": "data", - }, - ExpressionAttributeValues={":tb2": "other_value"}, - ) - - table.update_item( - Key={"forum_name": "the-key", "subject": "123"}, - UpdateExpression="SET body.#nested.#nonexistentnested = :tb2", - ExpressionAttributeNames={ - "#nested": "nested", - "#nonexistentnested": "nonexistentnested", - }, - ExpressionAttributeValues={":tb2": {"data": "other_value"}}, - ) - - resp = table.scan() - resp["Items"][0]["body"].should.equal( - {"nested": {"data": "new_value", "nonexistentnested": {"data": "other_value"}}} - ) - - # Test nested value for a nonexistent attribute throws a ClientError. - with pytest.raises(client.exceptions.ClientError): - table.update_item( - Key={"forum_name": "the-key", "subject": "123"}, - UpdateExpression="SET nonexistent.#nested = :tb", - ExpressionAttributeNames={"#nested": "nested"}, - ExpressionAttributeValues={":tb": "new_value"}, - ) - - -# https://github.com/spulec/moto/issues/1358 -@mock_dynamodb2 -def test_update_if_not_exists(): - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - - # Create the DynamoDB table. - dynamodb.create_table( - TableName="users", - KeySchema=[ - {"AttributeName": "forum_name", "KeyType": "HASH"}, - {"AttributeName": "subject", "KeyType": "RANGE"}, - ], - AttributeDefinitions=[ - {"AttributeName": "forum_name", "AttributeType": "S"}, - {"AttributeName": "subject", "AttributeType": "S"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - table = dynamodb.Table("users") - - table.put_item(Item={"forum_name": "the-key", "subject": "123"}) - - table.update_item( - Key={"forum_name": "the-key", "subject": "123"}, - # if_not_exists without space - UpdateExpression="SET created_at=if_not_exists(created_at,:created_at)", - ExpressionAttributeValues={":created_at": 123}, - ) - - resp = table.scan() - assert resp["Items"][0]["created_at"] == 123 - - table.update_item( - Key={"forum_name": "the-key", "subject": "123"}, - # if_not_exists with space - UpdateExpression="SET created_at = if_not_exists (created_at, :created_at)", - ExpressionAttributeValues={":created_at": 456}, - ) - - resp = table.scan() - # Still the original value - assert resp["Items"][0]["created_at"] == 123 - - -# https://github.com/spulec/moto/issues/1937 -@mock_dynamodb2 -def test_update_return_attributes(): - dynamodb = boto3.client("dynamodb", region_name="us-east-1") - - dynamodb.create_table( - TableName="moto-test", - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, - ) - - def update(col, to, rv): - return dynamodb.update_item( - TableName="moto-test", - Key={"id": {"S": "foo"}}, - AttributeUpdates={col: {"Value": {"S": to}, "Action": "PUT"}}, - ReturnValues=rv, - ) - - r = update("col1", "val1", "ALL_NEW") - assert r["Attributes"] == {"id": {"S": "foo"}, "col1": {"S": "val1"}} - - r = update("col1", "val2", "ALL_OLD") - assert r["Attributes"] == {"id": {"S": "foo"}, "col1": {"S": "val1"}} - - r = update("col2", "val3", "UPDATED_NEW") - assert r["Attributes"] == {"col2": {"S": "val3"}} - - r = update("col2", "val4", "UPDATED_OLD") - assert r["Attributes"] == {"col2": {"S": "val3"}} - - r = update("col1", "val5", "NONE") - assert r["Attributes"] == {} - - with pytest.raises(ClientError) as ex: - update("col1", "val6", "WRONG") - err = ex.value.response["Error"] - err["Code"].should.equal("ValidationException") - err["Message"].should.equal("Return values set to invalid value") - - -# https://github.com/spulec/moto/issues/3448 -@mock_dynamodb2 -def test_update_return_updated_new_attributes_when_same(): - dynamo_client = boto3.resource("dynamodb", region_name="us-east-1") - dynamo_client.create_table( - TableName="moto-test", - KeySchema=[{"AttributeName": "HashKey1", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "HashKey1", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, - ) - - dynamodb_table = dynamo_client.Table("moto-test") - dynamodb_table.put_item( - Item={"HashKey1": "HashKeyValue1", "listValuedAttribute1": ["a", "b"]} - ) - - def update(col, to, rv): - return dynamodb_table.update_item( - TableName="moto-test", - Key={"HashKey1": "HashKeyValue1"}, - UpdateExpression="SET listValuedAttribute1=:" + col, - ExpressionAttributeValues={":" + col: to}, - ReturnValues=rv, - ) - - r = update("a", ["a", "c"], "UPDATED_NEW") - assert r["Attributes"] == {"listValuedAttribute1": ["a", "c"]} - - r = update("a", {"a", "c"}, "UPDATED_NEW") - assert r["Attributes"] == {"listValuedAttribute1": {"a", "c"}} - - r = update("a", {1, 2}, "UPDATED_NEW") - assert r["Attributes"] == {"listValuedAttribute1": {1, 2}} - - with pytest.raises(ClientError) as ex: - update("a", ["a", "c"], "WRONG") - err = ex.value.response["Error"] - err["Code"].should.equal("ValidationException") - err["Message"].should.equal("Return values set to invalid value") - - -@mock_dynamodb2 -def test_put_return_attributes(): - dynamodb = boto3.client("dynamodb", region_name="us-east-1") - - dynamodb.create_table( - TableName="moto-test", - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, - ) - - r = dynamodb.put_item( - TableName="moto-test", - Item={"id": {"S": "foo"}, "col1": {"S": "val1"}}, - ReturnValues="NONE", - ) - assert "Attributes" not in r - - r = dynamodb.put_item( - TableName="moto-test", - Item={"id": {"S": "foo"}, "col1": {"S": "val2"}}, - ReturnValues="ALL_OLD", - ) - assert r["Attributes"] == {"id": {"S": "foo"}, "col1": {"S": "val1"}} - - with pytest.raises(ClientError) as ex: - dynamodb.put_item( - TableName="moto-test", - Item={"id": {"S": "foo"}, "col1": {"S": "val3"}}, - ReturnValues="ALL_NEW", - ) - ex.value.response["Error"]["Code"].should.equal("ValidationException") - ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) - ex.value.response["Error"]["Message"].should.equal( - "Return values set to invalid value" - ) - - -@mock_dynamodb2 -def test_query_global_secondary_index_when_created_via_update_table_resource(): - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - - # Create the DynamoDB table. - dynamodb.create_table( - TableName="users", - KeySchema=[{"AttributeName": "user_id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "user_id", "AttributeType": "N"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - table = dynamodb.Table("users") - table.update( - AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], - GlobalSecondaryIndexUpdates=[ - { - "Create": { - "IndexName": "forum_name_index", - "KeySchema": [{"AttributeName": "forum_name", "KeyType": "HASH"}], - "Projection": {"ProjectionType": "ALL"}, - "ProvisionedThroughput": { - "ReadCapacityUnits": 5, - "WriteCapacityUnits": 5, - }, - } - } - ], - ) - - next_user_id = 1 - for my_forum_name in ["cats", "dogs"]: - for my_subject in [ - "my pet is the cutest", - "wow look at what my pet did", - "don't you love my pet?", - ]: - table.put_item( - Item={ - "user_id": next_user_id, - "forum_name": my_forum_name, - "subject": my_subject, - } - ) - next_user_id += 1 - - # get all the cat users - forum_only_query_response = table.query( - IndexName="forum_name_index", - Select="ALL_ATTRIBUTES", - KeyConditionExpression=Key("forum_name").eq("cats"), - ) - forum_only_items = forum_only_query_response["Items"] - assert len(forum_only_items) == 3 - for item in forum_only_items: - assert item["forum_name"] == "cats" - - # query all cat users with a particular subject - forum_and_subject_query_results = table.query( - IndexName="forum_name_index", - Select="ALL_ATTRIBUTES", - KeyConditionExpression=Key("forum_name").eq("cats"), - FilterExpression=Attr("subject").eq("my pet is the cutest"), - ) - forum_and_subject_items = forum_and_subject_query_results["Items"] - assert len(forum_and_subject_items) == 1 - assert forum_and_subject_items[0] == { - "user_id": Decimal("1"), - "forum_name": "cats", - "subject": "my pet is the cutest", - } - - -@mock_dynamodb2 -def test_query_gsi_with_range_key(): - dynamodb = boto3.client("dynamodb", region_name="us-east-1") - dynamodb.create_table( - TableName="test", - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - AttributeDefinitions=[ - {"AttributeName": "id", "AttributeType": "S"}, - {"AttributeName": "gsi_hash_key", "AttributeType": "S"}, - {"AttributeName": "gsi_range_key", "AttributeType": "S"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, - GlobalSecondaryIndexes=[ - { - "IndexName": "test_gsi", - "KeySchema": [ - {"AttributeName": "gsi_hash_key", "KeyType": "HASH"}, - {"AttributeName": "gsi_range_key", "KeyType": "RANGE"}, - ], - "Projection": {"ProjectionType": "ALL"}, - "ProvisionedThroughput": { - "ReadCapacityUnits": 1, - "WriteCapacityUnits": 1, - }, - } - ], - ) - - dynamodb.put_item( - TableName="test", - Item={ - "id": {"S": "test1"}, - "gsi_hash_key": {"S": "key1"}, - "gsi_range_key": {"S": "range1"}, - }, - ) - dynamodb.put_item( - TableName="test", Item={"id": {"S": "test2"}, "gsi_hash_key": {"S": "key1"}} - ) - - res = dynamodb.query( - TableName="test", - IndexName="test_gsi", - KeyConditionExpression="gsi_hash_key = :gsi_hash_key and gsi_range_key = :gsi_range_key", - ExpressionAttributeValues={ - ":gsi_hash_key": {"S": "key1"}, - ":gsi_range_key": {"S": "range1"}, - }, - ) - res.should.have.key("Count").equal(1) - res.should.have.key("Items") - res["Items"][0].should.equal( - { - "id": {"S": "test1"}, - "gsi_hash_key": {"S": "key1"}, - "gsi_range_key": {"S": "range1"}, - } - ) - - -@mock_dynamodb2 -def test_scan_by_non_exists_index(): - dynamodb = boto3.client("dynamodb", region_name="us-east-1") - - dynamodb.create_table( - TableName="test", - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - AttributeDefinitions=[ - {"AttributeName": "id", "AttributeType": "S"}, - {"AttributeName": "gsi_col", "AttributeType": "S"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, - GlobalSecondaryIndexes=[ - { - "IndexName": "test_gsi", - "KeySchema": [{"AttributeName": "gsi_col", "KeyType": "HASH"}], - "Projection": {"ProjectionType": "ALL"}, - "ProvisionedThroughput": { - "ReadCapacityUnits": 1, - "WriteCapacityUnits": 1, - }, - } - ], - ) - - with pytest.raises(ClientError) as ex: - dynamodb.scan(TableName="test", IndexName="non_exists_index") - - ex.value.response["Error"]["Code"].should.equal("ValidationException") - ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) - ex.value.response["Error"]["Message"].should.equal( - "The table does not have the specified index: non_exists_index" - ) - - -@mock_dynamodb2 -def test_query_by_non_exists_index(): - dynamodb = boto3.client("dynamodb", region_name="us-east-1") - - dynamodb.create_table( - TableName="test", - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - AttributeDefinitions=[ - {"AttributeName": "id", "AttributeType": "S"}, - {"AttributeName": "gsi_col", "AttributeType": "S"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, - GlobalSecondaryIndexes=[ - { - "IndexName": "test_gsi", - "KeySchema": [{"AttributeName": "gsi_col", "KeyType": "HASH"}], - "Projection": {"ProjectionType": "ALL"}, - "ProvisionedThroughput": { - "ReadCapacityUnits": 1, - "WriteCapacityUnits": 1, - }, - } - ], - ) - - with pytest.raises(ClientError) as ex: - dynamodb.query( - TableName="test", - IndexName="non_exists_index", - KeyConditionExpression="CarModel=M", - ) - - ex.value.response["Error"]["Code"].should.equal("ResourceNotFoundException") - ex.value.response["Error"]["Message"].should.equal( - "Invalid index: non_exists_index for table: test. Available indexes are: test_gsi" - ) - - -@mock_dynamodb2 -def test_batch_items_returns_all(): - dynamodb = _create_user_table() - returned_items = dynamodb.batch_get_item( - RequestItems={ - "users": { - "Keys": [ - {"username": {"S": "user0"}}, - {"username": {"S": "user1"}}, - {"username": {"S": "user2"}}, - {"username": {"S": "user3"}}, - ], - "ConsistentRead": True, - } - } - )["Responses"]["users"] - assert len(returned_items) == 3 - assert [item["username"]["S"] for item in returned_items] == [ - "user1", - "user2", - "user3", - ] - - -@mock_dynamodb2 -def test_batch_items_throws_exception_when_requesting_100_items_for_single_table(): - dynamodb = _create_user_table() - with pytest.raises(ClientError) as ex: - dynamodb.batch_get_item( - RequestItems={ - "users": { - "Keys": [ - {"username": {"S": "user" + str(i)}} for i in range(0, 104) - ], - "ConsistentRead": True, - } - } - ) - ex.value.response["Error"]["Code"].should.equal("ValidationException") - msg = ex.value.response["Error"]["Message"] - msg.should.contain("1 validation error detected: Value") - msg.should.contain( - "at 'requestItems.users.member.keys' failed to satisfy constraint: Member must have length less than or equal to 100" - ) - - -@mock_dynamodb2 -def test_batch_items_throws_exception_when_requesting_100_items_across_all_tables(): - dynamodb = _create_user_table() - with pytest.raises(ClientError) as ex: - dynamodb.batch_get_item( - RequestItems={ - "users": { - "Keys": [ - {"username": {"S": "user" + str(i)}} for i in range(0, 75) - ], - "ConsistentRead": True, - }, - "users2": { - "Keys": [ - {"username": {"S": "user" + str(i)}} for i in range(0, 75) - ], - "ConsistentRead": True, - }, - } - ) - ex.value.response["Error"]["Code"].should.equal("ValidationException") - ex.value.response["Error"]["Message"].should.equal( - "Too many items requested for the BatchGetItem call" - ) - - -@mock_dynamodb2 -def test_batch_items_with_basic_projection_expression(): - dynamodb = _create_user_table() - returned_items = dynamodb.batch_get_item( - RequestItems={ - "users": { - "Keys": [ - {"username": {"S": "user0"}}, - {"username": {"S": "user1"}}, - {"username": {"S": "user2"}}, - {"username": {"S": "user3"}}, - ], - "ConsistentRead": True, - "ProjectionExpression": "username", - } - } - )["Responses"]["users"] - - returned_items.should.have.length_of(3) - [item["username"]["S"] for item in returned_items].should.be.equal( - ["user1", "user2", "user3"] - ) - [item.get("foo") for item in returned_items].should.be.equal([None, None, None]) - - # The projection expression should not remove data from storage - returned_items = dynamodb.batch_get_item( - RequestItems={ - "users": { - "Keys": [ - {"username": {"S": "user0"}}, - {"username": {"S": "user1"}}, - {"username": {"S": "user2"}}, - {"username": {"S": "user3"}}, - ], - "ConsistentRead": True, - } - } - )["Responses"]["users"] - - [item["username"]["S"] for item in returned_items].should.be.equal( - ["user1", "user2", "user3"] - ) - [item["foo"]["S"] for item in returned_items].should.be.equal(["bar", "bar", "bar"]) - - -@mock_dynamodb2 -def test_batch_items_with_basic_projection_expression_and_attr_expression_names(): - dynamodb = _create_user_table() - returned_items = dynamodb.batch_get_item( - RequestItems={ - "users": { - "Keys": [ - {"username": {"S": "user0"}}, - {"username": {"S": "user1"}}, - {"username": {"S": "user2"}}, - {"username": {"S": "user3"}}, - ], - "ConsistentRead": True, - "ProjectionExpression": "#rl", - "ExpressionAttributeNames": {"#rl": "username"}, - } - } - )["Responses"]["users"] - - returned_items.should.have.length_of(3) - [item["username"]["S"] for item in returned_items].should.be.equal( - ["user1", "user2", "user3"] - ) - [item.get("foo") for item in returned_items].should.be.equal([None, None, None]) - - -@mock_dynamodb2 -def test_batch_items_should_throw_exception_for_duplicate_request(): - client = _create_user_table() - with pytest.raises(ClientError) as ex: - client.batch_get_item( - RequestItems={ - "users": { - "Keys": [ - {"username": {"S": "user0"}}, - {"username": {"S": "user0"}}, - ], - "ConsistentRead": True, - } - } - ) - ex.value.response["Error"]["Code"].should.equal("ValidationException") - ex.value.response["Error"]["Message"].should.equal( - "Provided list of item keys contains duplicates" - ) - - -@mock_dynamodb2 -def test_index_with_unknown_attributes_should_fail(): - dynamodb = boto3.client("dynamodb", region_name="us-east-1") - - expected_exception = ( - "Some index key attributes are not defined in AttributeDefinitions." - ) - - with pytest.raises(ClientError) as ex: - dynamodb.create_table( - AttributeDefinitions=[ - {"AttributeName": "customer_nr", "AttributeType": "S"}, - {"AttributeName": "last_name", "AttributeType": "S"}, - ], - TableName="table_with_missing_attribute_definitions", - KeySchema=[ - {"AttributeName": "customer_nr", "KeyType": "HASH"}, - {"AttributeName": "last_name", "KeyType": "RANGE"}, - ], - LocalSecondaryIndexes=[ - { - "IndexName": "indexthataddsanadditionalattribute", - "KeySchema": [ - {"AttributeName": "customer_nr", "KeyType": "HASH"}, - {"AttributeName": "postcode", "KeyType": "RANGE"}, - ], - "Projection": {"ProjectionType": "ALL"}, - } - ], - BillingMode="PAY_PER_REQUEST", - ) - - ex.value.response["Error"]["Code"].should.equal("ValidationException") - ex.value.response["Error"]["Message"].should.contain(expected_exception) - - -@mock_dynamodb2 -def test_update_list_index__set_existing_index(): - table_name = "test_list_index_access" - client = create_table_with_list(table_name) - client.put_item( - TableName=table_name, - Item={ - "id": {"S": "foo"}, - "itemlist": {"L": [{"S": "bar1"}, {"S": "bar2"}, {"S": "bar3"}]}, - }, - ) - client.update_item( - TableName=table_name, - Key={"id": {"S": "foo"}}, - UpdateExpression="set itemlist[1]=:Item", - ExpressionAttributeValues={":Item": {"S": "bar2_update"}}, - ) - # - result = client.get_item(TableName=table_name, Key={"id": {"S": "foo"}})["Item"] - result["id"].should.equal({"S": "foo"}) - result["itemlist"].should.equal( - {"L": [{"S": "bar1"}, {"S": "bar2_update"}, {"S": "bar3"}]} - ) - - -@mock_dynamodb2 -def test_update_list_index__set_existing_nested_index(): - table_name = "test_list_index_access" - client = create_table_with_list(table_name) - client.put_item( - TableName=table_name, - Item={ - "id": {"S": "foo2"}, - "itemmap": { - "M": {"itemlist": {"L": [{"S": "bar1"}, {"S": "bar2"}, {"S": "bar3"}]}} - }, - }, - ) - client.update_item( - TableName=table_name, - Key={"id": {"S": "foo2"}}, - UpdateExpression="set itemmap.itemlist[1]=:Item", - ExpressionAttributeValues={":Item": {"S": "bar2_update"}}, - ) - # - result = client.get_item(TableName=table_name, Key={"id": {"S": "foo2"}})["Item"] - result["id"].should.equal({"S": "foo2"}) - result["itemmap"]["M"]["itemlist"]["L"].should.equal( - [{"S": "bar1"}, {"S": "bar2_update"}, {"S": "bar3"}] - ) - - -@mock_dynamodb2 -def test_update_list_index__set_index_out_of_range(): - table_name = "test_list_index_access" - client = create_table_with_list(table_name) - client.put_item( - TableName=table_name, - Item={ - "id": {"S": "foo"}, - "itemlist": {"L": [{"S": "bar1"}, {"S": "bar2"}, {"S": "bar3"}]}, - }, - ) - client.update_item( - TableName=table_name, - Key={"id": {"S": "foo"}}, - UpdateExpression="set itemlist[10]=:Item", - ExpressionAttributeValues={":Item": {"S": "bar10"}}, - ) - # - result = client.get_item(TableName=table_name, Key={"id": {"S": "foo"}})["Item"] - assert result["id"] == {"S": "foo"} - assert result["itemlist"] == { - "L": [{"S": "bar1"}, {"S": "bar2"}, {"S": "bar3"}, {"S": "bar10"}] - } - - -@mock_dynamodb2 -def test_update_list_index__set_nested_index_out_of_range(): - table_name = "test_list_index_access" - client = create_table_with_list(table_name) - client.put_item( - TableName=table_name, - Item={ - "id": {"S": "foo2"}, - "itemmap": { - "M": {"itemlist": {"L": [{"S": "bar1"}, {"S": "bar2"}, {"S": "bar3"}]}} - }, - }, - ) - client.update_item( - TableName=table_name, - Key={"id": {"S": "foo2"}}, - UpdateExpression="set itemmap.itemlist[10]=:Item", - ExpressionAttributeValues={":Item": {"S": "bar10"}}, - ) - # - result = client.get_item(TableName=table_name, Key={"id": {"S": "foo2"}})["Item"] - assert result["id"] == {"S": "foo2"} - assert result["itemmap"]["M"]["itemlist"]["L"] == [ - {"S": "bar1"}, - {"S": "bar2"}, - {"S": "bar3"}, - {"S": "bar10"}, - ] - - -@mock_dynamodb2 -def test_update_list_index__set_double_nested_index(): - table_name = "test_list_index_access" - client = create_table_with_list(table_name) - client.put_item( - TableName=table_name, - Item={ - "id": {"S": "foo2"}, - "itemmap": { - "M": { - "itemlist": { - "L": [ - {"M": {"foo": {"S": "bar11"}, "foos": {"S": "bar12"}}}, - {"M": {"foo": {"S": "bar21"}, "foos": {"S": "bar21"}}}, - ] - } - } - }, - }, - ) - client.update_item( - TableName=table_name, - Key={"id": {"S": "foo2"}}, - UpdateExpression="set itemmap.itemlist[1].foos=:Item", - ExpressionAttributeValues={":Item": {"S": "bar22"}}, - ) - # - result = client.get_item(TableName=table_name, Key={"id": {"S": "foo2"}})["Item"] - assert result["id"] == {"S": "foo2"} - len(result["itemmap"]["M"]["itemlist"]["L"]).should.equal(2) - result["itemmap"]["M"]["itemlist"]["L"][0].should.equal( - {"M": {"foo": {"S": "bar11"}, "foos": {"S": "bar12"}}} - ) # unchanged - result["itemmap"]["M"]["itemlist"]["L"][1].should.equal( - {"M": {"foo": {"S": "bar21"}, "foos": {"S": "bar22"}}} - ) # updated - - -@mock_dynamodb2 -def test_update_list_index__set_index_of_a_string(): - table_name = "test_list_index_access" - client = create_table_with_list(table_name) - client.put_item( - TableName=table_name, Item={"id": {"S": "foo2"}, "itemstr": {"S": "somestring"}} - ) - with pytest.raises(ClientError) as ex: - client.update_item( - TableName=table_name, - Key={"id": {"S": "foo2"}}, - UpdateExpression="set itemstr[1]=:Item", - ExpressionAttributeValues={":Item": {"S": "string_update"}}, - ) - client.get_item(TableName=table_name, Key={"id": {"S": "foo2"}})["Item"] - - ex.value.response["Error"]["Code"].should.equal("ValidationException") - ex.value.response["Error"]["Message"].should.equal( - "The document path provided in the update expression is invalid for update" - ) - - -@mock_dynamodb2 -def test_remove_top_level_attribute(): - table_name = "test_remove" - client = create_table_with_list(table_name) - client.put_item( - TableName=table_name, Item={"id": {"S": "foo"}, "item": {"S": "bar"}} - ) - client.update_item( - TableName=table_name, - Key={"id": {"S": "foo"}}, - UpdateExpression="REMOVE #i", - ExpressionAttributeNames={"#i": "item"}, - ) - # - result = client.get_item(TableName=table_name, Key={"id": {"S": "foo"}})["Item"] - result.should.equal({"id": {"S": "foo"}}) - - -@mock_dynamodb2 -def test_remove_top_level_attribute_non_existent(): - """ - Remove statements do not require attribute to exist they silently pass - """ - table_name = "test_remove" - client = create_table_with_list(table_name) - ddb_item = {"id": {"S": "foo"}, "item": {"S": "bar"}} - client.put_item(TableName=table_name, Item=ddb_item) - client.update_item( - TableName=table_name, - Key={"id": {"S": "foo"}}, - UpdateExpression="REMOVE non_existent_attribute", - ExpressionAttributeNames={"#i": "item"}, - ) - result = client.get_item(TableName=table_name, Key={"id": {"S": "foo"}})["Item"] - result.should.equal(ddb_item) - - -@mock_dynamodb2 -def test_remove_list_index__remove_existing_index(): - table_name = "test_list_index_access" - client = create_table_with_list(table_name) - client.put_item( - TableName=table_name, - Item={ - "id": {"S": "foo"}, - "itemlist": {"L": [{"S": "bar1"}, {"S": "bar2"}, {"S": "bar3"}]}, - }, - ) - client.update_item( - TableName=table_name, - Key={"id": {"S": "foo"}}, - UpdateExpression="REMOVE itemlist[1]", - ) - # - result = client.get_item(TableName=table_name, Key={"id": {"S": "foo"}})["Item"] - result["id"].should.equal({"S": "foo"}) - result["itemlist"].should.equal({"L": [{"S": "bar1"}, {"S": "bar3"}]}) - - -@mock_dynamodb2 -def test_remove_list_index__remove_existing_nested_index(): - table_name = "test_list_index_access" - client = create_table_with_list(table_name) - client.put_item( - TableName=table_name, - Item={ - "id": {"S": "foo2"}, - "itemmap": {"M": {"itemlist": {"L": [{"S": "bar1"}, {"S": "bar2"}]}}}, - }, - ) - client.update_item( - TableName=table_name, - Key={"id": {"S": "foo2"}}, - UpdateExpression="REMOVE itemmap.itemlist[1]", - ) - # - result = client.get_item(TableName=table_name, Key={"id": {"S": "foo2"}})["Item"] - result["id"].should.equal({"S": "foo2"}) - result["itemmap"]["M"]["itemlist"]["L"].should.equal([{"S": "bar1"}]) - - -@mock_dynamodb2 -def test_remove_list_index__remove_existing_double_nested_index(): - table_name = "test_list_index_access" - client = create_table_with_list(table_name) - client.put_item( - TableName=table_name, - Item={ - "id": {"S": "foo2"}, - "itemmap": { - "M": { - "itemlist": { - "L": [ - {"M": {"foo00": {"S": "bar1"}, "foo01": {"S": "bar2"}}}, - {"M": {"foo10": {"S": "bar1"}, "foo11": {"S": "bar2"}}}, - ] - } - } - }, - }, - ) - client.update_item( - TableName=table_name, - Key={"id": {"S": "foo2"}}, - UpdateExpression="REMOVE itemmap.itemlist[1].foo10", - ) - # - result = client.get_item(TableName=table_name, Key={"id": {"S": "foo2"}})["Item"] - assert result["id"] == {"S": "foo2"} - assert result["itemmap"]["M"]["itemlist"]["L"][0]["M"].should.equal( - {"foo00": {"S": "bar1"}, "foo01": {"S": "bar2"}} - ) # untouched - assert result["itemmap"]["M"]["itemlist"]["L"][1]["M"].should.equal( - {"foo11": {"S": "bar2"}} - ) # changed - - -@mock_dynamodb2 -def test_remove_list_index__remove_index_out_of_range(): - table_name = "test_list_index_access" - client = create_table_with_list(table_name) - client.put_item( - TableName=table_name, - Item={ - "id": {"S": "foo"}, - "itemlist": {"L": [{"S": "bar1"}, {"S": "bar2"}, {"S": "bar3"}]}, - }, - ) - client.update_item( - TableName=table_name, - Key={"id": {"S": "foo"}}, - UpdateExpression="REMOVE itemlist[10]", - ) - # - result = client.get_item(TableName=table_name, Key={"id": {"S": "foo"}})["Item"] - assert result["id"] == {"S": "foo"} - assert result["itemlist"] == {"L": [{"S": "bar1"}, {"S": "bar2"}, {"S": "bar3"}]} - - -def create_table_with_list(table_name): - client = boto3.client("dynamodb", region_name="us-east-1") - client.create_table( - TableName=table_name, - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], - BillingMode="PAY_PER_REQUEST", - ) - return client - - -@mock_dynamodb2 -def test_sorted_query_with_numerical_sort_key(): - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - dynamodb.create_table( - TableName="CarCollection", - KeySchema=[ - {"AttributeName": "CarModel", "KeyType": "HASH"}, - {"AttributeName": "CarPrice", "KeyType": "RANGE"}, - ], - AttributeDefinitions=[ - {"AttributeName": "CarModel", "AttributeType": "S"}, - {"AttributeName": "CarPrice", "AttributeType": "N"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, - ) - - def create_item(price): - return {"CarModel": "M", "CarPrice": price} - - table = dynamodb.Table("CarCollection") - items = list(map(create_item, [2, 1, 10, 3])) - for item in items: - table.put_item(Item=item) - - response = table.query(KeyConditionExpression=Key("CarModel").eq("M")) - - response_items = response["Items"] - assert len(items) == len(response_items) - assert all(isinstance(item["CarPrice"], Decimal) for item in response_items) - response_prices = [item["CarPrice"] for item in response_items] - expected_prices = [Decimal(item["CarPrice"]) for item in items] - expected_prices.sort() - assert ( - expected_prices == response_prices - ), "result items are not sorted by numerical value" - - -# https://github.com/spulec/moto/issues/1874 -@mock_dynamodb2 -def test_item_size_is_under_400KB(): - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - client = boto3.client("dynamodb", region_name="us-east-1") - - dynamodb.create_table( - TableName="moto-test", - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, - ) - table = dynamodb.Table("moto-test") - - large_item = "x" * 410 * 1000 - assert_failure_due_to_item_size( - func=client.put_item, - TableName="moto-test", - Item={"id": {"S": "foo"}, "cont": {"S": large_item}}, - ) - assert_failure_due_to_item_size( - func=table.put_item, Item={"id": "bar", "cont": large_item} - ) - assert_failure_due_to_item_size_to_update( - func=client.update_item, - TableName="moto-test", - Key={"id": {"S": "foo2"}}, - UpdateExpression="set cont=:Item", - ExpressionAttributeValues={":Item": {"S": large_item}}, - ) - # Assert op fails when updating a nested item - assert_failure_due_to_item_size( - func=table.put_item, Item={"id": "bar", "itemlist": [{"cont": large_item}]} - ) - assert_failure_due_to_item_size( - func=client.put_item, - TableName="moto-test", - Item={ - "id": {"S": "foo"}, - "itemlist": {"L": [{"M": {"item1": {"S": large_item}}}]}, - }, - ) - - -def assert_failure_due_to_item_size(func, **kwargs): - with pytest.raises(ClientError) as ex: - func(**kwargs) - ex.value.response["Error"]["Code"].should.equal("ValidationException") - ex.value.response["Error"]["Message"].should.equal( - "Item size has exceeded the maximum allowed size" - ) - - -def assert_failure_due_to_item_size_to_update(func, **kwargs): - with pytest.raises(ClientError) as ex: - func(**kwargs) - ex.value.response["Error"]["Code"].should.equal("ValidationException") - ex.value.response["Error"]["Message"].should.equal( - "Item size to update has exceeded the maximum allowed size" - ) - - -@mock_dynamodb2 -def test_update_supports_complex_expression_attribute_values(): - client = boto3.client("dynamodb", region_name="us-east-1") - - client.create_table( - AttributeDefinitions=[{"AttributeName": "SHA256", "AttributeType": "S"}], - TableName="TestTable", - KeySchema=[{"AttributeName": "SHA256", "KeyType": "HASH"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - - client.update_item( - TableName="TestTable", - Key={"SHA256": {"S": "sha-of-file"}}, - UpdateExpression=( - "SET MD5 = :md5," "MyStringSet = :string_set," "MyMap = :map" - ), - ExpressionAttributeValues={ - ":md5": {"S": "md5-of-file"}, - ":string_set": {"SS": ["string1", "string2"]}, - ":map": {"M": {"EntryKey": {"SS": ["thing1", "thing2"]}}}, - }, - ) - result = client.get_item( - TableName="TestTable", Key={"SHA256": {"S": "sha-of-file"}} - )["Item"] - result.should.equal( - { - "MyStringSet": {"SS": ["string1", "string2"]}, - "MyMap": {"M": {"EntryKey": {"SS": ["thing1", "thing2"]}}}, - "SHA256": {"S": "sha-of-file"}, - "MD5": {"S": "md5-of-file"}, - } - ) - - -@mock_dynamodb2 -def test_update_supports_list_append(): - # Verify whether the list_append operation works as expected - client = boto3.client("dynamodb", region_name="us-east-1") - - client.create_table( - AttributeDefinitions=[{"AttributeName": "SHA256", "AttributeType": "S"}], - TableName="TestTable", - KeySchema=[{"AttributeName": "SHA256", "KeyType": "HASH"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - client.put_item( - TableName="TestTable", - Item={"SHA256": {"S": "sha-of-file"}, "crontab": {"L": [{"S": "bar1"}]}}, - ) - - # Update item using list_append expression - updated_item = client.update_item( - TableName="TestTable", - Key={"SHA256": {"S": "sha-of-file"}}, - UpdateExpression="SET crontab = list_append(crontab, :i)", - ExpressionAttributeValues={":i": {"L": [{"S": "bar2"}]}}, - ReturnValues="UPDATED_NEW", - ) - - # Verify updated item is correct - updated_item["Attributes"].should.equal( - {"crontab": {"L": [{"S": "bar1"}, {"S": "bar2"}]}} - ) - # Verify item is appended to the existing list - result = client.get_item( - TableName="TestTable", Key={"SHA256": {"S": "sha-of-file"}} - )["Item"] - result.should.equal( - { - "SHA256": {"S": "sha-of-file"}, - "crontab": {"L": [{"S": "bar1"}, {"S": "bar2"}]}, - } - ) - - -@mock_dynamodb2 -def test_update_supports_nested_list_append(): - # Verify whether we can append a list that's inside a map - client = boto3.client("dynamodb", region_name="us-east-1") - - client.create_table( - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], - TableName="TestTable", - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - client.put_item( - TableName="TestTable", - Item={ - "id": {"S": "nested_list_append"}, - "a": {"M": {"b": {"L": [{"S": "bar1"}]}}}, - }, - ) - - # Update item using list_append expression - updated_item = client.update_item( - TableName="TestTable", - Key={"id": {"S": "nested_list_append"}}, - UpdateExpression="SET a.#b = list_append(a.#b, :i)", - ExpressionAttributeValues={":i": {"L": [{"S": "bar2"}]}}, - ExpressionAttributeNames={"#b": "b"}, - ReturnValues="UPDATED_NEW", - ) - - # Verify updated item is correct - updated_item["Attributes"].should.equal( - {"a": {"M": {"b": {"L": [{"S": "bar1"}, {"S": "bar2"}]}}}} - ) - result = client.get_item( - TableName="TestTable", Key={"id": {"S": "nested_list_append"}} - )["Item"] - result.should.equal( - { - "id": {"S": "nested_list_append"}, - "a": {"M": {"b": {"L": [{"S": "bar1"}, {"S": "bar2"}]}}}, - } - ) - - -@mock_dynamodb2 -def test_update_supports_multiple_levels_nested_list_append(): - # Verify whether we can append a list that's inside a map that's inside a map (Inception!) - client = boto3.client("dynamodb", region_name="us-east-1") - - client.create_table( - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], - TableName="TestTable", - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - client.put_item( - TableName="TestTable", - Item={ - "id": {"S": "nested_list_append"}, - "a": {"M": {"b": {"M": {"c": {"L": [{"S": "bar1"}]}}}}}, - }, - ) - - # Update item using list_append expression - updated_item = client.update_item( - TableName="TestTable", - Key={"id": {"S": "nested_list_append"}}, - UpdateExpression="SET a.#b.c = list_append(a.#b.#c, :i)", - ExpressionAttributeValues={":i": {"L": [{"S": "bar2"}]}}, - ExpressionAttributeNames={"#b": "b", "#c": "c"}, - ReturnValues="UPDATED_NEW", - ) - - # Verify updated item is correct - updated_item["Attributes"].should.equal( - {"a": {"M": {"b": {"M": {"c": {"L": [{"S": "bar1"}, {"S": "bar2"}]}}}}}} - ) - # Verify item is appended to the existing list - result = client.get_item( - TableName="TestTable", Key={"id": {"S": "nested_list_append"}} - )["Item"] - result.should.equal( - { - "id": {"S": "nested_list_append"}, - "a": {"M": {"b": {"M": {"c": {"L": [{"S": "bar1"}, {"S": "bar2"}]}}}}}, - } - ) - - -@mock_dynamodb2 -def test_update_supports_nested_list_append_onto_another_list(): - # Verify whether we can take the contents of one list, and use that to fill another list - # Note that the contents of the other list is completely overwritten - client = boto3.client("dynamodb", region_name="us-east-1") - - client.create_table( - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], - TableName="TestTable", - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - client.put_item( - TableName="TestTable", - Item={ - "id": {"S": "list_append_another"}, - "a": {"M": {"b": {"L": [{"S": "bar1"}]}, "c": {"L": [{"S": "car1"}]}}}, - }, - ) - - # Update item using list_append expression - updated_item = client.update_item( - TableName="TestTable", - Key={"id": {"S": "list_append_another"}}, - UpdateExpression="SET a.#c = list_append(a.#b, :i)", - ExpressionAttributeValues={":i": {"L": [{"S": "bar2"}]}}, - ExpressionAttributeNames={"#b": "b", "#c": "c"}, - ReturnValues="UPDATED_NEW", - ) - - # Verify updated item is correct - updated_item["Attributes"].should.equal( - {"a": {"M": {"c": {"L": [{"S": "bar1"}, {"S": "bar2"}]}}}} - ) - # Verify item is appended to the existing list - result = client.get_item( - TableName="TestTable", Key={"id": {"S": "list_append_another"}} - )["Item"] - result.should.equal( - { - "id": {"S": "list_append_another"}, - "a": { - "M": { - "b": {"L": [{"S": "bar1"}]}, - "c": {"L": [{"S": "bar1"}, {"S": "bar2"}]}, - } - }, - } - ) - - -@mock_dynamodb2 -def test_update_supports_list_append_maps(): - client = boto3.client("dynamodb", region_name="us-west-1") - client.create_table( - AttributeDefinitions=[ - {"AttributeName": "id", "AttributeType": "S"}, - {"AttributeName": "rid", "AttributeType": "S"}, - ], - TableName="TestTable", - KeySchema=[ - {"AttributeName": "id", "KeyType": "HASH"}, - {"AttributeName": "rid", "KeyType": "RANGE"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - client.put_item( - TableName="TestTable", - Item={ - "id": {"S": "nested_list_append"}, - "rid": {"S": "range_key"}, - "a": {"L": [{"M": {"b": {"S": "bar1"}}}]}, - }, - ) - - # Update item using list_append expression - updated_item = client.update_item( - TableName="TestTable", - Key={"id": {"S": "nested_list_append"}, "rid": {"S": "range_key"}}, - UpdateExpression="SET a = list_append(a, :i)", - ExpressionAttributeValues={":i": {"L": [{"M": {"b": {"S": "bar2"}}}]}}, - ReturnValues="UPDATED_NEW", - ) - - # Verify updated item is correct - updated_item["Attributes"].should.equal( - {"a": {"L": [{"M": {"b": {"S": "bar1"}}}, {"M": {"b": {"S": "bar2"}}}]}} - ) - # Verify item is appended to the existing list - result = client.query( - TableName="TestTable", - KeyConditionExpression="id = :i AND begins_with(rid, :r)", - ExpressionAttributeValues={ - ":i": {"S": "nested_list_append"}, - ":r": {"S": "range_key"}, - }, - )["Items"] - result.should.equal( - [ - { - "a": {"L": [{"M": {"b": {"S": "bar1"}}}, {"M": {"b": {"S": "bar2"}}}]}, - "rid": {"S": "range_key"}, - "id": {"S": "nested_list_append"}, - } - ] - ) - - -@mock_dynamodb2 -def test_update_supports_nested_update_if_nested_value_not_exists(): - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - name = "TestTable" - - dynamodb.create_table( - TableName=name, - KeySchema=[{"AttributeName": "user_id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "user_id", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - - table = dynamodb.Table(name) - table.put_item( - Item={"user_id": "1234", "friends": {"5678": {"name": "friend_5678"}},}, - ) - table.update_item( - Key={"user_id": "1234"}, - ExpressionAttributeNames={"#friends": "friends", "#friendid": "0000",}, - ExpressionAttributeValues={":friend": {"name": "friend_0000"},}, - UpdateExpression="SET #friends.#friendid = :friend", - ReturnValues="UPDATED_NEW", - ) - item = table.get_item(Key={"user_id": "1234"})["Item"] - assert item == { - "user_id": "1234", - "friends": {"5678": {"name": "friend_5678"}, "0000": {"name": "friend_0000"},}, - } - - -@mock_dynamodb2 -def test_update_supports_list_append_with_nested_if_not_exists_operation(): - dynamo = boto3.resource("dynamodb", region_name="us-west-1") - table_name = "test" - - dynamo.create_table( - TableName=table_name, - AttributeDefinitions=[{"AttributeName": "Id", "AttributeType": "S"}], - KeySchema=[{"AttributeName": "Id", "KeyType": "HASH"}], - ProvisionedThroughput={"ReadCapacityUnits": 20, "WriteCapacityUnits": 20}, - ) - - table = dynamo.Table(table_name) - - table.put_item(Item={"Id": "item-id", "nest1": {"nest2": {}}}) - updated_item = table.update_item( - Key={"Id": "item-id"}, - UpdateExpression="SET nest1.nest2.event_history = list_append(if_not_exists(nest1.nest2.event_history, :empty_list), :new_value)", - ExpressionAttributeValues={":empty_list": [], ":new_value": ["some_value"]}, - ReturnValues="UPDATED_NEW", - ) - - # Verify updated item is correct - updated_item["Attributes"].should.equal( - {"nest1": {"nest2": {"event_history": ["some_value"]}}} - ) - - table.get_item(Key={"Id": "item-id"})["Item"].should.equal( - {"Id": "item-id", "nest1": {"nest2": {"event_history": ["some_value"]}}} - ) - - -@mock_dynamodb2 -def test_update_supports_list_append_with_nested_if_not_exists_operation_and_property_already_exists(): - dynamo = boto3.resource("dynamodb", region_name="us-west-1") - table_name = "test" - - dynamo.create_table( - TableName=table_name, - AttributeDefinitions=[{"AttributeName": "Id", "AttributeType": "S"}], - KeySchema=[{"AttributeName": "Id", "KeyType": "HASH"}], - ProvisionedThroughput={"ReadCapacityUnits": 20, "WriteCapacityUnits": 20}, - ) - - table = dynamo.Table(table_name) - - table.put_item(Item={"Id": "item-id", "event_history": ["other_value"]}) - updated_item = table.update_item( - Key={"Id": "item-id"}, - UpdateExpression="SET event_history = list_append(if_not_exists(event_history, :empty_list), :new_value)", - ExpressionAttributeValues={":empty_list": [], ":new_value": ["some_value"]}, - ReturnValues="UPDATED_NEW", - ) - - # Verify updated item is correct - updated_item["Attributes"].should.equal( - {"event_history": ["other_value", "some_value"]} - ) - - table.get_item(Key={"Id": "item-id"})["Item"].should.equal( - {"Id": "item-id", "event_history": ["other_value", "some_value"]} - ) - - -def _create_user_table(): - client = boto3.client("dynamodb", region_name="us-east-1") - client.create_table( - TableName="users", - KeySchema=[{"AttributeName": "username", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "username", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - client.put_item( - TableName="users", Item={"username": {"S": "user1"}, "foo": {"S": "bar"}} - ) - client.put_item( - TableName="users", Item={"username": {"S": "user2"}, "foo": {"S": "bar"}} - ) - client.put_item( - TableName="users", Item={"username": {"S": "user3"}, "foo": {"S": "bar"}} - ) - return client - - -@mock_dynamodb2 -def test_update_item_if_original_value_is_none(): - dynamo = boto3.resource("dynamodb", region_name="eu-central-1") - dynamo.create_table( - AttributeDefinitions=[{"AttributeName": "job_id", "AttributeType": "S"}], - TableName="origin-rbu-dev", - KeySchema=[{"AttributeName": "job_id", "KeyType": "HASH"}], - ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, - ) - table = dynamo.Table("origin-rbu-dev") - table.put_item(Item={"job_id": "a", "job_name": None}) - table.update_item( - Key={"job_id": "a"}, - UpdateExpression="SET job_name = :output", - ExpressionAttributeValues={":output": "updated"}, - ) - table.scan()["Items"][0]["job_name"].should.equal("updated") - - -@mock_dynamodb2 -def test_update_nested_item_if_original_value_is_none(): - dynamo = boto3.resource("dynamodb", region_name="eu-central-1") - dynamo.create_table( - AttributeDefinitions=[{"AttributeName": "job_id", "AttributeType": "S"}], - TableName="origin-rbu-dev", - KeySchema=[{"AttributeName": "job_id", "KeyType": "HASH"}], - ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, - ) - table = dynamo.Table("origin-rbu-dev") - table.put_item(Item={"job_id": "a", "job_details": {"job_name": None}}) - updated_item = table.update_item( - Key={"job_id": "a"}, - UpdateExpression="SET job_details.job_name = :output", - ExpressionAttributeValues={":output": "updated"}, - ReturnValues="UPDATED_NEW", - ) - - # Verify updated item is correct - updated_item["Attributes"].should.equal({"job_details": {"job_name": "updated"}}) - - table.scan()["Items"][0]["job_details"]["job_name"].should.equal("updated") - - -@mock_dynamodb2 -def test_allow_update_to_item_with_different_type(): - dynamo = boto3.resource("dynamodb", region_name="eu-central-1") - dynamo.create_table( - AttributeDefinitions=[{"AttributeName": "job_id", "AttributeType": "S"}], - TableName="origin-rbu-dev", - KeySchema=[{"AttributeName": "job_id", "KeyType": "HASH"}], - ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, - ) - table = dynamo.Table("origin-rbu-dev") - table.put_item(Item={"job_id": "a", "job_details": {"job_name": {"nested": "yes"}}}) - table.put_item(Item={"job_id": "b", "job_details": {"job_name": {"nested": "yes"}}}) - updated_item = table.update_item( - Key={"job_id": "a"}, - UpdateExpression="SET job_details.job_name = :output", - ExpressionAttributeValues={":output": "updated"}, - ReturnValues="UPDATED_NEW", - ) - - # Verify updated item is correct - updated_item["Attributes"].should.equal({"job_details": {"job_name": "updated"}}) - - table.get_item(Key={"job_id": "a"})["Item"]["job_details"][ - "job_name" - ].should.be.equal("updated") - table.get_item(Key={"job_id": "b"})["Item"]["job_details"][ - "job_name" - ].should.be.equal({"nested": "yes"}) - - -@mock_dynamodb2 -def test_query_catches_when_no_filters(): - dynamo = boto3.resource("dynamodb", region_name="eu-central-1") - dynamo.create_table( - AttributeDefinitions=[{"AttributeName": "job_id", "AttributeType": "S"}], - TableName="origin-rbu-dev", - KeySchema=[{"AttributeName": "job_id", "KeyType": "HASH"}], - ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, - ) - table = dynamo.Table("origin-rbu-dev") - - with pytest.raises(ClientError) as ex: - table.query(TableName="original-rbu-dev") - - ex.value.response["Error"]["Code"].should.equal("ValidationException") - ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) - ex.value.response["Error"]["Message"].should.equal( - "Either KeyConditions or QueryFilter should be present" - ) - - -@mock_dynamodb2 -def test_invalid_transact_get_items(): - - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - dynamodb.create_table( - TableName="test1", - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - table = dynamodb.Table("test1") - table.put_item( - Item={"id": "1", "val": "1",} - ) - - table.put_item( - Item={"id": "1", "val": "2",} - ) - - client = boto3.client("dynamodb", region_name="us-east-1") - - with pytest.raises(ClientError) as ex: - client.transact_get_items( - TransactItems=[ - {"Get": {"Key": {"id": {"S": "1"}}, "TableName": "test1"}} - for i in range(26) - ] - ) - - ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) - ex.value.response["Error"]["Message"].should.match( - r"failed to satisfy constraint: Member must have length less than or equal to 25", - re.I, - ) - - with pytest.raises(ClientError) as ex: - client.transact_get_items( - TransactItems=[ - {"Get": {"Key": {"id": {"S": "1"},}, "TableName": "test1",}}, - {"Get": {"Key": {"id": {"S": "1"},}, "TableName": "non_exists_table",}}, - ] - ) - - ex.value.response["Error"]["Code"].should.equal("ResourceNotFoundException") - ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) - ex.value.response["Error"]["Message"].should.equal("Requested resource not found") - - -@mock_dynamodb2 -def test_valid_transact_get_items(): - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - dynamodb.create_table( - TableName="test1", - KeySchema=[ - {"AttributeName": "id", "KeyType": "HASH"}, - {"AttributeName": "sort_key", "KeyType": "RANGE"}, - ], - AttributeDefinitions=[ - {"AttributeName": "id", "AttributeType": "S"}, - {"AttributeName": "sort_key", "AttributeType": "S"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - table1 = dynamodb.Table("test1") - table1.put_item( - Item={"id": "1", "sort_key": "1",} - ) - - table1.put_item( - Item={"id": "1", "sort_key": "2",} - ) - - dynamodb.create_table( - TableName="test2", - KeySchema=[ - {"AttributeName": "id", "KeyType": "HASH"}, - {"AttributeName": "sort_key", "KeyType": "RANGE"}, - ], - AttributeDefinitions=[ - {"AttributeName": "id", "AttributeType": "S"}, - {"AttributeName": "sort_key", "AttributeType": "S"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - table2 = dynamodb.Table("test2") - table2.put_item( - Item={"id": "1", "sort_key": "1",} - ) - - client = boto3.client("dynamodb", region_name="us-east-1") - res = client.transact_get_items( - TransactItems=[ - { - "Get": { - "Key": {"id": {"S": "1"}, "sort_key": {"S": "1"}}, - "TableName": "test1", - } - }, - { - "Get": { - "Key": {"id": {"S": "non_exists_key"}, "sort_key": {"S": "2"}}, - "TableName": "test1", - } - }, - ] - ) - res["Responses"][0]["Item"].should.equal({"id": {"S": "1"}, "sort_key": {"S": "1"}}) - len(res["Responses"]).should.equal(2) - res["Responses"][1].should.equal({}) - - res = client.transact_get_items( - TransactItems=[ - { - "Get": { - "Key": {"id": {"S": "1"}, "sort_key": {"S": "1"}}, - "TableName": "test1", - } - }, - { - "Get": { - "Key": {"id": {"S": "1"}, "sort_key": {"S": "2"}}, - "TableName": "test1", - } - }, - { - "Get": { - "Key": {"id": {"S": "1"}, "sort_key": {"S": "1"}}, - "TableName": "test2", - } - }, - ] - ) - - res["Responses"][0]["Item"].should.equal({"id": {"S": "1"}, "sort_key": {"S": "1"}}) - - res["Responses"][1]["Item"].should.equal({"id": {"S": "1"}, "sort_key": {"S": "2"}}) - - res["Responses"][2]["Item"].should.equal({"id": {"S": "1"}, "sort_key": {"S": "1"}}) - - res = client.transact_get_items( - TransactItems=[ - { - "Get": { - "Key": {"id": {"S": "1"}, "sort_key": {"S": "1"}}, - "TableName": "test1", - } - }, - { - "Get": { - "Key": {"id": {"S": "1"}, "sort_key": {"S": "2"}}, - "TableName": "test1", - } - }, - { - "Get": { - "Key": {"id": {"S": "1"}, "sort_key": {"S": "1"}}, - "TableName": "test2", - } - }, - ], - ReturnConsumedCapacity="TOTAL", - ) - - res["ConsumedCapacity"][0].should.equal( - {"TableName": "test1", "CapacityUnits": 4.0, "ReadCapacityUnits": 4.0} - ) - - res["ConsumedCapacity"][1].should.equal( - {"TableName": "test2", "CapacityUnits": 2.0, "ReadCapacityUnits": 2.0} - ) - - res = client.transact_get_items( - TransactItems=[ - { - "Get": { - "Key": {"id": {"S": "1"}, "sort_key": {"S": "1"}}, - "TableName": "test1", - } - }, - { - "Get": { - "Key": {"id": {"S": "1"}, "sort_key": {"S": "2"}}, - "TableName": "test1", - } - }, - { - "Get": { - "Key": {"id": {"S": "1"}, "sort_key": {"S": "1"}}, - "TableName": "test2", - } - }, - ], - ReturnConsumedCapacity="INDEXES", - ) - - res["ConsumedCapacity"][0].should.equal( - { - "TableName": "test1", - "CapacityUnits": 4.0, - "ReadCapacityUnits": 4.0, - "Table": {"CapacityUnits": 4.0, "ReadCapacityUnits": 4.0,}, - } - ) - - res["ConsumedCapacity"][1].should.equal( - { - "TableName": "test2", - "CapacityUnits": 2.0, - "ReadCapacityUnits": 2.0, - "Table": {"CapacityUnits": 2.0, "ReadCapacityUnits": 2.0,}, - } - ) - - -@mock_dynamodb2 -def test_gsi_verify_negative_number_order(): - table_schema = { - "KeySchema": [{"AttributeName": "partitionKey", "KeyType": "HASH"}], - "GlobalSecondaryIndexes": [ - { - "IndexName": "GSI-K1", - "KeySchema": [ - {"AttributeName": "gsiK1PartitionKey", "KeyType": "HASH"}, - {"AttributeName": "gsiK1SortKey", "KeyType": "RANGE"}, - ], - "Projection": {"ProjectionType": "KEYS_ONLY",}, - } - ], - "AttributeDefinitions": [ - {"AttributeName": "partitionKey", "AttributeType": "S"}, - {"AttributeName": "gsiK1PartitionKey", "AttributeType": "S"}, - {"AttributeName": "gsiK1SortKey", "AttributeType": "N"}, - ], - } - - item1 = { - "partitionKey": "pk-1", - "gsiK1PartitionKey": "gsi-k1", - "gsiK1SortKey": Decimal("-0.6"), - } - - item2 = { - "partitionKey": "pk-2", - "gsiK1PartitionKey": "gsi-k1", - "gsiK1SortKey": Decimal("-0.7"), - } - - item3 = { - "partitionKey": "pk-3", - "gsiK1PartitionKey": "gsi-k1", - "gsiK1SortKey": Decimal("0.7"), - } - - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - dynamodb.create_table( - TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema - ) - table = dynamodb.Table("test-table") - table.put_item(Item=item3) - table.put_item(Item=item1) - table.put_item(Item=item2) - - resp = table.query( - KeyConditionExpression=Key("gsiK1PartitionKey").eq("gsi-k1"), - IndexName="GSI-K1", - ) - # Items should be ordered with the lowest number first - [float(item["gsiK1SortKey"]) for item in resp["Items"]].should.equal( - [-0.7, -0.6, 0.7] - ) - - -@mock_dynamodb2 -def test_transact_write_items_put(): - table_schema = { - "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], - "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], - } - dynamodb = boto3.client("dynamodb", region_name="us-east-1") - dynamodb.create_table( - TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema - ) - # Put multiple items - dynamodb.transact_write_items( - TransactItems=[ - { - "Put": { - "Item": {"id": {"S": "foo{}".format(str(i))}, "foo": {"S": "bar"},}, - "TableName": "test-table", - } - } - for i in range(0, 5) - ] - ) - # Assert all are present - items = dynamodb.scan(TableName="test-table")["Items"] - items.should.have.length_of(5) - - -@mock_dynamodb2 -def test_transact_write_items_put_conditional_expressions(): - table_schema = { - "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], - "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], - } - dynamodb = boto3.client("dynamodb", region_name="us-east-1") - dynamodb.create_table( - TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema - ) - dynamodb.put_item( - TableName="test-table", Item={"id": {"S": "foo2"},}, - ) - # Put multiple items - with pytest.raises(ClientError) as ex: - dynamodb.transact_write_items( - TransactItems=[ - { - "Put": { - "Item": { - "id": {"S": "foo{}".format(str(i))}, - "foo": {"S": "bar"}, - }, - "TableName": "test-table", - "ConditionExpression": "#i <> :i", - "ExpressionAttributeNames": {"#i": "id"}, - "ExpressionAttributeValues": { - ":i": { - "S": "foo2" - } # This item already exist, so the ConditionExpression should fail - }, - } - } - for i in range(0, 5) - ] - ) - # Assert the exception is correct - ex.value.response["Error"]["Code"].should.equal("TransactionCanceledException") - ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) - # Assert all are present - items = dynamodb.scan(TableName="test-table")["Items"] - items.should.have.length_of(1) - items[0].should.equal({"id": {"S": "foo2"}}) - - -@mock_dynamodb2 -def test_transact_write_items_conditioncheck_passes(): - table_schema = { - "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], - "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], - } - dynamodb = boto3.client("dynamodb", region_name="us-east-1") - dynamodb.create_table( - TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema - ) - # Insert an item without email address - dynamodb.put_item( - TableName="test-table", Item={"id": {"S": "foo"},}, - ) - # Put an email address, after verifying it doesn't exist yet - dynamodb.transact_write_items( - TransactItems=[ - { - "ConditionCheck": { - "Key": {"id": {"S": "foo"}}, - "TableName": "test-table", - "ConditionExpression": "attribute_not_exists(#e)", - "ExpressionAttributeNames": {"#e": "email_address"}, - } - }, - { - "Put": { - "Item": { - "id": {"S": "foo"}, - "email_address": {"S": "test@moto.com"}, - }, - "TableName": "test-table", - } - }, - ] - ) - # Assert all are present - items = dynamodb.scan(TableName="test-table")["Items"] - items.should.have.length_of(1) - items[0].should.equal({"email_address": {"S": "test@moto.com"}, "id": {"S": "foo"}}) - - -@mock_dynamodb2 -def test_transact_write_items_conditioncheck_fails(): - table_schema = { - "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], - "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], - } - dynamodb = boto3.client("dynamodb", region_name="us-east-1") - dynamodb.create_table( - TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema - ) - # Insert an item with email address - dynamodb.put_item( - TableName="test-table", - Item={"id": {"S": "foo"}, "email_address": {"S": "test@moto.com"}}, - ) - # Try to put an email address, but verify whether it exists - # ConditionCheck should fail - with pytest.raises(ClientError) as ex: - dynamodb.transact_write_items( - TransactItems=[ - { - "ConditionCheck": { - "Key": {"id": {"S": "foo"}}, - "TableName": "test-table", - "ConditionExpression": "attribute_not_exists(#e)", - "ExpressionAttributeNames": {"#e": "email_address"}, - } - }, - { - "Put": { - "Item": { - "id": {"S": "foo"}, - "email_address": {"S": "update@moto.com"}, - }, - "TableName": "test-table", - } - }, - ] - ) - # Assert the exception is correct - ex.value.response["Error"]["Code"].should.equal("TransactionCanceledException") - ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) - - # Assert the original email address is still present - items = dynamodb.scan(TableName="test-table")["Items"] - items.should.have.length_of(1) - items[0].should.equal({"email_address": {"S": "test@moto.com"}, "id": {"S": "foo"}}) - - -@mock_dynamodb2 -def test_transact_write_items_delete(): - table_schema = { - "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], - "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], - } - dynamodb = boto3.client("dynamodb", region_name="us-east-1") - dynamodb.create_table( - TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema - ) - # Insert an item - dynamodb.put_item( - TableName="test-table", Item={"id": {"S": "foo"},}, - ) - # Delete the item - dynamodb.transact_write_items( - TransactItems=[ - {"Delete": {"Key": {"id": {"S": "foo"}}, "TableName": "test-table",}} - ] - ) - # Assert the item is deleted - items = dynamodb.scan(TableName="test-table")["Items"] - items.should.have.length_of(0) - - -@mock_dynamodb2 -def test_transact_write_items_delete_with_successful_condition_expression(): - table_schema = { - "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], - "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], - } - dynamodb = boto3.client("dynamodb", region_name="us-east-1") - dynamodb.create_table( - TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema - ) - # Insert an item without email address - dynamodb.put_item( - TableName="test-table", Item={"id": {"S": "foo"},}, - ) - # ConditionExpression will pass - no email address has been specified yet - dynamodb.transact_write_items( - TransactItems=[ - { - "Delete": { - "Key": {"id": {"S": "foo"},}, - "TableName": "test-table", - "ConditionExpression": "attribute_not_exists(#e)", - "ExpressionAttributeNames": {"#e": "email_address"}, - } - } - ] - ) - # Assert the item is deleted - items = dynamodb.scan(TableName="test-table")["Items"] - items.should.have.length_of(0) - - -@mock_dynamodb2 -def test_transact_write_items_delete_with_failed_condition_expression(): - table_schema = { - "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], - "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], - } - dynamodb = boto3.client("dynamodb", region_name="us-east-1") - dynamodb.create_table( - TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema - ) - # Insert an item with email address - dynamodb.put_item( - TableName="test-table", - Item={"id": {"S": "foo"}, "email_address": {"S": "test@moto.com"}}, - ) - # Try to delete an item that does not have an email address - # ConditionCheck should fail - with pytest.raises(ClientError) as ex: - dynamodb.transact_write_items( - TransactItems=[ - { - "Delete": { - "Key": {"id": {"S": "foo"},}, - "TableName": "test-table", - "ConditionExpression": "attribute_not_exists(#e)", - "ExpressionAttributeNames": {"#e": "email_address"}, - } - } - ] - ) - # Assert the exception is correct - ex.value.response["Error"]["Code"].should.equal("TransactionCanceledException") - ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) - # Assert the original item is still present - items = dynamodb.scan(TableName="test-table")["Items"] - items.should.have.length_of(1) - items[0].should.equal({"email_address": {"S": "test@moto.com"}, "id": {"S": "foo"}}) - - -@mock_dynamodb2 -def test_transact_write_items_update(): - table_schema = { - "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], - "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], - } - dynamodb = boto3.client("dynamodb", region_name="us-east-1") - dynamodb.create_table( - TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema - ) - # Insert an item - dynamodb.put_item(TableName="test-table", Item={"id": {"S": "foo"}}) - # Update the item - dynamodb.transact_write_items( - TransactItems=[ - { - "Update": { - "Key": {"id": {"S": "foo"}}, - "TableName": "test-table", - "UpdateExpression": "SET #e = :v", - "ExpressionAttributeNames": {"#e": "email_address"}, - "ExpressionAttributeValues": {":v": {"S": "test@moto.com"}}, - } - } - ] - ) - # Assert the item is updated - items = dynamodb.scan(TableName="test-table")["Items"] - items.should.have.length_of(1) - items[0].should.equal({"id": {"S": "foo"}, "email_address": {"S": "test@moto.com"}}) - - -@mock_dynamodb2 -def test_transact_write_items_update_with_failed_condition_expression(): - table_schema = { - "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], - "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], - } - dynamodb = boto3.client("dynamodb", region_name="us-east-1") - dynamodb.create_table( - TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema - ) - # Insert an item with email address - dynamodb.put_item( - TableName="test-table", - Item={"id": {"S": "foo"}, "email_address": {"S": "test@moto.com"}}, - ) - # Try to update an item that does not have an email address - # ConditionCheck should fail - with pytest.raises(ClientError) as ex: - dynamodb.transact_write_items( - TransactItems=[ - { - "Update": { - "Key": {"id": {"S": "foo"}}, - "TableName": "test-table", - "UpdateExpression": "SET #e = :v", - "ConditionExpression": "attribute_not_exists(#e)", - "ExpressionAttributeNames": {"#e": "email_address"}, - "ExpressionAttributeValues": {":v": {"S": "update@moto.com"}}, - } - } - ] - ) - # Assert the exception is correct - ex.value.response["Error"]["Code"].should.equal("TransactionCanceledException") - ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) - # Assert the original item is still present - items = dynamodb.scan(TableName="test-table")["Items"] - items.should.have.length_of(1) - items[0].should.equal({"email_address": {"S": "test@moto.com"}, "id": {"S": "foo"}}) - - -@mock_dynamodb2 -def test_dynamodb_max_1mb_limit(): - ddb = boto3.resource("dynamodb", region_name="eu-west-1") - - table_name = "populated-mock-table" - table = ddb.create_table( - TableName=table_name, - KeySchema=[ - {"AttributeName": "partition_key", "KeyType": "HASH"}, - {"AttributeName": "sort_key", "KeyType": "RANGE"}, - ], - AttributeDefinitions=[ - {"AttributeName": "partition_key", "AttributeType": "S"}, - {"AttributeName": "sort_key", "AttributeType": "S"}, - ], - BillingMode="PAY_PER_REQUEST", - ) - - # Populate the table - items = [ - { - "partition_key": "partition_key_val", # size=30 - "sort_key": "sort_key_value____" + str(i), # size=30 - } - for i in range(10000, 29999) - ] - with table.batch_writer() as batch: - for item in items: - batch.put_item(Item=item) - - response = table.query( - KeyConditionExpression=Key("partition_key").eq("partition_key_val") - ) - # We shouldn't get everything back - the total result set is well over 1MB - len(items).should.be.greater_than(response["Count"]) - response["LastEvaluatedKey"].shouldnt.be(None) - - -def assert_raise_syntax_error(client_error, token, near): - """ - Assert whether a client_error is as expected Syntax error. Syntax error looks like: `syntax_error_template` - - Args: - client_error(ClientError): The ClientError exception that was raised - token(str): The token that ws unexpected - near(str): The part in the expression that shows where the error occurs it generally has the preceding token the - optional separation and the problematic token. - """ - syntax_error_template = ( - 'Invalid UpdateExpression: Syntax error; token: "{token}", near: "{near}"' - ) - expected_syntax_error = syntax_error_template.format(token=token, near=near) - assert client_error.response["Error"]["Code"] == "ValidationException" - assert expected_syntax_error == client_error.response["Error"]["Message"] - - -@mock_dynamodb2 -def test_update_expression_with_numeric_literal_instead_of_value(): - """ - DynamoDB requires literals to be passed in as values. If they are put literally in the expression a token error will - be raised - """ - dynamodb = boto3.client("dynamodb", region_name="eu-west-1") - - dynamodb.create_table( - TableName="moto-test", - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], - BillingMode="PAY_PER_REQUEST", - ) - - try: - dynamodb.update_item( - TableName="moto-test", - Key={"id": {"S": "1"}}, - UpdateExpression="SET MyStr = myNum + 1", - ) - assert False, "Validation exception not thrown" - except dynamodb.exceptions.ClientError as e: - assert_raise_syntax_error(e, "1", "+ 1") - - -@mock_dynamodb2 -def test_update_expression_with_multiple_set_clauses_must_be_comma_separated(): - """ - An UpdateExpression can have multiple set clauses but if they are passed in without the separating comma. - """ - dynamodb = boto3.client("dynamodb", region_name="eu-west-1") - - dynamodb.create_table( - TableName="moto-test", - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], - BillingMode="PAY_PER_REQUEST", - ) - - try: - dynamodb.update_item( - TableName="moto-test", - Key={"id": {"S": "1"}}, - UpdateExpression="SET MyStr = myNum Mystr2 myNum2", - ) - assert False, "Validation exception not thrown" - except dynamodb.exceptions.ClientError as e: - assert_raise_syntax_error(e, "Mystr2", "myNum Mystr2 myNum2") - - -@mock_dynamodb2 -def test_list_tables_exclusive_start_table_name_empty(): - client = boto3.client("dynamodb", region_name="us-east-1") - - resp = client.list_tables(Limit=1, ExclusiveStartTableName="whatever") - - len(resp["TableNames"]).should.equal(0) - - -def assert_correct_client_error( - client_error, code, message_template, message_values=None, braces=None -): - """ - Assert whether a client_error is as expected. Allow for a list of values to be passed into the message - - Args: - client_error(ClientError): The ClientError exception that was raised - code(str): The code for the error (e.g. ValidationException) - message_template(str): Error message template. if message_values is not None then this template has a {values} - as placeholder. For example: - 'Value provided in ExpressionAttributeValues unused in expressions: keys: {values}' - message_values(list of str|None): The values that are passed in the error message - braces(list of str|None): List of length 2 with opening and closing brace for the values. By default it will be - surrounded by curly brackets - """ - braces = braces or ["{", "}"] - assert client_error.response["Error"]["Code"] == code - if message_values is not None: - values_string = "{open_brace}(?P.*){close_brace}".format( - open_brace=braces[0], close_brace=braces[1] - ) - re_msg = re.compile(message_template.format(values=values_string)) - match_result = re_msg.match(client_error.response["Error"]["Message"]) - assert match_result is not None - values_string = match_result.groupdict()["values"] - values = [key for key in values_string.split(", ")] - assert len(message_values) == len(values) - for value in message_values: - assert value in values - else: - assert client_error.response["Error"]["Message"] == message_template - - -def create_simple_table_and_return_client(): - dynamodb = boto3.client("dynamodb", region_name="eu-west-1") - dynamodb.create_table( - TableName="moto-test", - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"},], - ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, - ) - dynamodb.put_item( - TableName="moto-test", - Item={"id": {"S": "1"}, "myNum": {"N": "1"}, "MyStr": {"S": "1"},}, - ) - return dynamodb - - -# https://github.com/spulec/moto/issues/2806 -# https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_UpdateItem.html -# #DDB-UpdateItem-request-UpdateExpression -@mock_dynamodb2 -def test_update_item_with_attribute_in_right_hand_side_and_operation(): - dynamodb = create_simple_table_and_return_client() - - dynamodb.update_item( - TableName="moto-test", - Key={"id": {"S": "1"}}, - UpdateExpression="SET myNum = myNum+:val", - ExpressionAttributeValues={":val": {"N": "3"}}, - ) - - result = dynamodb.get_item(TableName="moto-test", Key={"id": {"S": "1"}}) - assert result["Item"]["myNum"]["N"] == "4" - - dynamodb.update_item( - TableName="moto-test", - Key={"id": {"S": "1"}}, - UpdateExpression="SET myNum = myNum - :val", - ExpressionAttributeValues={":val": {"N": "1"}}, - ) - result = dynamodb.get_item(TableName="moto-test", Key={"id": {"S": "1"}}) - assert result["Item"]["myNum"]["N"] == "3" - - -@mock_dynamodb2 -def test_non_existing_attribute_should_raise_exception(): - """ - Does error message get correctly raised if attribute is referenced but it does not exist for the item. - """ - dynamodb = create_simple_table_and_return_client() - - try: - dynamodb.update_item( - TableName="moto-test", - Key={"id": {"S": "1"}}, - UpdateExpression="SET MyStr = no_attr + MyStr", - ) - assert False, "Validation exception not thrown" - except dynamodb.exceptions.ClientError as e: - assert_correct_client_error( - e, - "ValidationException", - "The provided expression refers to an attribute that does not exist in the item", - ) - - -@mock_dynamodb2 -def test_update_expression_with_plus_in_attribute_name(): - """ - Does error message get correctly raised if attribute contains a plus and is passed in without an AttributeName. And - lhs & rhs are not attribute IDs by themselve. - """ - dynamodb = create_simple_table_and_return_client() - - dynamodb.put_item( - TableName="moto-test", - Item={"id": {"S": "1"}, "my+Num": {"S": "1"}, "MyStr": {"S": "aaa"},}, - ) - try: - dynamodb.update_item( - TableName="moto-test", - Key={"id": {"S": "1"}}, - UpdateExpression="SET MyStr = my+Num", - ) - assert False, "Validation exception not thrown" - except dynamodb.exceptions.ClientError as e: - assert_correct_client_error( - e, - "ValidationException", - "The provided expression refers to an attribute that does not exist in the item", - ) - - -@mock_dynamodb2 -def test_update_expression_with_minus_in_attribute_name(): - """ - Does error message get correctly raised if attribute contains a minus and is passed in without an AttributeName. And - lhs & rhs are not attribute IDs by themselve. - """ - dynamodb = create_simple_table_and_return_client() - - dynamodb.put_item( - TableName="moto-test", - Item={"id": {"S": "1"}, "my-Num": {"S": "1"}, "MyStr": {"S": "aaa"},}, - ) - try: - dynamodb.update_item( - TableName="moto-test", - Key={"id": {"S": "1"}}, - UpdateExpression="SET MyStr = my-Num", - ) - assert False, "Validation exception not thrown" - except dynamodb.exceptions.ClientError as e: - assert_correct_client_error( - e, - "ValidationException", - "The provided expression refers to an attribute that does not exist in the item", - ) - - -@mock_dynamodb2 -def test_update_expression_with_space_in_attribute_name(): - """ - Does error message get correctly raised if attribute contains a space and is passed in without an AttributeName. And - lhs & rhs are not attribute IDs by themselves. - """ - dynamodb = create_simple_table_and_return_client() - - dynamodb.put_item( - TableName="moto-test", - Item={"id": {"S": "1"}, "my Num": {"S": "1"}, "MyStr": {"S": "aaa"},}, - ) - - try: - dynamodb.update_item( - TableName="moto-test", - Key={"id": {"S": "1"}}, - UpdateExpression="SET MyStr = my Num", - ) - assert False, "Validation exception not thrown" - except dynamodb.exceptions.ClientError as e: - assert_raise_syntax_error(e, "Num", "my Num") - - -@mock_dynamodb2 -def test_summing_up_2_strings_raises_exception(): - """ - Update set supports different DynamoDB types but some operations are not supported. For example summing up 2 strings - raises an exception. It results in ClientError with code ValidationException: - Saying An operand in the update expression has an incorrect data type - """ - dynamodb = create_simple_table_and_return_client() - - try: - dynamodb.update_item( - TableName="moto-test", - Key={"id": {"S": "1"}}, - UpdateExpression="SET MyStr = MyStr + MyStr", - ) - assert False, "Validation exception not thrown" - except dynamodb.exceptions.ClientError as e: - assert_correct_client_error( - e, - "ValidationException", - "An operand in the update expression has an incorrect data type", - ) - - -# https://github.com/spulec/moto/issues/2806 -@mock_dynamodb2 -def test_update_item_with_attribute_in_right_hand_side(): - """ - After tokenization and building expression make sure referenced attributes are replaced with their current value - """ - dynamodb = create_simple_table_and_return_client() - - # Make sure there are 2 values - dynamodb.put_item( - TableName="moto-test", - Item={"id": {"S": "1"}, "myVal1": {"S": "Value1"}, "myVal2": {"S": "Value2"}}, - ) - - dynamodb.update_item( - TableName="moto-test", - Key={"id": {"S": "1"}}, - UpdateExpression="SET myVal1 = myVal2", - ) - - result = dynamodb.get_item(TableName="moto-test", Key={"id": {"S": "1"}}) - assert result["Item"]["myVal1"]["S"] == result["Item"]["myVal2"]["S"] == "Value2" - - -@mock_dynamodb2 -def test_multiple_updates(): - dynamodb = create_simple_table_and_return_client() - dynamodb.put_item( - TableName="moto-test", - Item={"id": {"S": "1"}, "myNum": {"N": "1"}, "path": {"N": "6"}}, - ) - dynamodb.update_item( - TableName="moto-test", - Key={"id": {"S": "1"}}, - UpdateExpression="SET myNum = #p + :val, newAttr = myNum", - ExpressionAttributeValues={":val": {"N": "1"}}, - ExpressionAttributeNames={"#p": "path"}, - ) - result = dynamodb.get_item(TableName="moto-test", Key={"id": {"S": "1"}})["Item"] - expected_result = { - "myNum": {"N": "7"}, - "newAttr": {"N": "1"}, - "path": {"N": "6"}, - "id": {"S": "1"}, - } - assert result == expected_result - - -@mock_dynamodb2 -def test_update_item_atomic_counter(): - table = "table_t" - ddb_mock = boto3.client("dynamodb", region_name="eu-west-3") - ddb_mock.create_table( - TableName=table, - KeySchema=[{"AttributeName": "t_id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "t_id", "AttributeType": "S"}], - BillingMode="PAY_PER_REQUEST", - ) - - key = {"t_id": {"S": "item1"}} - - ddb_mock.put_item( - TableName=table, - Item={"t_id": {"S": "item1"}, "n_i": {"N": "5"}, "n_f": {"N": "5.3"}}, - ) - - ddb_mock.update_item( - TableName=table, - Key=key, - UpdateExpression="set n_i = n_i + :inc1, n_f = n_f + :inc2", - ExpressionAttributeValues={":inc1": {"N": "1.2"}, ":inc2": {"N": "0.05"}}, - ) - updated_item = ddb_mock.get_item(TableName=table, Key=key)["Item"] - updated_item["n_i"]["N"].should.equal("6.2") - updated_item["n_f"]["N"].should.equal("5.35") - - -@mock_dynamodb2 -def test_update_item_atomic_counter_return_values(): - table = "table_t" - ddb_mock = boto3.client("dynamodb", region_name="eu-west-3") - ddb_mock.create_table( - TableName=table, - KeySchema=[{"AttributeName": "t_id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "t_id", "AttributeType": "S"}], - BillingMode="PAY_PER_REQUEST", - ) - - key = {"t_id": {"S": "item1"}} - - ddb_mock.put_item(TableName=table, Item={"t_id": {"S": "item1"}, "v": {"N": "5"}}) - - response = ddb_mock.update_item( - TableName=table, - Key=key, - UpdateExpression="set v = v + :inc", - ExpressionAttributeValues={":inc": {"N": "1"}}, - ReturnValues="UPDATED_OLD", - ) - assert ( - "v" in response["Attributes"] - ), "v has been updated, and should be returned here" - response["Attributes"]["v"]["N"].should.equal("5") - - # second update - response = ddb_mock.update_item( - TableName=table, - Key=key, - UpdateExpression="set v = v + :inc", - ExpressionAttributeValues={":inc": {"N": "1"}}, - ReturnValues="UPDATED_OLD", - ) - assert ( - "v" in response["Attributes"] - ), "v has been updated, and should be returned here" - response["Attributes"]["v"]["N"].should.equal("6") - - # third update - response = ddb_mock.update_item( - TableName=table, - Key=key, - UpdateExpression="set v = v + :inc", - ExpressionAttributeValues={":inc": {"N": "1"}}, - ReturnValues="UPDATED_NEW", - ) - assert ( - "v" in response["Attributes"] - ), "v has been updated, and should be returned here" - response["Attributes"]["v"]["N"].should.equal("8") - - -@mock_dynamodb2 -def test_update_item_atomic_counter_from_zero(): - table = "table_t" - ddb_mock = boto3.client("dynamodb", region_name="eu-west-1") - ddb_mock.create_table( - TableName=table, - KeySchema=[{"AttributeName": "t_id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "t_id", "AttributeType": "S"}], - BillingMode="PAY_PER_REQUEST", - ) - - key = {"t_id": {"S": "item1"}} - - ddb_mock.put_item( - TableName=table, Item=key, - ) - - ddb_mock.update_item( - TableName=table, - Key=key, - UpdateExpression="add n_i :inc1, n_f :inc2", - ExpressionAttributeValues={":inc1": {"N": "1.2"}, ":inc2": {"N": "-0.5"}}, - ) - updated_item = ddb_mock.get_item(TableName=table, Key=key)["Item"] - assert updated_item["n_i"]["N"] == "1.2" - assert updated_item["n_f"]["N"] == "-0.5" - - -@mock_dynamodb2 -def test_update_item_add_to_non_existent_set(): - table = "table_t" - ddb_mock = boto3.client("dynamodb", region_name="eu-west-1") - ddb_mock.create_table( - TableName=table, - KeySchema=[{"AttributeName": "t_id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "t_id", "AttributeType": "S"}], - BillingMode="PAY_PER_REQUEST", - ) - key = {"t_id": {"S": "item1"}} - ddb_mock.put_item( - TableName=table, Item=key, - ) - - ddb_mock.update_item( - TableName=table, - Key=key, - UpdateExpression="add s_i :s1", - ExpressionAttributeValues={":s1": {"SS": ["hello"]}}, - ) - updated_item = ddb_mock.get_item(TableName=table, Key=key)["Item"] - assert updated_item["s_i"]["SS"] == ["hello"] - - -@mock_dynamodb2 -def test_update_item_add_to_non_existent_number_set(): - table = "table_t" - ddb_mock = boto3.client("dynamodb", region_name="eu-west-1") - ddb_mock.create_table( - TableName=table, - KeySchema=[{"AttributeName": "t_id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "t_id", "AttributeType": "S"}], - BillingMode="PAY_PER_REQUEST", - ) - key = {"t_id": {"S": "item1"}} - ddb_mock.put_item( - TableName=table, Item=key, - ) - - ddb_mock.update_item( - TableName=table, - Key=key, - UpdateExpression="add s_i :s1", - ExpressionAttributeValues={":s1": {"NS": ["3"]}}, - ) - updated_item = ddb_mock.get_item(TableName=table, Key=key)["Item"] - assert updated_item["s_i"]["NS"] == ["3"] - - -@mock_dynamodb2 -def test_transact_write_items_fails_with_transaction_canceled_exception(): - table_schema = { - "KeySchema": [{"AttributeName": "id", "KeyType": "HASH"}], - "AttributeDefinitions": [{"AttributeName": "id", "AttributeType": "S"},], - } - dynamodb = boto3.client("dynamodb", region_name="us-east-1") - dynamodb.create_table( - TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema - ) - # Insert one item - dynamodb.put_item(TableName="test-table", Item={"id": {"S": "foo"}}) - # Update two items, the one that exists and another that doesn't - with pytest.raises(ClientError) as ex: - dynamodb.transact_write_items( - TransactItems=[ - { - "Update": { - "Key": {"id": {"S": "foo"}}, - "TableName": "test-table", - "UpdateExpression": "SET #k = :v", - "ConditionExpression": "attribute_exists(id)", - "ExpressionAttributeNames": {"#k": "key"}, - "ExpressionAttributeValues": {":v": {"S": "value"}}, - } - }, - { - "Update": { - "Key": {"id": {"S": "doesnotexist"}}, - "TableName": "test-table", - "UpdateExpression": "SET #e = :v", - "ConditionExpression": "attribute_exists(id)", - "ExpressionAttributeNames": {"#e": "key"}, - "ExpressionAttributeValues": {":v": {"S": "value"}}, - } - }, - ] - ) - ex.value.response["Error"]["Code"].should.equal("TransactionCanceledException") - ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) - ex.value.response["Error"]["Message"].should.equal( - "Transaction cancelled, please refer cancellation reasons for specific reasons [None, ConditionalCheckFailed]" - ) - - -@mock_dynamodb2 -def test_gsi_projection_type_keys_only(): - table_schema = { - "KeySchema": [{"AttributeName": "partitionKey", "KeyType": "HASH"}], - "GlobalSecondaryIndexes": [ - { - "IndexName": "GSI-K1", - "KeySchema": [ - {"AttributeName": "gsiK1PartitionKey", "KeyType": "HASH"}, - {"AttributeName": "gsiK1SortKey", "KeyType": "RANGE"}, - ], - "Projection": {"ProjectionType": "KEYS_ONLY",}, - } - ], - "AttributeDefinitions": [ - {"AttributeName": "partitionKey", "AttributeType": "S"}, - {"AttributeName": "gsiK1PartitionKey", "AttributeType": "S"}, - {"AttributeName": "gsiK1SortKey", "AttributeType": "S"}, - ], - } - - item = { - "partitionKey": "pk-1", - "gsiK1PartitionKey": "gsi-pk", - "gsiK1SortKey": "gsi-sk", - "someAttribute": "lore ipsum", - } - - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - dynamodb.create_table( - TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema - ) - table = dynamodb.Table("test-table") - table.put_item(Item=item) - - items = table.query( - KeyConditionExpression=Key("gsiK1PartitionKey").eq("gsi-pk"), - IndexName="GSI-K1", - )["Items"] - items.should.have.length_of(1) - # Item should only include GSI Keys and Table Keys, as per the ProjectionType - items[0].should.equal( - { - "gsiK1PartitionKey": "gsi-pk", - "gsiK1SortKey": "gsi-sk", - "partitionKey": "pk-1", - } - ) - - -@mock_dynamodb2 -def test_gsi_projection_type_include(): - table_schema = { - "KeySchema": [{"AttributeName": "partitionKey", "KeyType": "HASH"}], - "GlobalSecondaryIndexes": [ - { - "IndexName": "GSI-INC", - "KeySchema": [ - {"AttributeName": "gsiK1PartitionKey", "KeyType": "HASH"}, - {"AttributeName": "gsiK1SortKey", "KeyType": "RANGE"}, - ], - "Projection": { - "ProjectionType": "INCLUDE", - "NonKeyAttributes": ["projectedAttribute"], - }, - } - ], - "AttributeDefinitions": [ - {"AttributeName": "partitionKey", "AttributeType": "S"}, - {"AttributeName": "gsiK1PartitionKey", "AttributeType": "S"}, - {"AttributeName": "gsiK1SortKey", "AttributeType": "S"}, - ], - } - - item = { - "partitionKey": "pk-1", - "gsiK1PartitionKey": "gsi-pk", - "gsiK1SortKey": "gsi-sk", - "projectedAttribute": "lore ipsum", - "nonProjectedAttribute": "dolor sit amet", - } - - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - dynamodb.create_table( - TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema - ) - table = dynamodb.Table("test-table") - table.put_item(Item=item) - - items = table.query( - KeyConditionExpression=Key("gsiK1PartitionKey").eq("gsi-pk"), - IndexName="GSI-INC", - )["Items"] - items.should.have.length_of(1) - # Item should only include keys and additionally projected attributes only - items[0].should.equal( - { - "gsiK1PartitionKey": "gsi-pk", - "gsiK1SortKey": "gsi-sk", - "partitionKey": "pk-1", - "projectedAttribute": "lore ipsum", - } - ) - - -@mock_dynamodb2 -def test_lsi_projection_type_keys_only(): - table_schema = { - "KeySchema": [ - {"AttributeName": "partitionKey", "KeyType": "HASH"}, - {"AttributeName": "sortKey", "KeyType": "RANGE"}, - ], - "LocalSecondaryIndexes": [ - { - "IndexName": "LSI", - "KeySchema": [ - {"AttributeName": "partitionKey", "KeyType": "HASH"}, - {"AttributeName": "lsiK1SortKey", "KeyType": "RANGE"}, - ], - "Projection": {"ProjectionType": "KEYS_ONLY",}, - } - ], - "AttributeDefinitions": [ - {"AttributeName": "partitionKey", "AttributeType": "S"}, - {"AttributeName": "sortKey", "AttributeType": "S"}, - {"AttributeName": "lsiK1SortKey", "AttributeType": "S"}, - ], - } - - item = { - "partitionKey": "pk-1", - "sortKey": "sk-1", - "lsiK1SortKey": "lsi-sk", - "someAttribute": "lore ipsum", - } - - dynamodb = boto3.resource("dynamodb", region_name="us-east-1") - dynamodb.create_table( - TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema - ) - table = dynamodb.Table("test-table") - table.put_item(Item=item) - - items = table.query( - KeyConditionExpression=Key("partitionKey").eq("pk-1"), IndexName="LSI", - )["Items"] - items.should.have.length_of(1) - # Item should only include GSI Keys and Table Keys, as per the ProjectionType - items[0].should.equal( - {"partitionKey": "pk-1", "sortKey": "sk-1", "lsiK1SortKey": "lsi-sk"} - ) - - -@mock_dynamodb2 -@pytest.mark.parametrize( - "attr_name", - ["orders", "#placeholder"], - ids=["use attribute name", "use expression attribute name"], -) -def test_set_attribute_is_dropped_if_empty_after_update_expression(attr_name): - table_name, item_key, set_item = "test-table", "test-id", "test-data" - expression_attribute_names = {"#placeholder": "orders"} - client = boto3.client("dynamodb", region_name="us-east-1") - client.create_table( - TableName=table_name, - KeySchema=[{"AttributeName": "customer", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "customer", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - - client.update_item( - TableName=table_name, - Key={"customer": {"S": item_key}}, - UpdateExpression="ADD {} :order".format(attr_name), - ExpressionAttributeNames=expression_attribute_names, - ExpressionAttributeValues={":order": {"SS": [set_item]}}, - ) - resp = client.scan(TableName=table_name, ProjectionExpression="customer, orders") - item = resp["Items"][0] - item.should.have.key("customer") - item.should.have.key("orders") - - client.update_item( - TableName=table_name, - Key={"customer": {"S": item_key}}, - UpdateExpression="DELETE {} :order".format(attr_name), - ExpressionAttributeNames=expression_attribute_names, - ExpressionAttributeValues={":order": {"SS": [set_item]}}, - ) - resp = client.scan(TableName=table_name, ProjectionExpression="customer, orders") - item = resp["Items"][0] - item.should.have.key("customer") - item.should_not.have.key("orders") - - -@mock_dynamodb2 -def test_transact_get_items_should_return_empty_map_for_non_existent_item(): - client = boto3.client("dynamodb", region_name="us-west-2") - table_name = "test-table" - key_schema = [{"AttributeName": "id", "KeyType": "HASH"}] - attribute_definitions = [{"AttributeName": "id", "AttributeType": "S"}] - client.create_table( - TableName=table_name, - KeySchema=key_schema, - AttributeDefinitions=attribute_definitions, - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - item = {"id": {"S": "1"}} - client.put_item(TableName=table_name, Item=item) - items = client.transact_get_items( - TransactItems=[ - {"Get": {"Key": {"id": {"S": "1"}}, "TableName": table_name}}, - {"Get": {"Key": {"id": {"S": "2"}}, "TableName": table_name}}, - ] - ).get("Responses", []) - items.should.have.length_of(2) - items[0].should.equal({"Item": item}) - items[1].should.equal({}) - - -@mock_dynamodb2 -def test_dynamodb_update_item_fails_on_string_sets(): - dynamodb = boto3.resource("dynamodb", region_name="eu-west-1") - client = boto3.client("dynamodb", region_name="eu-west-1") - - table = dynamodb.create_table( - TableName="test", - KeySchema=[{"AttributeName": "record_id", "KeyType": "HASH"},], - AttributeDefinitions=[{"AttributeName": "record_id", "AttributeType": "S"},], - BillingMode="PAY_PER_REQUEST", - ) - table.meta.client.get_waiter("table_exists").wait(TableName="test") - attribute = {"test_field": {"Value": {"SS": ["test1", "test2"],}, "Action": "PUT",}} - - client.update_item( - TableName="test", - Key={"record_id": {"S": "testrecord"}}, - AttributeUpdates=attribute, - ) - - -@mock_dynamodb2 -def test_update_item_add_to_list_using_legacy_attribute_updates(): - resource = boto3.resource("dynamodb", region_name="us-west-2") - resource.create_table( - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], - TableName="TestTable", - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - table = resource.Table("TestTable") - table.wait_until_exists() - table.put_item(Item={"id": "list_add", "attr": ["a", "b", "c"]},) - - table.update_item( - TableName="TestTable", - Key={"id": "list_add"}, - AttributeUpdates={"attr": {"Action": "ADD", "Value": ["d", "e"]}}, - ) - - resp = table.get_item(Key={"id": "list_add"}) - resp["Item"]["attr"].should.equal(["a", "b", "c", "d", "e"]) - - -@mock_dynamodb2 -def test_get_item_for_non_existent_table_raises_error(): - client = boto3.client("dynamodb", "us-east-1") - with pytest.raises(ClientError) as ex: - client.get_item(TableName="non-existent", Key={"site-id": {"S": "foo"}}) - ex.value.response["Error"]["Code"].should.equal("ResourceNotFoundException") - ex.value.response["Error"]["Message"].should.equal("Requested resource not found") - - -@mock_dynamodb2 -def test_error_when_providing_expression_and_nonexpression_params(): - client = boto3.client("dynamodb", "eu-central-1") - table_name = "testtable" - client.create_table( - TableName=table_name, - KeySchema=[{"AttributeName": "pkey", "KeyType": "HASH"},], - AttributeDefinitions=[{"AttributeName": "pkey", "AttributeType": "S"},], - BillingMode="PAY_PER_REQUEST", - ) - - with pytest.raises(ClientError) as ex: - client.update_item( - TableName=table_name, - Key={"pkey": {"S": "testrecord"}}, - AttributeUpdates={ - "test_field": {"Value": {"SS": ["test1", "test2"],}, "Action": "PUT"} - }, - UpdateExpression="DELETE orders :order", - ExpressionAttributeValues={":order": {"SS": ["item"]}}, - ) - err = ex.value.response["Error"] - err["Code"].should.equal("ValidationException") - err["Message"].should.equal( - "Can not use both expression and non-expression parameters in the same request: Non-expression parameters: {AttributeUpdates} Expression parameters: {UpdateExpression}" - ) - - -@mock_dynamodb2 -def test_attribute_item_delete(): - name = "TestTable" - conn = boto3.client("dynamodb", region_name="eu-west-1") - conn.create_table( - TableName=name, - AttributeDefinitions=[{"AttributeName": "name", "AttributeType": "S"}], - KeySchema=[{"AttributeName": "name", "KeyType": "HASH"}], - BillingMode="PAY_PER_REQUEST", - ) - - item_name = "foo" - conn.put_item( - TableName=name, Item={"name": {"S": item_name}, "extra": {"S": "bar"}} - ) - - conn.update_item( - TableName=name, - Key={"name": {"S": item_name}}, - AttributeUpdates={"extra": {"Action": "DELETE"}}, - ) - items = conn.scan(TableName=name)["Items"] - items.should.equal([{"name": {"S": "foo"}}]) - - -@mock_dynamodb2 -def test_gsi_key_can_be_updated(): - name = "TestTable" - conn = boto3.client("dynamodb", region_name="eu-west-2") - conn.create_table( - TableName=name, - KeySchema=[{"AttributeName": "main_key", "KeyType": "HASH"}], - AttributeDefinitions=[ - {"AttributeName": "main_key", "AttributeType": "S"}, - {"AttributeName": "index_key", "AttributeType": "S"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - GlobalSecondaryIndexes=[ - { - "IndexName": "test_index", - "KeySchema": [{"AttributeName": "index_key", "KeyType": "HASH"}], - "Projection": {"ProjectionType": "ALL",}, - "ProvisionedThroughput": { - "ReadCapacityUnits": 1, - "WriteCapacityUnits": 1, - }, - } - ], - ) - - conn.put_item( - TableName=name, - Item={ - "main_key": {"S": "testkey1"}, - "extra_data": {"S": "testdata"}, - "index_key": {"S": "indexkey1"}, - }, - ) - - conn.update_item( - TableName=name, - Key={"main_key": {"S": "testkey1"}}, - UpdateExpression="set index_key=:new_index_key", - ExpressionAttributeValues={":new_index_key": {"S": "new_value"}}, - ) - - item = conn.scan(TableName=name)["Items"][0] - item["index_key"].should.equal({"S": "new_value"}) - item["main_key"].should.equal({"S": "testkey1"}) - - -@mock_dynamodb2 -def test_gsi_key_cannot_be_empty(): - name = "TestTable" - conn = boto3.client("dynamodb", region_name="eu-west-2") - conn.create_table( - TableName=name, - KeySchema=[{"AttributeName": "main_key", "KeyType": "HASH"}], - AttributeDefinitions=[ - {"AttributeName": "main_key", "AttributeType": "S"}, - {"AttributeName": "index_key", "AttributeType": "S"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - GlobalSecondaryIndexes=[ - { - "IndexName": "test_index", - "KeySchema": [{"AttributeName": "index_key", "KeyType": "HASH"}], - "Projection": {"ProjectionType": "ALL",}, - "ProvisionedThroughput": { - "ReadCapacityUnits": 1, - "WriteCapacityUnits": 1, - }, - } - ], - ) - - conn.put_item( - TableName=name, - Item={ - "main_key": {"S": "testkey1"}, - "extra_data": {"S": "testdata"}, - "index_key": {"S": "indexkey1"}, - }, - ) - - with pytest.raises(ClientError) as ex: - conn.update_item( - TableName=name, - Key={"main_key": {"S": "testkey1"}}, - UpdateExpression="set index_key=:new_index_key", - ExpressionAttributeValues={":new_index_key": {"S": ""}}, - ) - err = ex.value.response["Error"] - err["Code"].should.equal("ValidationException") - err["Message"].should.equal( - "One or more parameter values are not valid. The update expression attempted to update a secondary index key to a value that is not supported. The AttributeValue for a key attribute cannot contain an empty string value." - ) - - -@mock_dynamodb2 -def test_create_backup_for_non_existent_table_raises_error(): - client = boto3.client("dynamodb", "us-east-1") - with pytest.raises(ClientError) as ex: - client.create_backup(TableName="non-existent", BackupName="backup") - error = ex.value.response["Error"] - error["Code"].should.equal("TableNotFoundException") - error["Message"].should.equal("Table not found: non-existent") - - -@mock_dynamodb2 -def test_create_backup(): - client = boto3.client("dynamodb", "us-east-1") - table_name = "test-table" - client.create_table( - TableName=table_name, - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - backup_name = "backup-test-table" - resp = client.create_backup(TableName=table_name, BackupName=backup_name) - details = resp.get("BackupDetails") - details.should.have.key("BackupArn").should.contain(table_name) - details.should.have.key("BackupName").should.equal(backup_name) - details.should.have.key("BackupSizeBytes").should.be.a(int) - details.should.have.key("BackupStatus") - details.should.have.key("BackupType").should.equal("USER") - details.should.have.key("BackupCreationDateTime").should.be.a(datetime) - - -@mock_dynamodb2 -def test_create_multiple_backups_with_same_name(): - client = boto3.client("dynamodb", "us-east-1") - table_name = "test-table" - client.create_table( - TableName=table_name, - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - backup_name = "backup-test-table" - backup_arns = [] - for _ in range(4): - backup = client.create_backup(TableName=table_name, BackupName=backup_name).get( - "BackupDetails" - ) - backup["BackupName"].should.equal(backup_name) - backup_arns.should_not.contain(backup["BackupArn"]) - backup_arns.append(backup["BackupArn"]) - - -@mock_dynamodb2 -def test_describe_backup_for_non_existent_backup_raises_error(): - client = boto3.client("dynamodb", "us-east-1") - non_existent_arn = "arn:aws:dynamodb:us-east-1:123456789012:table/table-name/backup/01623095754481-2cfcd6f9" - with pytest.raises(ClientError) as ex: - client.describe_backup(BackupArn=non_existent_arn) - error = ex.value.response["Error"] - error["Code"].should.equal("BackupNotFoundException") - error["Message"].should.equal("Backup not found: {}".format(non_existent_arn)) - - -@mock_dynamodb2 -def test_describe_backup(): - client = boto3.client("dynamodb", "us-east-1") - table_name = "test-table" - table = client.create_table( - TableName=table_name, - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ).get("TableDescription") - backup_name = "backup-test-table" - backup_arn = ( - client.create_backup(TableName=table_name, BackupName=backup_name) - .get("BackupDetails") - .get("BackupArn") - ) - resp = client.describe_backup(BackupArn=backup_arn) - description = resp.get("BackupDescription") - details = description.get("BackupDetails") - details.should.have.key("BackupArn").should.contain(table_name) - details.should.have.key("BackupName").should.equal(backup_name) - details.should.have.key("BackupSizeBytes").should.be.a(int) - details.should.have.key("BackupStatus") - details.should.have.key("BackupType").should.equal("USER") - details.should.have.key("BackupCreationDateTime").should.be.a(datetime) - source = description.get("SourceTableDetails") - source.should.have.key("TableName").should.equal(table_name) - source.should.have.key("TableArn").should.equal(table["TableArn"]) - source.should.have.key("TableSizeBytes").should.be.a(int) - source.should.have.key("KeySchema").should.equal(table["KeySchema"]) - source.should.have.key("TableCreationDateTime").should.equal( - table["CreationDateTime"] - ) - source.should.have.key("ProvisionedThroughput").should.be.a(dict) - source.should.have.key("ItemCount").should.equal(table["ItemCount"]) - - -@mock_dynamodb2 -def test_list_backups_for_non_existent_table(): - client = boto3.client("dynamodb", "us-east-1") - resp = client.list_backups(TableName="non-existent") - resp["BackupSummaries"].should.have.length_of(0) - - -@mock_dynamodb2 -def test_list_backups(): - client = boto3.client("dynamodb", "us-east-1") - table_names = ["test-table-1", "test-table-2"] - backup_names = ["backup-1", "backup-2"] - for table_name in table_names: - client.create_table( - TableName=table_name, - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - for backup_name in backup_names: - client.create_backup(TableName=table_name, BackupName=backup_name) - resp = client.list_backups(BackupType="USER") - resp["BackupSummaries"].should.have.length_of(4) - for table_name in table_names: - resp = client.list_backups(TableName=table_name) - resp["BackupSummaries"].should.have.length_of(2) - for summary in resp["BackupSummaries"]: - summary.should.have.key("TableName").should.equal(table_name) - summary.should.have.key("TableArn").should.contain(table_name) - summary.should.have.key("BackupName").should.be.within(backup_names) - summary.should.have.key("BackupArn") - summary.should.have.key("BackupCreationDateTime").should.be.a(datetime) - summary.should.have.key("BackupStatus") - summary.should.have.key("BackupType").should.be.within(["USER", "SYSTEM"]) - summary.should.have.key("BackupSizeBytes").should.be.a(int) - - -@mock_dynamodb2 -def test_restore_table_from_non_existent_backup_raises_error(): - client = boto3.client("dynamodb", "us-east-1") - non_existent_arn = "arn:aws:dynamodb:us-east-1:123456789012:table/table-name/backup/01623095754481-2cfcd6f9" - with pytest.raises(ClientError) as ex: - client.restore_table_from_backup( - TargetTableName="from-backup", BackupArn=non_existent_arn - ) - error = ex.value.response["Error"] - error["Code"].should.equal("BackupNotFoundException") - error["Message"].should.equal("Backup not found: {}".format(non_existent_arn)) - - -@mock_dynamodb2 -def test_restore_table_from_backup_raises_error_when_table_already_exists(): - client = boto3.client("dynamodb", "us-east-1") - table_name = "test-table" - client.create_table( - TableName=table_name, - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - resp = client.create_backup(TableName=table_name, BackupName="backup") - backup = resp.get("BackupDetails") - with pytest.raises(ClientError) as ex: - client.restore_table_from_backup( - TargetTableName=table_name, BackupArn=backup["BackupArn"] - ) - error = ex.value.response["Error"] - error["Code"].should.equal("TableAlreadyExistsException") - error["Message"].should.equal("Table already exists: {}".format(table_name)) - - -@mock_dynamodb2 -def test_restore_table_from_backup(): - client = boto3.client("dynamodb", "us-east-1") - table_name = "test-table" - resp = client.create_table( - TableName=table_name, - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - table = resp.get("TableDescription") - for i in range(5): - client.put_item(TableName=table_name, Item={"id": {"S": "item %d" % i}}) - - backup_arn = ( - client.create_backup(TableName=table_name, BackupName="backup") - .get("BackupDetails") - .get("BackupArn") - ) - - restored_table_name = "restored-from-backup" - restored = client.restore_table_from_backup( - TargetTableName=restored_table_name, BackupArn=backup_arn - ).get("TableDescription") - restored.should.have.key("AttributeDefinitions").should.equal( - table["AttributeDefinitions"] - ) - restored.should.have.key("TableName").should.equal(restored_table_name) - restored.should.have.key("KeySchema").should.equal(table["KeySchema"]) - restored.should.have.key("TableStatus") - restored.should.have.key("ItemCount").should.equal(5) - restored.should.have.key("TableArn").should.contain(restored_table_name) - restored.should.have.key("RestoreSummary").should.be.a(dict) - summary = restored.get("RestoreSummary") - summary.should.have.key("SourceBackupArn").should.equal(backup_arn) - summary.should.have.key("SourceTableArn").should.equal(table["TableArn"]) - summary.should.have.key("RestoreDateTime").should.be.a(datetime) - summary.should.have.key("RestoreInProgress").should.equal(False) - - -@mock_dynamodb2 -def test_restore_table_to_point_in_time(): - client = boto3.client("dynamodb", "us-east-1") - table_name = "test-table" - resp = client.create_table( - TableName=table_name, - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - table = resp.get("TableDescription") - for i in range(5): - client.put_item(TableName=table_name, Item={"id": {"S": "item %d" % i}}) - - restored_table_name = "restored-from-pit" - restored = client.restore_table_to_point_in_time( - TargetTableName=restored_table_name, SourceTableName=table_name - ).get("TableDescription") - restored.should.have.key("TableName").should.equal(restored_table_name) - restored.should.have.key("KeySchema").should.equal(table["KeySchema"]) - restored.should.have.key("TableStatus") - restored.should.have.key("ItemCount").should.equal(5) - restored.should.have.key("TableArn").should.contain(restored_table_name) - restored.should.have.key("RestoreSummary").should.be.a(dict) - summary = restored.get("RestoreSummary") - summary.should.have.key("SourceTableArn").should.equal(table["TableArn"]) - summary.should.have.key("RestoreDateTime").should.be.a(datetime) - summary.should.have.key("RestoreInProgress").should.equal(False) - - -@mock_dynamodb2 -def test_restore_table_to_point_in_time_raises_error_when_source_not_exist(): - client = boto3.client("dynamodb", "us-east-1") - table_name = "test-table" - restored_table_name = "restored-from-pit" - with pytest.raises(ClientError) as ex: - client.restore_table_to_point_in_time( - TargetTableName=restored_table_name, SourceTableName=table_name - ) - error = ex.value.response["Error"] - error["Code"].should.equal("SourceTableNotFoundException") - error["Message"].should.equal("Source table not found: %s" % table_name) - - -@mock_dynamodb2 -def test_restore_table_to_point_in_time_raises_error_when_dest_exist(): - client = boto3.client("dynamodb", "us-east-1") - table_name = "test-table" - restored_table_name = "restored-from-pit" - client.create_table( - TableName=table_name, - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - client.create_table( - TableName=restored_table_name, - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - with pytest.raises(ClientError) as ex: - client.restore_table_to_point_in_time( - TargetTableName=restored_table_name, SourceTableName=table_name - ) - error = ex.value.response["Error"] - error["Code"].should.equal("TableAlreadyExistsException") - error["Message"].should.equal("Table already exists: %s" % restored_table_name) - - -@mock_dynamodb2 -def test_delete_non_existent_backup_raises_error(): - client = boto3.client("dynamodb", "us-east-1") - non_existent_arn = "arn:aws:dynamodb:us-east-1:123456789012:table/table-name/backup/01623095754481-2cfcd6f9" - with pytest.raises(ClientError) as ex: - client.delete_backup(BackupArn=non_existent_arn) - error = ex.value.response["Error"] - error["Code"].should.equal("BackupNotFoundException") - error["Message"].should.equal("Backup not found: {}".format(non_existent_arn)) - - -@mock_dynamodb2 -def test_delete_backup(): - client = boto3.client("dynamodb", "us-east-1") - table_name = "test-table-1" - backup_names = ["backup-1", "backup-2"] - client.create_table( - TableName=table_name, - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - for backup_name in backup_names: - client.create_backup(TableName=table_name, BackupName=backup_name) - resp = client.list_backups(TableName=table_name, BackupType="USER") - resp["BackupSummaries"].should.have.length_of(2) - backup_to_delete = resp["BackupSummaries"][0]["BackupArn"] - backup_deleted = client.delete_backup(BackupArn=backup_to_delete).get( - "BackupDescription" - ) - backup_deleted.should.have.key("SourceTableDetails") - backup_deleted.should.have.key("BackupDetails") - details = backup_deleted["BackupDetails"] - details.should.have.key("BackupArn").should.equal(backup_to_delete) - details.should.have.key("BackupName").should.be.within(backup_names) - details.should.have.key("BackupStatus").should.equal("DELETED") - resp = client.list_backups(TableName=table_name, BackupType="USER") - resp["BackupSummaries"].should.have.length_of(1) - - -@mock_dynamodb2 -def test_source_and_restored_table_items_are_not_linked(): - client = boto3.client("dynamodb", "us-east-1") - - def add_guids_to_table(table, num_items): - guids = [] - for _ in range(num_items): - guid = str(uuid.uuid4()) - client.put_item(TableName=table, Item={"id": {"S": guid}}) - guids.append(guid) - return guids - - source_table_name = "source-table" - client.create_table( - TableName=source_table_name, - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - guids_original = add_guids_to_table(source_table_name, 5) - - backup_arn = ( - client.create_backup(TableName=source_table_name, BackupName="backup") - .get("BackupDetails") - .get("BackupArn") - ) - guids_added_after_backup = add_guids_to_table(source_table_name, 5) - - restored_table_name = "restored-from-backup" - client.restore_table_from_backup( - TargetTableName=restored_table_name, BackupArn=backup_arn - ) - guids_added_after_restore = add_guids_to_table(restored_table_name, 5) - - source_table_items = client.scan(TableName=source_table_name) - source_table_items.should.have.key("Count").should.equal(10) - source_table_guids = [x["id"]["S"] for x in source_table_items["Items"]] - set(source_table_guids).should.equal( - set(guids_original) | set(guids_added_after_backup) - ) - - restored_table_items = client.scan(TableName=restored_table_name) - restored_table_items.should.have.key("Count").should.equal(10) - restored_table_guids = [x["id"]["S"] for x in restored_table_items["Items"]] - set(restored_table_guids).should.equal( - set(guids_original) | set(guids_added_after_restore) - ) - - -@mock_dynamodb2 -@pytest.mark.parametrize("region", ["eu-central-1", "ap-south-1"]) -def test_describe_endpoints(region): - client = boto3.client("dynamodb", region) - res = client.describe_endpoints()["Endpoints"] - res.should.equal( - [ - { - "Address": "dynamodb.{}.amazonaws.com".format(region), - "CachePeriodInMinutes": 1440, - }, - ] - ) - - -@mock_dynamodb2 -def test_update_non_existing_item_raises_error_and_does_not_contain_item_afterwards(): - """ - https://github.com/spulec/moto/issues/3729 - Exception is raised, but item was persisted anyway - Happened because we would create a placeholder, before validating/executing the UpdateExpression - :return: - """ - name = "TestTable" - conn = boto3.client("dynamodb", region_name="us-west-2") - hkey = "primary_partition_key" - conn.create_table( - TableName=name, - KeySchema=[{"AttributeName": hkey, "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": hkey, "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - update_expression = { - "Key": {hkey: "some_identification_string"}, - "UpdateExpression": "set #AA.#AB = :aa", - "ExpressionAttributeValues": {":aa": "abc"}, - "ExpressionAttributeNames": {"#AA": "some_dict", "#AB": "key1"}, - "ConditionExpression": "attribute_not_exists(#AA.#AB)", - } - table = boto3.resource("dynamodb", region_name="us-west-2").Table(name) - with pytest.raises(ClientError) as err: - table.update_item(**update_expression) - err.value.response["Error"]["Code"].should.equal("ValidationException") - - conn.scan(TableName=name)["Items"].should.have.length_of(0) - - -@mock_dynamodb2 -def test_batch_write_item(): - conn = boto3.resource("dynamodb", region_name="us-west-2") - tables = [f"table-{i}" for i in range(3)] - for name in tables: - conn.create_table( - TableName=name, - KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], - BillingMode="PAY_PER_REQUEST", - ) - - conn.batch_write_item( - RequestItems={ - tables[0]: [{"PutRequest": {"Item": {"id": "0"}}}], - tables[1]: [{"PutRequest": {"Item": {"id": "1"}}}], - tables[2]: [{"PutRequest": {"Item": {"id": "2"}}}], - } - ) - - for idx, name in enumerate(tables): - table = conn.Table(f"table-{idx}") - res = table.get_item(Key={"id": str(idx)}) - assert res["Item"].should.equal({"id": str(idx)}) - scan = table.scan() - assert scan["Count"].should.equal(1) - - conn.batch_write_item( - RequestItems={ - tables[0]: [{"DeleteRequest": {"Key": {"id": "0"}}}], - tables[1]: [{"DeleteRequest": {"Key": {"id": "1"}}}], - tables[2]: [{"DeleteRequest": {"Key": {"id": "2"}}}], - } - ) - - for idx, name in enumerate(tables): - table = conn.Table(f"table-{idx}") - scan = table.scan() - assert scan["Count"].should.equal(0) - - -@mock_dynamodb2 -def test_gsi_lastevaluatedkey(): - # github.com/spulec/moto/issues/3968 - conn = boto3.resource("dynamodb", region_name="us-west-2") - name = "test-table" - table = conn.Table(name) - - conn.create_table( - TableName=name, - KeySchema=[{"AttributeName": "main_key", "KeyType": "HASH"}], - AttributeDefinitions=[ - {"AttributeName": "main_key", "AttributeType": "S"}, - {"AttributeName": "index_key", "AttributeType": "S"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - GlobalSecondaryIndexes=[ - { - "IndexName": "test_index", - "KeySchema": [{"AttributeName": "index_key", "KeyType": "HASH"}], - "Projection": {"ProjectionType": "ALL",}, - "ProvisionedThroughput": { - "ReadCapacityUnits": 1, - "WriteCapacityUnits": 1, - }, - } - ], - ) - - table.put_item( - Item={ - "main_key": "testkey1", - "extra_data": "testdata", - "index_key": "indexkey", - }, - ) - table.put_item( - Item={ - "main_key": "testkey2", - "extra_data": "testdata", - "index_key": "indexkey", - }, - ) - - response = table.query( - Limit=1, - KeyConditionExpression=Key("index_key").eq("indexkey"), - IndexName="test_index", - ) - - items = response["Items"] - items.should.have.length_of(1) - items[0].should.equal( - {"main_key": "testkey1", "extra_data": "testdata", "index_key": "indexkey"} - ) - - last_evaluated_key = response["LastEvaluatedKey"] - last_evaluated_key.should.have.length_of(2) - last_evaluated_key.should.equal({"main_key": "testkey1", "index_key": "indexkey"}) - - -@mock_dynamodb2 -def test_filter_expression_execution_order(): - # As mentioned here: https://github.com/spulec/moto/issues/3909 - # and documented here: https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Query.html#Query.FilterExpression - # the filter expression should be evaluated after the query. - # The same applies to scan operations: - # https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Scan.html#Scan.FilterExpression - - # If we set limit=1 and apply a filter expression whixh excludes the first result - # then we should get no items in response. - - conn = boto3.resource("dynamodb", region_name="us-west-2") - name = "test-filter-expression-table" - table = conn.Table(name) - - conn.create_table( - TableName=name, - KeySchema=[ - {"AttributeName": "hash_key", "KeyType": "HASH"}, - {"AttributeName": "range_key", "KeyType": "RANGE"}, - ], - AttributeDefinitions=[ - {"AttributeName": "hash_key", "AttributeType": "S"}, - {"AttributeName": "range_key", "AttributeType": "S"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - - table.put_item( - Item={"hash_key": "keyvalue", "range_key": "A", "filtered_attribute": "Y"}, - ) - table.put_item( - Item={"hash_key": "keyvalue", "range_key": "B", "filtered_attribute": "Z"}, - ) - - # test query - - query_response_1 = table.query( - Limit=1, - KeyConditionExpression=Key("hash_key").eq("keyvalue"), - FilterExpression=Attr("filtered_attribute").eq("Z"), - ) - - query_items_1 = query_response_1["Items"] - query_items_1.should.have.length_of(0) - - query_last_evaluated_key = query_response_1["LastEvaluatedKey"] - query_last_evaluated_key.should.have.length_of(2) - query_last_evaluated_key.should.equal({"hash_key": "keyvalue", "range_key": "A"}) - - query_response_2 = table.query( - Limit=1, - KeyConditionExpression=Key("hash_key").eq("keyvalue"), - FilterExpression=Attr("filtered_attribute").eq("Z"), - ExclusiveStartKey=query_last_evaluated_key, - ) - - query_items_2 = query_response_2["Items"] - query_items_2.should.have.length_of(1) - query_items_2[0].should.equal( - {"hash_key": "keyvalue", "filtered_attribute": "Z", "range_key": "B"} - ) - - # test scan - - scan_response_1 = table.scan( - Limit=1, FilterExpression=Attr("filtered_attribute").eq("Z"), - ) - - scan_items_1 = scan_response_1["Items"] - scan_items_1.should.have.length_of(0) - - scan_last_evaluated_key = scan_response_1["LastEvaluatedKey"] - scan_last_evaluated_key.should.have.length_of(2) - scan_last_evaluated_key.should.equal({"hash_key": "keyvalue", "range_key": "A"}) - - scan_response_2 = table.scan( - Limit=1, - FilterExpression=Attr("filtered_attribute").eq("Z"), - ExclusiveStartKey=query_last_evaluated_key, - ) - - scan_items_2 = scan_response_2["Items"] - scan_items_2.should.have.length_of(1) - scan_items_2[0].should.equal( - {"hash_key": "keyvalue", "filtered_attribute": "Z", "range_key": "B"} - ) - - -@mock_dynamodb2 -def test_projection_expression_execution_order(): - # projection expression needs to be applied after calculation of - # LastEvaluatedKey as it is possible for LastEvaluatedKey to - # include attributes which are not projected. - - conn = boto3.resource("dynamodb", region_name="us-west-2") - name = "test-projection-expression-with-gsi" - table = conn.Table(name) - - conn.create_table( - TableName=name, - KeySchema=[ - {"AttributeName": "hash_key", "KeyType": "HASH"}, - {"AttributeName": "range_key", "KeyType": "RANGE"}, - ], - AttributeDefinitions=[ - {"AttributeName": "hash_key", "AttributeType": "S"}, - {"AttributeName": "range_key", "AttributeType": "S"}, - {"AttributeName": "index_key", "AttributeType": "S"}, - ], - GlobalSecondaryIndexes=[ - { - "IndexName": "test_index", - "KeySchema": [{"AttributeName": "index_key", "KeyType": "HASH"}], - "Projection": {"ProjectionType": "ALL",}, - "ProvisionedThroughput": { - "ReadCapacityUnits": 1, - "WriteCapacityUnits": 1, - }, - } - ], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - - table.put_item(Item={"hash_key": "keyvalue", "range_key": "A", "index_key": "Z"},) - table.put_item(Item={"hash_key": "keyvalue", "range_key": "B", "index_key": "Z"},) - - # test query - - # if projection expression is applied before LastEvaluatedKey is computed - # then this raises an exception. - table.query( - Limit=1, - IndexName="test_index", - KeyConditionExpression=Key("index_key").eq("Z"), - ProjectionExpression="#a", - ExpressionAttributeNames={"#a": "hashKey"}, - ) - # if projection expression is applied before LastEvaluatedKey is computed - # then this raises an exception. - table.scan( - Limit=1, - IndexName="test_index", - ProjectionExpression="#a", - ExpressionAttributeNames={"#a": "hashKey"}, - ) + res = table.query(KeyConditionExpression=Key("pk").eq("pk") & Key("sk").gte("sk-1")) + res["Items"].should.have.length_of(2) + res["Items"].should.equal([{"pk": "pk", "sk": "sk-1"}, {"pk": "pk", "sk": "sk-2"}]) diff --git a/tests/test_dynamodb2/test_server.py b/tests/test_dynamodb2/test_server.py deleted file mode 100644 index 1e461c466267..000000000000 --- a/tests/test_dynamodb2/test_server.py +++ /dev/null @@ -1,19 +0,0 @@ -import sure # noqa # pylint: disable=unused-import - -import moto.server as server - -""" -Test the different server responses -""" - - -def test_table_list(): - backend = server.create_backend_app("dynamodb2") - test_client = backend.test_client() - res = test_client.get("/") - res.status_code.should.equal(404) - - headers = {"X-Amz-Target": "TestTable.ListTables"} - res = test_client.get("/", headers=headers) - res.data.should.contain(b"TableNames") - res.headers.should.have.key("X-Amz-Crc32") diff --git a/tests/test_dynamodb_v20111205/__init__.py b/tests/test_dynamodb_v20111205/__init__.py new file mode 100644 index 000000000000..08a1c1568c9c --- /dev/null +++ b/tests/test_dynamodb_v20111205/__init__.py @@ -0,0 +1 @@ +# This file is intentionally left blank. diff --git a/tests/test_dynamodb_v20111205/test_server.py b/tests/test_dynamodb_v20111205/test_server.py new file mode 100644 index 000000000000..7255598cac82 --- /dev/null +++ b/tests/test_dynamodb_v20111205/test_server.py @@ -0,0 +1,1336 @@ +import json +import sure # noqa # pylint: disable=unused-import +import pytest + +import moto.server as server +from moto.dynamodb_v20111205 import dynamodb_backends + +""" +Test the different server responses +Docs: +https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Appendix.APIv20111205.html +""" + +TABLE_NAME = "my_table_name" +TABLE_WITH_RANGE_NAME = "my_table_with_range_name" + + +@pytest.fixture(autouse=True) +def test_client(): + backend = server.create_backend_app("dynamodb_v20111205") + test_client = backend.test_client() + + yield test_client + + for _, backend in dynamodb_backends.items(): + backend.reset() + + +def test_404(test_client): + + res = test_client.get("/") + res.status_code.should.equal(404) + + +def test_table_list(test_client): + headers = {"X-Amz-Target": "TestTable.ListTables"} + res = test_client.get("/", headers=headers) + json.loads(res.data).should.equal({"TableNames": []}) + + +def test_create_table(test_client): + res = create_table(test_client) + res = json.loads(res.data)["Table"] + res.should.have.key("CreationDateTime") + del res["CreationDateTime"] + res.should.equal( + { + "KeySchema": { + "HashKeyElement": {"AttributeName": "hkey", "AttributeType": "S"}, + "RangeKeyElement": {"AttributeName": "rkey", "AttributeType": "N"}, + }, + "ProvisionedThroughput": {"ReadCapacityUnits": 5, "WriteCapacityUnits": 10}, + "TableName": TABLE_WITH_RANGE_NAME, + "TableStatus": "ACTIVE", + "ItemCount": 0, + "TableSizeBytes": 0, + } + ) + + headers = {"X-Amz-Target": "TestTable.ListTables"} + res = test_client.get("/", headers=headers) + res = json.loads(res.data) + res.should.equal({"TableNames": [TABLE_WITH_RANGE_NAME]}) + + +def test_create_table_without_range_key(test_client): + res = create_table(test_client, use_range_key=False) + res = json.loads(res.data)["Table"] + res.should.have.key("CreationDateTime") + del res["CreationDateTime"] + res.should.equal( + { + "KeySchema": { + "HashKeyElement": {"AttributeName": "hkey", "AttributeType": "S"} + }, + "ProvisionedThroughput": {"ReadCapacityUnits": 5, "WriteCapacityUnits": 10}, + "TableName": TABLE_NAME, + "TableStatus": "ACTIVE", + "ItemCount": 0, + "TableSizeBytes": 0, + } + ) + + headers = {"X-Amz-Target": "TestTable.ListTables"} + res = test_client.get("/", headers=headers) + res = json.loads(res.data) + res.should.equal({"TableNames": [TABLE_NAME]}) + + +# This test is pointless, as we treat DynamoDB as a global resource +def test_create_table_in_different_regions(test_client): + create_table(test_client) + create_table(test_client, name="Table2", region="us-west-2") + + headers = {"X-Amz-Target": "TestTable.ListTables"} + res = test_client.get("/", headers=headers) + res = json.loads(res.data) + res.should.equal({"TableNames": [TABLE_WITH_RANGE_NAME, "Table2"]}) + + +def test_update_item(test_client): + + create_table(test_client) + + headers, res = put_item(test_client) + + # UpdateItem + headers["X-Amz-Target"] = "DynamoDB_20111205.UpdateItem" + request_body = { + "TableName": TABLE_WITH_RANGE_NAME, + "Key": { + "HashKeyElement": {"S": "customer"}, + "RangeKeyElement": {"N": "12341234"}, + }, + "AttributeUpdates": {"new_att": {"Value": {"SS": ["val"]}, "Action": "PUT"}}, + } + res = test_client.post("/", headers=headers, json=request_body) + + # UpdateItem + headers["X-Amz-Target"] = "DynamoDB_20111205.UpdateItem" + request_body = { + "TableName": TABLE_WITH_RANGE_NAME, + "Key": { + "HashKeyElement": {"S": "customer"}, + "RangeKeyElement": {"N": "12341234"}, + }, + "AttributeUpdates": {"new_n": {"Value": {"N": "42"}, "Action": "PUT"}}, + } + res = test_client.post("/", headers=headers, json=request_body) + res = json.loads(res.data) + + res["ConsumedCapacityUnits"].should.equal(0.5) + res["Attributes"].should.equal( + { + "hkey": "customer", + "name": "myname", + "rkey": "12341234", + "new_att": ["val"], + "new_n": "42", + } + ) + + # UpdateItem - multiples + headers["X-Amz-Target"] = "DynamoDB_20111205.UpdateItem" + request_body = { + "TableName": TABLE_WITH_RANGE_NAME, + "Key": { + "HashKeyElement": {"S": "customer"}, + "RangeKeyElement": {"N": "12341234"}, + }, + "AttributeUpdates": { + "new_n": {"Value": {"N": 7}, "Action": "ADD"}, + "new_att": {"Value": {"S": "val2"}, "Action": "ADD"}, + "name": {"Action": "DELETE"}, + }, + } + res = test_client.post("/", headers=headers, json=request_body) + res = json.loads(res.data) + + res["ConsumedCapacityUnits"].should.equal(0.5) + res["Attributes"].should.equal( + { + "hkey": "customer", + "rkey": "12341234", + "new_att": ["val", "val2"], + "new_n": "49", + } + ) + + # GetItem + headers["X-Amz-Target"] = "DynamoDB_20111205.GetItem" + request_body = { + "TableName": TABLE_WITH_RANGE_NAME, + "Key": { + "HashKeyElement": {"S": "customer"}, + "RangeKeyElement": {"N": "12341234"}, + }, + } + res = test_client.post("/", headers=headers, json=request_body) + res = json.loads(res.data) + res["Item"].should.have.key("new_att").equal({"SS": ["val", "val2"]}) + res["Item"].should.have.key("new_n").equal({"N": "49"}) + res["Item"].shouldnt.have.key("name") + + +@pytest.mark.parametrize( + "use_range_key", [True, False], ids=["using range key", "using hash key only"] +) +def test_delete_table(use_range_key, test_client): + create_table(test_client, use_range_key=use_range_key) + + headers = {"X-Amz-Target": "DynamoDB_20111205.DeleteTable"} + name = TABLE_WITH_RANGE_NAME if use_range_key else TABLE_NAME + test_client.post("/", headers=headers, json={"TableName": name}) + + headers = {"X-Amz-Target": "DynamoDB_20111205.ListTables"} + res = test_client.post("/", headers=headers) + res = json.loads(res.data) + res.should.equal({"TableNames": []}) + + +def test_delete_unknown_table(test_client): + headers = {"X-Amz-Target": "DynamoDB_20111205.DeleteTable"} + res = test_client.post("/", headers=headers, json={"TableName": "unknown_table"}) + res.status_code.should.equal(400) + + json.loads(res.data).should.equal( + {"__type": "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException"} + ) + + +def test_describe_table(test_client): + create_table(test_client) + + headers = { + "X-Amz-Target": "DynamoDB_20111205.DescribeTable", + "Content-Type": "application/x-amz-json-1.0", + } + res = test_client.post( + "/", headers=headers, json={"TableName": TABLE_WITH_RANGE_NAME} + ) + res = json.loads(res.data)["Table"] + res.should.have.key("CreationDateTime") + del res["CreationDateTime"] + res.should.equal( + { + "KeySchema": { + "HashKeyElement": {"AttributeName": "hkey", "AttributeType": "S"}, + "RangeKeyElement": {"AttributeName": "rkey", "AttributeType": "N"}, + }, + "ProvisionedThroughput": {"ReadCapacityUnits": 5, "WriteCapacityUnits": 10}, + "TableName": TABLE_WITH_RANGE_NAME, + "TableStatus": "ACTIVE", + "ItemCount": 0, + "TableSizeBytes": 0, + } + ) + + +def test_describe_missing_table(test_client): + headers = { + "X-Amz-Target": "DynamoDB_20111205.DescribeTable", + "Content-Type": "application/x-amz-json-1.0", + } + res = test_client.post("/", headers=headers, json={"TableName": "unknown_table"}) + res.status_code.should.equal(400) + json.loads(res.data).should.equal( + {"__type": "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException"} + ) + + +@pytest.mark.parametrize( + "use_range_key", [True, False], ids=["using range key", "using hash key only"] +) +def test_update_table(test_client, use_range_key): + table_name = TABLE_WITH_RANGE_NAME if use_range_key else TABLE_NAME + create_table(test_client, use_range_key=use_range_key) + + headers = { + "X-Amz-Target": "DynamoDB_20111205.UpdateTable", + "Content-Type": "application/x-amz-json-1.0", + } + request_data = { + "TableName": table_name, + "ProvisionedThroughput": {"ReadCapacityUnits": 5, "WriteCapacityUnits": 15}, + } + test_client.post("/", headers=headers, json=request_data) + + # DescribeTable - verify the throughput is persisted + headers = { + "X-Amz-Target": "DynamoDB_20111205.DescribeTable", + "Content-Type": "application/x-amz-json-1.0", + } + res = test_client.post("/", headers=headers, json={"TableName": table_name}) + throughput = json.loads(res.data)["Table"]["ProvisionedThroughput"] + + throughput.should.equal({"ReadCapacityUnits": 5, "WriteCapacityUnits": 15}) + + +def test_put_return_none(test_client): + create_table(test_client) + + headers = { + "X-Amz-Target": "DynamoDB_20111205.PutItem", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": TABLE_WITH_RANGE_NAME, + "Item": { + "hkey": {"S": "customer"}, + "rkey": {"N": "12341234"}, + "name": {"S": "myname"}, + }, + "ReturnValues": "NONE", + } + res = test_client.post("/", headers=headers, json=request_body) + res = json.loads(res.data) + # This seems wrong - it should return nothing, considering return_values is set to none + res["Attributes"].should.equal( + {"hkey": "customer", "name": "myname", "rkey": "12341234"} + ) + + +def test_put_return_none_without_range_key(test_client): + create_table(test_client, use_range_key=False) + + headers = { + "X-Amz-Target": "DynamoDB_20111205.PutItem", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": TABLE_NAME, + "Item": {"hkey": {"S": "customer"}, "name": {"S": "myname"}}, + "ReturnValues": "NONE", + } + res = test_client.post("/", headers=headers, json=request_body) + res = json.loads(res.data) + print(res) + # This seems wrong - it should return nothing, considering return_values is set to none + res["Attributes"].should.equal({"hkey": "customer", "name": "myname"}) + + +def test_put_item_from_unknown_table(test_client): + headers = { + "X-Amz-Target": "DynamoDB_20111205.PutItem", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": "unknown_table", + "Item": { + "hkey": {"S": "customer"}, + "rkey": {"N": "12341234"}, + "name": {"S": "myname"}, + }, + "ReturnValues": "NONE", + } + res = test_client.post("/", headers=headers, json=request_body) + + res.status_code.should.equal(400) + json.loads(res.data).should.equal( + {"__type": "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException"} + ) + + +def test_get_item_from_unknown_table(test_client): + headers = { + "X-Amz-Target": "DynamoDB_20111205.GetItem", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": "unknown_table", + "Key": { + "HashKeyElement": {"S": "customer"}, + "RangeKeyElement": {"N": "12341234"}, + }, + } + res = test_client.post("/", headers=headers, json=request_body) + + res.status_code.should.equal(404) + json.loads(res.data).should.equal( + {"__type": "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException"} + ) + + +@pytest.mark.parametrize( + "use_range_key", [True, False], ids=["using range key", "using hash key only"] +) +def test_get_unknown_item_from_table(use_range_key, test_client): + create_table(test_client, use_range_key=use_range_key) + + headers = { + "X-Amz-Target": "DynamoDB_20111205.GetItem", + "Content-Type": "application/x-amz-json-1.0", + } + table_name = TABLE_WITH_RANGE_NAME if use_range_key else TABLE_NAME + request_body = { + "TableName": table_name, + "Key": {"HashKeyElement": {"S": "customer"}}, + } + if use_range_key: + request_body["Key"]["RangeKeyElement"] = {"N": "12341234"} + res = test_client.post("/", headers=headers, json=request_body) + + res.status_code.should.equal(404) + json.loads(res.data).should.equal( + {"__type": "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException"} + ) + + +def test_get_item_without_range_key(test_client): + create_table(test_client) + + headers = { + "X-Amz-Target": "DynamoDB_20111205.GetItem", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": TABLE_WITH_RANGE_NAME, + "Key": {"HashKeyElement": {"S": "customer"}}, + } + res = test_client.post("/", headers=headers, json=request_body) + + res.status_code.should.equal(400) + json.loads(res.data).should.equal( + {"__type": "com.amazon.coral.validate#ValidationException"} + ) + + +def test_put_and_get_item(test_client): + create_table(test_client) + + headers, res = put_item(test_client) + + res["ConsumedCapacityUnits"].should.equal(1) + res["Attributes"].should.equal( + {"hkey": "customer", "name": "myname", "rkey": "12341234"} + ) + + # GetItem + headers["X-Amz-Target"] = "DynamoDB_20111205.GetItem" + request_body = { + "TableName": TABLE_WITH_RANGE_NAME, + "Key": { + "HashKeyElement": {"S": "customer"}, + "RangeKeyElement": {"N": "12341234"}, + }, + } + res = test_client.post("/", headers=headers, json=request_body) + res = json.loads(res.data) + res["ConsumedCapacityUnits"].should.equal(0.5) + res["Item"].should.equal( + {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "12341234"}} + ) + + # GetItem - return single attribute + headers["X-Amz-Target"] = "DynamoDB_20111205.GetItem" + request_body = { + "TableName": TABLE_WITH_RANGE_NAME, + "Key": { + "HashKeyElement": {"S": "customer"}, + "RangeKeyElement": {"N": "12341234"}, + }, + "AttributesToGet": ["name"], + } + res = test_client.post("/", headers=headers, json=request_body) + res = json.loads(res.data) + res["ConsumedCapacityUnits"].should.equal(0.5) + res["Item"].should.equal({"name": {"S": "myname"}}) + + +def test_put_and_get_item_without_range_key(test_client): + create_table(test_client, use_range_key=False) + + headers, res = put_item(test_client, use_range_key=False) + + res["ConsumedCapacityUnits"].should.equal(1) + res["Attributes"].should.equal({"hkey": "customer", "name": "myname"}) + + # GetItem + headers["X-Amz-Target"] = "DynamoDB_20111205.GetItem" + request_body = { + "TableName": TABLE_NAME, + "Key": {"HashKeyElement": {"S": "customer"}}, + } + res = test_client.post("/", headers=headers, json=request_body) + res = json.loads(res.data) + res["ConsumedCapacityUnits"].should.equal(0.5) + res["Item"].should.equal({"hkey": {"S": "customer"}, "name": {"S": "myname"}}) + + # GetItem - return single attribute + headers["X-Amz-Target"] = "DynamoDB_20111205.GetItem" + request_body = { + "TableName": TABLE_NAME, + "Key": {"HashKeyElement": {"S": "customer"}}, + "AttributesToGet": ["name"], + } + res = test_client.post("/", headers=headers, json=request_body) + res = json.loads(res.data) + res["ConsumedCapacityUnits"].should.equal(0.5) + res["Item"].should.equal({"name": {"S": "myname"}}) + + +def test_scan_simple(test_client): + create_table(test_client) + + put_item(test_client) + put_item(test_client, rkey="12341235") + put_item(test_client, rkey="12341236") + + headers = { + "X-Amz-Target": "DynamoDB_20111205.Scan", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = {"TableName": TABLE_WITH_RANGE_NAME} + res = test_client.post("/", headers=headers, json=request_body) + res = json.loads(res.data) + + res.should.have.key("Count").equal(3) + res.should.have.key("ScannedCount").equal(3) + res.should.have.key("ConsumedCapacityUnits").equal(1) + res.should.have.key("Items").length_of(3) + + items = res["Items"] + items.should.contain( + {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "12341234"}} + ) + items.should.contain( + {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "12341235"}} + ) + items.should.contain( + {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "12341236"}} + ) + + +def test_scan_with_filter(test_client): + create_table(test_client) + + put_item(test_client, rkey="1230", name="somename") + put_item(test_client, rkey="1234", name=None) + put_item(test_client, rkey="1246") + + # SCAN specific item + headers = { + "X-Amz-Target": "DynamoDB_20111205.Scan", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": TABLE_WITH_RANGE_NAME, + "ScanFilter": { + "rkey": {"AttributeValueList": [{"S": "1234"}], "ComparisonOperator": "EQ"} + }, + } + res = test_client.post("/", headers=headers, json=request_body) + res = json.loads(res.data) + + res.should.have.key("Count").equal(1) + res.should.have.key("ScannedCount").equal(3) + res.should.have.key("ConsumedCapacityUnits").equal(1) + res.should.have.key("Items").length_of(1) + + items = res["Items"] + items.should.contain({"hkey": {"S": "customer"}, "rkey": {"N": "1234"}}) + + # SCAN begins_with + headers = { + "X-Amz-Target": "DynamoDB_20111205.Scan", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": TABLE_WITH_RANGE_NAME, + "ScanFilter": { + "rkey": { + "AttributeValueList": [{"S": "124"}], + "ComparisonOperator": "BEGINS_WITH", + } + }, + } + res = test_client.post("/", headers=headers, json=request_body) + items = json.loads(res.data)["Items"] + + items.should.contain( + {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "1246"}} + ) + + # SCAN contains + headers = { + "X-Amz-Target": "DynamoDB_20111205.Scan", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": TABLE_WITH_RANGE_NAME, + "ScanFilter": { + "name": { + "AttributeValueList": [{"S": "mena"}], + "ComparisonOperator": "CONTAINS", + } + }, + } + res = test_client.post("/", headers=headers, json=request_body) + items = json.loads(res.data)["Items"] + + items.should.contain( + {"hkey": {"S": "customer"}, "name": {"S": "somename"}, "rkey": {"N": "1230"}} + ) + + # SCAN null + headers = { + "X-Amz-Target": "DynamoDB_20111205.Scan", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": TABLE_WITH_RANGE_NAME, + "ScanFilter": {"name": {"ComparisonOperator": "NULL"}}, + } + res = test_client.post("/", headers=headers, json=request_body) + items = json.loads(res.data)["Items"] + + items.should.contain({"hkey": {"S": "customer"}, "rkey": {"N": "1234"}}) + + # SCAN NOT NULL + headers = { + "X-Amz-Target": "DynamoDB_20111205.Scan", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": TABLE_WITH_RANGE_NAME, + "ScanFilter": {"name": {"ComparisonOperator": "NOT_NULL"}}, + } + res = test_client.post("/", headers=headers, json=request_body) + items = json.loads(res.data)["Items"] + + items.should.equal( + [ + { + "hkey": {"S": "customer"}, + "name": {"S": "somename"}, + "rkey": {"N": "1230"}, + }, + {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "1246"}}, + ] + ) + + # SCAN between + headers = { + "X-Amz-Target": "DynamoDB_20111205.Scan", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": TABLE_WITH_RANGE_NAME, + "ScanFilter": { + "rkey": { + "AttributeValueList": [{"S": "1230"}, {"S": "1240"}], + "ComparisonOperator": "BETWEEN", + } + }, + } + res = test_client.post("/", headers=headers, json=request_body) + items = json.loads(res.data)["Items"] + + items.should.contain( + {"hkey": {"S": "customer"}, "name": {"S": "somename"}, "rkey": {"N": "1230"}} + ) + + +def test_scan_with_filter_in_table_without_range_key(test_client): + create_table(test_client, use_range_key=False) + + put_item(test_client, use_range_key=False, hkey="customer1", name=None) + put_item(test_client, use_range_key=False, hkey="customer2") + put_item(test_client, use_range_key=False, hkey="customer3", name="special") + + # SCAN specific item + headers = { + "X-Amz-Target": "DynamoDB_20111205.Scan", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": TABLE_NAME, + "ScanFilter": { + "name": { + "AttributeValueList": [{"S": "special"}], + "ComparisonOperator": "EQ", + } + }, + } + res = test_client.post("/", headers=headers, json=request_body) + res = json.loads(res.data) + + res.should.have.key("Count").equal(1) + res.should.have.key("ScannedCount").equal(3) + res.should.have.key("ConsumedCapacityUnits").equal(1) + res.should.have.key("Items").length_of(1) + + items = res["Items"] + items.should.contain({"hkey": {"S": "customer3"}, "name": {"S": "special"}}) + + # SCAN begins_with + headers = { + "X-Amz-Target": "DynamoDB_20111205.Scan", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": TABLE_NAME, + "ScanFilter": { + "hkey": { + "AttributeValueList": [{"S": "cust"}], + "ComparisonOperator": "BEGINS_WITH", + } + }, + } + res = test_client.post("/", headers=headers, json=request_body) + items = json.loads(res.data)["Items"] + + items.should.have.length_of(3) # all customers start with cust + + # SCAN contains + headers = { + "X-Amz-Target": "DynamoDB_20111205.Scan", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": TABLE_NAME, + "ScanFilter": { + "name": { + "AttributeValueList": [{"S": "yna"}], + "ComparisonOperator": "CONTAINS", + } + }, + } + res = test_client.post("/", headers=headers, json=request_body) + items = json.loads(res.data)["Items"] + + items.should.have.equal([{"hkey": {"S": "customer2"}, "name": {"S": "myname"}}]) + + # SCAN null + headers = { + "X-Amz-Target": "DynamoDB_20111205.Scan", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": TABLE_NAME, + "ScanFilter": {"name": {"ComparisonOperator": "NULL"}}, + } + res = test_client.post("/", headers=headers, json=request_body) + items = json.loads(res.data)["Items"] + + items.should.equal([{"hkey": {"S": "customer1"}}]) + + # SCAN NOT NULL + headers = { + "X-Amz-Target": "DynamoDB_20111205.Scan", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": TABLE_NAME, + "ScanFilter": {"name": {"ComparisonOperator": "NOT_NULL"}}, + } + res = test_client.post("/", headers=headers, json=request_body) + items = json.loads(res.data)["Items"] + + items.should.have.length_of(2) + items.should.contain({"hkey": {"S": "customer2"}, "name": {"S": "myname"}}) + items.should.contain({"hkey": {"S": "customer3"}, "name": {"S": "special"}}) + + +def test_scan_with_undeclared_table(test_client): + headers = { + "X-Amz-Target": "DynamoDB_20111205.Scan", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = {"TableName": "unknown_table"} + res = test_client.post("/", headers=headers, json=request_body) + res.status_code.should.equal(400) + json.loads(res.data).should.equal( + {"__type": "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException"} + ) + + +def test_query_in_table_without_range_key(test_client): + create_table(test_client, use_range_key=False) + + put_item(test_client, use_range_key=False) + + headers = { + "X-Amz-Target": "DynamoDB_20111205.Query", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = {"TableName": TABLE_NAME, "HashKeyValue": {"S": "customer"}} + res = test_client.post("/", headers=headers, json=request_body) + res = json.loads(res.data) + + res.should.have.key("Count").equal(1) + res.should.have.key("ConsumedCapacityUnits").equal(1) + res.should.have.key("Items").length_of(1) + + items = res["Items"] + items.should.contain({"hkey": {"S": "customer"}, "name": {"S": "myname"}}) + + # QUERY for unknown value + headers = { + "X-Amz-Target": "DynamoDB_20111205.Query", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = {"TableName": TABLE_NAME, "HashKeyValue": {"S": "unknown-value"}} + res = test_client.post("/", headers=headers, json=request_body) + res = json.loads(res.data) + + # TODO: We should not get any results here + # res.should.have.key("Count").equal(0) + # res.should.have.key("Items").length_of(0) + + +def test_query_item_by_hash_only(test_client): + create_table(test_client) + + put_item(test_client) + put_item(test_client, rkey="12341235") + put_item(test_client, rkey="12341236") + + headers = { + "X-Amz-Target": "DynamoDB_20111205.Query", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": TABLE_WITH_RANGE_NAME, + "HashKeyValue": {"S": "customer"}, + } + res = test_client.post("/", headers=headers, json=request_body) + res = json.loads(res.data) + + res.should.have.key("Count").equal(3) + res.should.have.key("ConsumedCapacityUnits").equal(1) + res.should.have.key("Items").length_of(3) + + items = res["Items"] + items.should.contain( + {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "12341234"}} + ) + items.should.contain( + {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "12341235"}} + ) + items.should.contain( + {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "12341236"}} + ) + + +def test_query_item_by_range_key(test_client): + create_table(test_client, use_range_key=True) + + put_item(test_client, rkey="1234") + put_item(test_client, rkey="1235") + put_item(test_client, rkey="1247") + + # GT some + headers = { + "X-Amz-Target": "DynamoDB_20111205.Query", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": TABLE_WITH_RANGE_NAME, + "HashKeyValue": {"S": "customer"}, + "RangeKeyCondition": { + "AttributeValueList": [{"N": "1235"}], + "ComparisonOperator": "GT", + }, + } + res = test_client.post("/", headers=headers, json=request_body) + res = json.loads(res.data) + + res.should.have.key("Count").equal(1) + res.should.have.key("ConsumedCapacityUnits").equal(1) + res.should.have.key("Items").length_of(1) + + items = res["Items"] + items.should.contain( + {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "1247"}} + ) + + # GT all + headers = { + "X-Amz-Target": "DynamoDB_20111205.Query", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": TABLE_WITH_RANGE_NAME, + "HashKeyValue": {"S": "customer"}, + "RangeKeyCondition": { + "AttributeValueList": [{"N": "0"}], + "ComparisonOperator": "GT", + }, + } + res = test_client.post("/", headers=headers, json=request_body) + res = json.loads(res.data) + + res.should.have.key("Count").equal(3) + res.should.have.key("Items").length_of(3) + + # GT none + headers = { + "X-Amz-Target": "DynamoDB_20111205.Query", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": TABLE_WITH_RANGE_NAME, + "HashKeyValue": {"S": "customer"}, + "RangeKeyCondition": { + "AttributeValueList": [{"N": "9999"}], + "ComparisonOperator": "GT", + }, + } + res = test_client.post("/", headers=headers, json=request_body) + res = json.loads(res.data) + + res["ConsumedCapacityUnits"].should.equal(1) + res["Items"].should.equal([]) + res["Count"].should.equal(0) + + # CONTAINS some + headers = { + "X-Amz-Target": "DynamoDB_20111205.Query", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": TABLE_WITH_RANGE_NAME, + "HashKeyValue": {"S": "customer"}, + "RangeKeyCondition": { + "AttributeValueList": [{"N": "24"}], + "ComparisonOperator": "CONTAINS", + }, + } + res = test_client.post("/", headers=headers, json=request_body) + items = json.loads(res.data)["Items"] + + items.should.equal( + [{"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "1247"}}] + ) + + # BEGINS_WITH + headers = { + "X-Amz-Target": "DynamoDB_20111205.Query", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": TABLE_WITH_RANGE_NAME, + "HashKeyValue": {"S": "customer"}, + "RangeKeyCondition": { + "AttributeValueList": [{"N": "123"}], + "ComparisonOperator": "BEGINS_WITH", + }, + } + res = test_client.post("/", headers=headers, json=request_body) + items = json.loads(res.data)["Items"] + + items.should.equal( + [ + {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "1234"}}, + {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "1235"}}, + ] + ) + + # CONTAINS + headers = { + "X-Amz-Target": "DynamoDB_20111205.Query", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": TABLE_WITH_RANGE_NAME, + "HashKeyValue": {"S": "customer"}, + "RangeKeyCondition": { + "AttributeValueList": [{"N": "0"}, {"N": "1240"}], + "ComparisonOperator": "BETWEEN", + }, + } + res = test_client.post("/", headers=headers, json=request_body) + items = json.loads(res.data)["Items"] + + items.should.equal( + [ + {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "1234"}}, + {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "1235"}}, + ] + ) + + +def test_query_item_with_undeclared_table(test_client): + headers = { + "X-Amz-Target": "DynamoDB_20111205.Query", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": "unknown_table", + "HashKeyValue": {"S": "customer"}, + "RangeKeyCondition": { + "AttributeValueList": [{"N": "1235"}], + "ComparisonOperator": "GT", + }, + } + res = test_client.post("/", headers=headers, json=request_body) + res.status_code.should.equal(400) + json.loads(res.data).should.equal( + {"__type": "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException"} + ) + + +def test_delete_item(test_client): + create_table(test_client) + + put_item(test_client) + put_item(test_client, rkey="12341235") + put_item(test_client, rkey="12341236") + + headers = { + "X-Amz-Target": "DynamoDB_20111205.DeleteItem", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": TABLE_WITH_RANGE_NAME, + "Key": { + "HashKeyElement": {"S": "customer"}, + "RangeKeyElement": {"N": "12341236"}, + }, + } + res = test_client.post("/", headers=headers, json=request_body) + res = json.loads(res.data) + res.should.equal({"Attributes": [], "ConsumedCapacityUnits": 0.5}) + + # GetItem + headers["X-Amz-Target"] = "DynamoDB_20111205.GetItem" + request_body = { + "TableName": TABLE_WITH_RANGE_NAME, + "Key": { + "HashKeyElement": {"S": "customer"}, + "RangeKeyElement": {"N": "12341234"}, + }, + } + res = test_client.post("/", headers=headers, json=request_body) + res = json.loads(res.data) + + res["Item"].should.have.key("hkey").equal({"S": "customer"}) + res["Item"].should.have.key("rkey").equal({"N": "12341234"}) + res["Item"].should.have.key("name").equal({"S": "myname"}) + + +def test_update_item_that_doesnt_exist(test_client): + create_table(test_client) + + # UpdateItem + headers = {"X-Amz-Target": "DynamoDB_20111205.UpdateItem"} + request_body = { + "TableName": "Table1", + "Key": { + "HashKeyElement": {"S": "customer"}, + "RangeKeyElement": {"N": "12341234"}, + }, + "AttributeUpdates": {"new_att": {"Value": {"SS": ["val"]}, "Action": "PUT"}}, + } + res = test_client.post("/", headers=headers, json=request_body) + res = json.loads(res.data) + res.should.equal( + {"__type": "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException"} + ) + + +def test_delete_item_without_range_key(test_client): + create_table(test_client, use_range_key=False) + + put_item(test_client, use_range_key=False) + + headers = { + "X-Amz-Target": "DynamoDB_20111205.DeleteItem", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": TABLE_NAME, + "Key": {"HashKeyElement": {"S": "customer"}}, + } + res = test_client.post("/", headers=headers, json=request_body) + res = json.loads(res.data) + res.should.equal({"Attributes": [], "ConsumedCapacityUnits": 0.5}) + + +def test_delete_item_with_return_values(test_client): + create_table(test_client) + + put_item(test_client) + put_item(test_client, rkey="12341235") + put_item(test_client, rkey="12341236") + + headers = { + "X-Amz-Target": "DynamoDB_20111205.DeleteItem", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": TABLE_WITH_RANGE_NAME, + "Key": { + "HashKeyElement": {"S": "customer"}, + "RangeKeyElement": {"N": "12341236"}, + }, + "ReturnValues": "ALL_OLD", + } + res = test_client.post("/", headers=headers, json=request_body) + res = json.loads(res.data) + res.should.equal( + { + "Attributes": {"hkey": "customer", "name": "myname", "rkey": "12341236"}, + "ConsumedCapacityUnits": 0.5, + } + ) + + +def test_delete_unknown_item(test_client): + create_table(test_client) + + headers = { + "X-Amz-Target": "DynamoDB_20111205.DeleteItem", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": TABLE_WITH_RANGE_NAME, + "Key": { + "HashKeyElement": {"S": "customer"}, + "RangeKeyElement": {"N": "12341236"}, + }, + "ReturnValues": "ALL_OLD", + } + res = test_client.post("/", headers=headers, json=request_body) + res.status_code.should.equal(400) + json.loads(res.data).should.equal( + {"__type": "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException"} + ) + + +def test_update_item_in_nonexisting_table(test_client): + # UpdateItem + headers = {"X-Amz-Target": "DynamoDB_20111205.UpdateItem"} + request_body = { + "TableName": "nonexistent", + "Key": { + "HashKeyElement": {"S": "customer"}, + "RangeKeyElement": {"N": "12341234"}, + }, + "AttributeUpdates": {"new_att": {"Value": {"SS": ["val"]}, "Action": "PUT"}}, + } + res = test_client.post("/", headers=headers, json=request_body) + res.status_code.should.equal(400) + json.loads(res.data).should.equal( + {"__type": "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException"} + ) + + +def test_delete_from_unknown_table(test_client): + headers = { + "X-Amz-Target": "DynamoDB_20111205.DeleteItem", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": "unknown_table", + "Key": { + "HashKeyElement": {"S": "customer"}, + "RangeKeyElement": {"N": "12341236"}, + }, + "ReturnValues": "ALL_OLD", + } + res = test_client.post("/", headers=headers, json=request_body) + res.status_code.should.equal(400) + json.loads(res.data).should.equal( + {"__type": "com.amazonaws.dynamodb.v20111205#ResourceNotFoundException"} + ) + + +def test_batch_get_item(test_client): + create_table(test_client) + + put_item(test_client) + put_item(test_client, rkey="12341235") + put_item(test_client, rkey="12341236") + + headers = { + "X-Amz-Target": "DynamoDB_20111205.BatchGetItem", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "RequestItems": { + TABLE_WITH_RANGE_NAME: { + "Keys": [ + { + "HashKeyElement": {"S": "customer"}, + "RangeKeyElement": {"N": "12341235"}, + }, + { + "HashKeyElement": {"S": "customer"}, + "RangeKeyElement": {"N": "12341236"}, + }, + ], + } + } + } + res = test_client.post("/", headers=headers, json=request_body) + res = json.loads(res.data)["Responses"] + + res.should.have.key("UnprocessedKeys").equal({}) + table_items = [i["Item"] for i in res[TABLE_WITH_RANGE_NAME]["Items"]] + table_items.should.have.length_of(2) + + table_items.should.contain( + {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "12341235"}} + ) + table_items.should.contain( + {"hkey": {"S": "customer"}, "name": {"S": "myname"}, "rkey": {"N": "12341236"}} + ) + + +def test_batch_get_item_without_range_key(test_client): + create_table(test_client, use_range_key=False) + + put_item(test_client, use_range_key=False, hkey="customer1") + put_item(test_client, use_range_key=False, hkey="customer2") + put_item(test_client, use_range_key=False, hkey="customer3") + + headers = { + "X-Amz-Target": "DynamoDB_20111205.BatchGetItem", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "RequestItems": { + TABLE_NAME: { + "Keys": [ + {"HashKeyElement": {"S": "customer1"}}, + {"HashKeyElement": {"S": "customer3"}}, + ], + } + } + } + res = test_client.post("/", headers=headers, json=request_body) + res = json.loads(res.data)["Responses"] + + res.should.have.key("UnprocessedKeys").equal({}) + table_items = [i["Item"] for i in res[TABLE_NAME]["Items"]] + table_items.should.have.length_of(2) + + table_items.should.contain({"hkey": {"S": "customer1"}, "name": {"S": "myname"}}) + table_items.should.contain({"hkey": {"S": "customer3"}, "name": {"S": "myname"}}) + + +def test_batch_write_item(test_client): + create_table(test_client) + + # BATCH-WRITE + headers = { + "X-Amz-Target": "DynamoDB_20111205.BatchWriteItem", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "RequestItems": { + TABLE_WITH_RANGE_NAME: [ + { + "PutRequest": { + "Item": {"hkey": {"S": "customer"}, "rkey": {"S": "1234"}} + } + }, + { + "PutRequest": { + "Item": {"hkey": {"S": "customer"}, "rkey": {"S": "1235"}} + } + }, + ], + } + } + test_client.post("/", headers=headers, json=request_body) + + # SCAN - verify all items are present + headers = { + "X-Amz-Target": "DynamoDB_20111205.Scan", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = {"TableName": TABLE_WITH_RANGE_NAME} + res = test_client.post("/", headers=headers, json=request_body) + res = json.loads(res.data) + + res.should.have.key("Count").equal(2) + + +def test_batch_write_item_without_range_key(test_client): + create_table(test_client, use_range_key=False) + + # BATCH-WRITE + headers = { + "X-Amz-Target": "DynamoDB_20111205.BatchWriteItem", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "RequestItems": { + TABLE_NAME: [ + {"PutRequest": {"Item": {"hkey": {"S": "customer"}}}}, + {"PutRequest": {"Item": {"hkey": {"S": "customer2"}}}}, + ], + } + } + test_client.post("/", headers=headers, json=request_body) + + # SCAN - verify all items are present + headers = { + "X-Amz-Target": "DynamoDB_20111205.Scan", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = {"TableName": TABLE_NAME} + res = test_client.post("/", headers=headers, json=request_body) + res = json.loads(res.data) + + res.should.have.key("Count").equal(2) + + +def put_item( + test_client, hkey="customer", rkey="12341234", name="myname", use_range_key=True +): + table_name = TABLE_WITH_RANGE_NAME if use_range_key else TABLE_NAME + headers = { + "X-Amz-Target": "DynamoDB_20111205.PutItem", + "Content-Type": "application/x-amz-json-1.0", + } + request_body = { + "TableName": table_name, + "Item": {"hkey": {"S": hkey}}, + "ReturnValues": "ALL_OLD", + } + if name: + request_body["Item"]["name"] = {"S": name} + if rkey and use_range_key: + request_body["Item"]["rkey"] = {"N": rkey} + res = test_client.post("/", headers=headers, json=request_body) + res = json.loads(res.data) + return headers, res + + +def create_table(test_client, name=None, region=None, use_range_key=True): + if not name: + name = TABLE_WITH_RANGE_NAME if use_range_key else TABLE_NAME + headers = { + "X-Amz-Target": "DynamoDB_20111205.CreateTable", + "Content-Type": "application/x-amz-json-1.0", + } + if region: + headers["Host"] = "dynamodb.{}.amazonaws.com".format(region) + request_body = { + "TableName": name, + "KeySchema": { + "HashKeyElement": {"AttributeName": "hkey", "AttributeType": "S"} + }, + "ProvisionedThroughput": {"ReadCapacityUnits": 5, "WriteCapacityUnits": 10}, + } + if use_range_key: + request_body["KeySchema"]["RangeKeyElement"] = { + "AttributeName": "rkey", + "AttributeType": "N", + } + return test_client.post("/", headers=headers, json=request_body) diff --git a/tests/test_dynamodb_v20111205/test_servermode.py b/tests/test_dynamodb_v20111205/test_servermode.py new file mode 100644 index 000000000000..2c2e3707a92b --- /dev/null +++ b/tests/test_dynamodb_v20111205/test_servermode.py @@ -0,0 +1,70 @@ +import json +import sure # noqa # pylint: disable=unused-import +import requests + +from moto import settings +from unittest import SkipTest +from uuid import uuid4 + +""" +Test the different server responses +Docs: +https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Appendix.APIv20111205.html +""" + + +def test_table_list(): + if not settings.TEST_SERVER_MODE: + raise SkipTest("Only run test with external server") + headers = { + "X-Amz-Target": "DynamoDB_20111205.ListTables", + "Host": "dynamodb.us-east-1.amazonaws.com", + } + requests.post(settings.test_server_mode_endpoint() + "/moto-api/reset") + res = requests.get(settings.test_server_mode_endpoint(), headers=headers) + res.status_code.should.equal(200) + json.loads(res.content).should.equal({"TableNames": []}) + + +def test_create_table(): + if not settings.TEST_SERVER_MODE: + raise SkipTest("Only run test with external server") + + table_name = str(uuid4()) + + headers = { + "X-Amz-Target": "DynamoDB_20111205.CreateTable", + "Content-Type": "application/x-amz-json-1.0", + "AUTHORIZATION": "AWS4-HMAC-SHA256 Credential=ACCESS_KEY/20220226/us-east-1/dynamodb/aws4_request, SignedHeaders=content-type;host;x-amz-date;x-amz-target, Signature=sig", + } + request_body = { + "TableName": table_name, + "KeySchema": { + "HashKeyElement": {"AttributeName": "hkey", "AttributeType": "S"} + }, + "ProvisionedThroughput": {"ReadCapacityUnits": 5, "WriteCapacityUnits": 10}, + } + res = requests.post( + settings.test_server_mode_endpoint(), headers=headers, json=request_body + ) + + res = json.loads(res.content)["Table"] + res.should.have.key("CreationDateTime") + del res["CreationDateTime"] + res.should.equal( + { + "KeySchema": { + "HashKeyElement": {"AttributeName": "hkey", "AttributeType": "S"} + }, + "ProvisionedThroughput": {"ReadCapacityUnits": 5, "WriteCapacityUnits": 10}, + "TableName": table_name, + "TableStatus": "ACTIVE", + "ItemCount": 0, + "TableSizeBytes": 0, + } + ) + + headers["X-Amz-Target"] = "DynamoDB_20111205.ListTables" + res = requests.get(settings.test_server_mode_endpoint(), headers=headers) + res = json.loads(res.content) + table_name.should.be.within(res["TableNames"]) diff --git a/tests/test_dynamodbstreams/test_dynamodbstreams.py b/tests/test_dynamodbstreams/test_dynamodbstreams.py index bb18d1d8c183..123b74505a03 100644 --- a/tests/test_dynamodbstreams/test_dynamodbstreams.py +++ b/tests/test_dynamodbstreams/test_dynamodbstreams.py @@ -3,7 +3,7 @@ import pytest import boto3 -from moto import mock_dynamodb2, mock_dynamodbstreams +from moto import mock_dynamodb, mock_dynamodbstreams class TestCore: @@ -11,7 +11,7 @@ class TestCore: mocks = [] def setup(self): - self.mocks = [mock_dynamodb2(), mock_dynamodbstreams()] + self.mocks = [mock_dynamodb(), mock_dynamodbstreams()] for m in self.mocks: m.start() @@ -197,7 +197,7 @@ class TestEdges: mocks = [] def setup(self): - self.mocks = [mock_dynamodb2(), mock_dynamodbstreams()] + self.mocks = [mock_dynamodb(), mock_dynamodbstreams()] for m in self.mocks: m.start()