Fix a paginator bug involving optional tokens.

botocore/paginate.py

@@ -200,9 +200,10 @@ def _inject_starting_params(self, op_kwargs):
 
     def _inject_token_into_kwargs(self, op_kwargs, next_token):
         for name, token in next_token.items():
-            if token is None or token == 'None':
-                continue
-            op_kwargs[name] = token
+            if (token is not None) and (token != 'None'):
+                op_kwargs[name] = token
+            elif name in op_kwargs:
+                del op_kwargs[name]
 
     def _handle_first_request(self, parsed, primary_result_key,
                               starting_truncation):

tests/unit/test_paginate.py

@@ -368,6 +368,54 @@ def test_s3_list_multipart_uploads(self):
              ])
 
 
+class TestOptionalTokens(unittest.TestCase):
+    """
+    Tests a paginator with an optional output token.
+
+    The Route53 ListResourceRecordSets paginator includes three output tokens,
+    one of which only appears in certain records. If this gets left in the
+    request params from a previous page, the API will skip over a record.
+
+    """
+    def setUp(self):
+        self.method = mock.Mock()
+        # This is based on Route53 pagination.
+        self.paginate_config = {
+            "output_token": ["NextRecordName",
+                             "NextRecordType",
+                             "NextRecordIdentifier"],
+            "input_token": ["StartRecordName",
+                            "StartRecordType",
+                            "StartRecordIdentifier"],
+            "result_key": 'Foo',
+        }
+        self.paginator = Paginator(self.method, self.paginate_config)
+
+    def test_clean_token(self):
+        responses = [
+            {"Foo": [1],
+             "IsTruncated": True,
+             "NextRecordName": "aaa.example.com",
+             "NextRecordType": "A",
+             "NextRecordIdentifier": "id"},
+            {"Foo": [2],
+             "IsTruncated": True,
+             "NextRecordName": "bbb.example.com",
+             "NextRecordType": "A"},
+            {"Foo": [3],
+             "IsTruncated": False},
+        ]
+        self.method.side_effect = responses
+        list(self.paginator.paginate())
+        self.assertEqual(
+            self.method.call_args_list,
+            [mock.call(),
+             mock.call(StartRecordName='aaa.example.com', StartRecordType='A',
+                       StartRecordIdentifier='id'),
+             mock.call(StartRecordName='bbb.example.com', StartRecordType='A')
+             ])
+
+
 class TestKeyIterators(unittest.TestCase):
     def setUp(self):
         self.method = mock.Mock()