Merge pull request #279 from KitHat/ST-602

ST-602 -- add pagination on plugin level

sovtoken/sovtoken/constants.py

@@ -27,3 +27,4 @@
 
 UTXO_LIMIT = 1000
 NEXT_SEQNO = "next"
+FROM_SEQNO = "from"

sovtoken/sovtoken/main.py

@@ -86,7 +86,7 @@ def register_req_handlers(node):
                                                         node.write_req_validator))
     node.write_manager.register_req_handler(MintHandler(node.db_manager,
                                                         node.write_req_validator))
-    node.read_manager.register_req_handler(GetUtxoHandler(node.db_manager))
+    node.read_manager.register_req_handler(GetUtxoHandler(node.db_manager, node.config.MSG_LEN_LIMIT))
 
 
 def register_batch_handlers(node):

sovtoken/sovtoken/messages/txn_validator.py

@@ -1,14 +1,15 @@
 from plenum.common.constants import TXN_TYPE
 from plenum.common.exceptions import InvalidClientRequest
-from plenum.common.messages.fields import IterableField
+from plenum.common.messages.fields import IterableField, TxnSeqNoField
 from plenum.common.request import Request
 
-from sovtoken.constants import MINT_PUBLIC, XFER_PUBLIC, GET_UTXO, INPUTS, SIGS, ADDRESS, OUTPUTS
+from sovtoken.constants import MINT_PUBLIC, XFER_PUBLIC, GET_UTXO, INPUTS, SIGS, ADDRESS, OUTPUTS, FROM_SEQNO
 from sovtoken.messages.fields import PublicOutputField, PublicOutputsField, PublicInputsField
 
 PUBLIC_OUTPUT_VALIDATOR = IterableField(PublicOutputField())
 PUBLIC_OUTPUTS_VALIDATOR = PublicOutputsField()
 PUBLIC_INPUTS_VALIDATOR = PublicInputsField()
+FROM_VALIDATOR = TxnSeqNoField()
 
 
 def outputs_validate(request: Request):
@@ -40,6 +41,16 @@ def inputs_validate(request: Request):
     return PUBLIC_INPUTS_VALIDATOR.validate(operation[INPUTS])
 
 
+def from_validate(request: Request):
+    operation = request.operation
+    if FROM_SEQNO in operation:
+        from_seqno = operation[FROM_SEQNO]
+        error = FROM_VALIDATOR.validate(from_seqno)
+        if error:
+            error = "'{}' validation failed: {}".format(FROM_SEQNO, error)
+        return error
+
+
 def address_validate(request: Request):
     operation = request.operation
     if ADDRESS not in operation:
@@ -77,4 +88,9 @@ def txn_xfer_public_validate(request: Request):
 def txt_get_utxo_validate(request: Request):
     operation = request.operation
     if operation[TXN_TYPE] == GET_UTXO:
-        return address_validate(request)
+        error = address_validate(request)
+        if error:
+            return error
+        else:
+            error = from_validate(request)
+        return error

sovtoken/sovtoken/request_handlers/read_req_handler/get_utxo_handler.py

@@ -5,7 +5,7 @@
 from copy import deepcopy
 
 from sovtoken import TokenTransactions
-from sovtoken.constants import ADDRESS, OUTPUTS, TOKEN_LEDGER_ID, NEXT_SEQNO, UTXO_LIMIT
+from sovtoken.constants import ADDRESS, OUTPUTS, TOKEN_LEDGER_ID, NEXT_SEQNO, UTXO_LIMIT, FROM_SEQNO
 from sovtoken.messages.txn_validator import txt_get_utxo_validate
 from sovtoken.request_handlers.token_utils import TokenStaticHelper
 from sovtoken.types import Output
@@ -19,12 +19,12 @@
 from plenum.server.database_manager import DatabaseManager
 from plenum.server.request_handlers.handler_interfaces.read_request_handler import ReadRequestHandler
 from state.trie.pruning_trie import rlp_decode
-from stp_core.config import MSG_LEN_LIMIT
 
 
 class GetUtxoHandler(ReadRequestHandler):
-    def __init__(self, database_manager: DatabaseManager):
+    def __init__(self, database_manager: DatabaseManager, msg_limit):
         super().__init__(database_manager, TokenTransactions.GET_UTXO.value, TOKEN_LEDGER_ID)
+        self._msg_limit = msg_limit
 
     def static_validation(self, request: Request):
         error = txt_get_utxo_validate(request)
@@ -40,6 +40,7 @@ def create_state_key(address: str, seq_no: int) -> bytes:
 
     def get_result(self, request: Request):
         address = request.operation[ADDRESS]
+        from_seqno = request.operation.get(FROM_SEQNO)
         encoded_root_hash = state_roots_serializer.serialize(
             bytes(self.state.committedHeadHash))
         proof, rv = self.state.generate_state_proof_for_keys_with_prefix(address,
@@ -69,6 +70,12 @@ def get_result(self, request: Request):
 
         utxos = outputs.sorted_list
         next_seqno = None
+        if from_seqno:
+            idx = next((idx for utxo, idx in zip(utxos, range(len(utxos))) if utxo.seqNo >= from_seqno), None)
+            if idx:
+                utxos = utxos[idx:]
+            else:
+                utxos = []
         if len(utxos) > UTXO_LIMIT:
             next_seqno = utxos[UTXO_LIMIT].seqNo
             utxos = utxos[:UTXO_LIMIT]
@@ -82,6 +89,6 @@ def get_result(self, request: Request):
         if proof:
             res_sub = deepcopy(result)
             res_sub[STATE_PROOF] = proof
-            if len(json.dumps(res_sub)) <= MSG_LEN_LIMIT:
+            if len(json.dumps(res_sub)) <= self._msg_limit:
                 result = res_sub
         return result

sovtoken/sovtoken/test/helpers/helper_request.py

@@ -11,7 +11,7 @@
 from plenum.common.constants import TXN_TYPE, CURRENT_PROTOCOL_VERSION, GET_TXN, DATA
 from plenum.common.request import Request
 from plenum.common.types import f
-from sovtoken.constants import ADDRESS, AMOUNT
+from sovtoken.constants import ADDRESS, AMOUNT, FROM_SEQNO
 
 SEC_PER_DAY = 24 * 60 * 60
 
@@ -44,13 +44,17 @@ def __init__(
         self._client_wallet = client_wallet
         self._steward_wallet = steward_wallet
 
-    def get_utxo(self, address):
+    def get_utxo(self, address, from_seqno=None):
         """ Builds a get_utxo request. """
 
         get_utxo_request_future = build_get_payment_sources_request(self._client_wallet_handle, VALID_IDENTIFIER, address)
         get_utxo_request = self._looper.loop.run_until_complete(get_utxo_request_future)[0]
         get_utxo_request = self._sdk.sdk_json_to_request_object(json.loads(get_utxo_request))
 
+        # TODO: as soon as SDK gets implemented "from", remove this two lines and pass it to the builder
+        if from_seqno:
+            get_utxo_request.operation[FROM_SEQNO] = from_seqno
+
         return get_utxo_request
 
     def get_txn(self, ledger_id, seq_no):

sovtoken/sovtoken/test/req_handlers/get_utxo_req_handler/conftest.py

@@ -3,6 +3,7 @@
 import pytest
 
 from sovtoken import TOKEN_LEDGER_ID, TokenTransactions
+from sovtoken.constants import FROM_SEQNO
 from sovtoken.request_handlers.read_req_handler.get_utxo_handler import GetUtxoHandler
 from sovtoken.request_handlers.token_utils import TokenStaticHelper
 from sovtoken.test.helper import libsovtoken_address_to_address
@@ -11,10 +12,12 @@
 
 from indy.payment import build_get_payment_sources_request
 
+from stp_core.config import MSG_LEN_LIMIT
+
 
 @pytest.fixture(scope="module")
 def get_utxo_handler(db_manager, bls_store):
-    return GetUtxoHandler(database_manager=db_manager)
+    return GetUtxoHandler(database_manager=db_manager, msg_limit=MSG_LEN_LIMIT)
 
 
 @pytest.fixture()
@@ -30,4 +33,12 @@ def get_utxo_request(looper, payment_address, wallet):
 def insert_over_thousand_utxos(db_manager, payment_address):
     token_state = db_manager.get_state(TOKEN_LEDGER_ID)
     for i in range(1200):
-        token_state.set(TokenStaticHelper.create_state_key(libsovtoken_address_to_address(payment_address), i), str(i).encode())
+        token_state.set(TokenStaticHelper.create_state_key(libsovtoken_address_to_address(payment_address), i), str(i).encode())
+
+
+@pytest.fixture(scope="module")
+def insert_utxos_after_gap(db_manager, payment_address):
+    token_state = db_manager.get_state(TOKEN_LEDGER_ID)
+    for i in range(1300, 2400):
+        token_state.set(TokenStaticHelper.create_state_key(libsovtoken_address_to_address(payment_address), i), str(i).encode())
+    return 1300

sovtoken/sovtoken/test/req_handlers/get_utxo_req_handler/test_get_utxo_get_result_big_number_of_utxos.py

@@ -1,4 +1,4 @@
-from sovtoken.constants import OUTPUTS
+from sovtoken.constants import OUTPUTS, UTXO_LIMIT, FROM_SEQNO
 from sovtoken.test.helper import libsovtoken_address_to_address
 
 from plenum.common.constants import STATE_PROOF
@@ -8,8 +8,43 @@ def test_get_utxo_request_has_utxos(get_utxo_request, get_utxo_handler, payment_
     result = get_utxo_handler.get_result(get_utxo_request)
     assert result[STATE_PROOF]
     assert result[OUTPUTS]
-    assert len(result[OUTPUTS]) == 1000
-    for i in range(1000):
+    assert len(result[OUTPUTS]) == UTXO_LIMIT
+    for i in range(UTXO_LIMIT):
         assert result[OUTPUTS][i].address == libsovtoken_address_to_address(payment_address)
         assert result[OUTPUTS][i].seqNo == i
         assert result[OUTPUTS][i].amount == i
+
+FROM_SHIFT = 200
+
+def test_get_utxo_request_has_utxos_with_from(get_utxo_request, get_utxo_handler, payment_address, insert_over_thousand_utxos):
+    get_utxo_request.operation[FROM_SEQNO] = FROM_SHIFT
+    result = get_utxo_handler.get_result(get_utxo_request)
+    assert result[STATE_PROOF]
+    assert result[OUTPUTS]
+    assert len(result[OUTPUTS]) == UTXO_LIMIT
+    for i in range(FROM_SHIFT, FROM_SHIFT+UTXO_LIMIT):
+        assert result[OUTPUTS][i-FROM_SHIFT].address == libsovtoken_address_to_address(payment_address)
+        assert result[OUTPUTS][i-FROM_SHIFT].seqNo == i
+        assert result[OUTPUTS][i-FROM_SHIFT].amount == i
+
+
+def test_get_utxo_request_has_utxos_with_from_bigger_than_utxos(get_utxo_request, get_utxo_handler, payment_address, insert_over_thousand_utxos):
+    get_utxo_request.operation[FROM_SEQNO] = 13000
+    result = get_utxo_handler.get_result(get_utxo_request)
+    assert result[STATE_PROOF]
+    assert result[OUTPUTS] == []
+
+
+def test_get_utxo_request_has_utxos_with_from_between_the_numbers(get_utxo_request, get_utxo_handler, payment_address,
+                                                                  insert_over_thousand_utxos, insert_utxos_after_gap):
+    gap = insert_utxos_after_gap
+    # there is a gap between 1200 and 1300 seqno's for this payment address
+    get_utxo_request.operation[FROM_SEQNO] = gap - 50
+    result = get_utxo_handler.get_result(get_utxo_request)
+    # 2300 utxos is too many for state proof, not checking it
+    assert result[OUTPUTS]
+    assert len(result[OUTPUTS]) == UTXO_LIMIT
+    for i in range(gap, gap+UTXO_LIMIT):
+        assert result[OUTPUTS][i-gap].address == libsovtoken_address_to_address(payment_address)
+        assert result[OUTPUTS][i-gap].seqNo == i
+        assert result[OUTPUTS][i-gap].amount == i

sovtoken/sovtoken/test/req_handlers/get_utxo_req_handler/test_get_utxo_static_validation.py

@@ -3,7 +3,7 @@
 from base58 import b58encode_check
 
 import pytest
-from sovtoken.constants import ADDRESS
+from sovtoken.constants import ADDRESS, FROM_SEQNO
 
 from plenum.common.exceptions import InvalidClientRequest, UnknownIdentifier
 from plenum.common.util import randomString
@@ -36,3 +36,10 @@ def test_get_utxo_request_invalid_vk_length(get_utxo_handler, get_utxo_request):
     with pytest.raises(InvalidClientRequest,
                        match="Not a valid address as it resolves to 31 byte verkey"):
         get_utxo_handler.static_validation(get_utxo_request)
+
+
+def test_get_utxo_request_invalid_from_seqno(get_utxo_handler, get_utxo_request):
+    operation = get_utxo_request.operation
+    operation[FROM_SEQNO] = "next"
+    with pytest.raises(InvalidClientRequest, match="'{}' validation failed".format(FROM_SEQNO)):
+        get_utxo_handler.static_validation(get_utxo_request)

sovtoken/sovtoken/test/test_get_utxo.py

@@ -7,10 +7,12 @@
 from sovtoken.test.helper import libsovtoken_address_to_address
 from sovtoken.test.helpers.helper_general import utxo_from_addr_and_seq_no
 
+from plenum.common.constants import STATE_PROOF
 from plenum.common.exceptions import RequestNackedException
 from plenum.common.txn_util import get_seq_no, get_payload_data
 from plenum.common.util import randomString
-from sovtoken.constants import OUTPUTS, ADDRESS, AMOUNT, PAYMENT_ADDRESS, TOKEN_LEDGER_ID, NEXT_SEQNO, SEQNO
+from sovtoken.constants import OUTPUTS, ADDRESS, AMOUNT, PAYMENT_ADDRESS, TOKEN_LEDGER_ID, NEXT_SEQNO, SEQNO, \
+    UTXO_LIMIT
 
 
 @pytest.fixture
@@ -130,7 +132,7 @@ def test_get_more_then_thousand_utxos(helpers, addresses, nodeSetWithIntegratedT
     states = [n.db_manager.get_state(TOKEN_LEDGER_ID) for n in nodeSetWithIntegratedTokenPlugin]
     utxos = []
 
-    for i in range(1200):
+    for i in range(UTXO_LIMIT+200):
         amount = randint(1, 5)
         key = TokenStaticHelper.create_state_key(libsovtoken_address_to_address(address_2), i+5)
         utxos.append((key, amount))
@@ -141,7 +143,43 @@ def test_get_more_then_thousand_utxos(helpers, addresses, nodeSetWithIntegratedT
     responses = helpers.sdk.send_and_check_request_objects([request])
     for response in responses:
         result = response[1]['result']
-        assert len(result[OUTPUTS]) == 1000
+        assert len(result[OUTPUTS]) == UTXO_LIMIT
         for output in result[OUTPUTS]:
             assert (TokenStaticHelper.create_state_key(output[ADDRESS], output[SEQNO]), output[AMOUNT]) in utxos
         assert result.get(NEXT_SEQNO, None)
+
+
+def test_get_more_then_thousand_utxos_with_from(helpers, addresses, nodeSetWithIntegratedTokenPlugin):
+    """
+    test if we send more have more than a thousand of UTXO's we will still receive a response.
+    """
+
+    address_1, address_2 = addresses
+
+    states = [n.db_manager.get_state(TOKEN_LEDGER_ID) for n in nodeSetWithIntegratedTokenPlugin]
+    utxos = []
+
+    for i in range(UTXO_LIMIT+200):
+        amount = randint(1, 5)
+        seq_no = i+5
+        key = TokenStaticHelper.create_state_key(libsovtoken_address_to_address(address_2), seq_no)
+        utxos.append((key, amount, seq_no))
+        for state in states:
+            state.set(key, str(amount).encode())
+
+    # NB: this transaction is needed just to update bls_store with new root hash
+    total = 1000
+    outputs = [{"address": address_1, "amount": total}]
+    mint_result = helpers.general.do_mint(outputs)
+
+    shift = 50
+    request = helpers.request.get_utxo(address_2, utxos[shift][2])
+    responses = helpers.sdk.send_and_check_request_objects([request])
+    utxos = utxos[shift:shift+UTXO_LIMIT]
+    for response in responses:
+        result = response[1]['result']
+        assert result[STATE_PROOF]
+        assert len(result[OUTPUTS]) == UTXO_LIMIT
+        for output in result[OUTPUTS]:
+            assert (TokenStaticHelper.create_state_key(output[ADDRESS], output[SEQNO]), output[AMOUNT], output[SEQNO]) in utxos
+        assert result.get(NEXT_SEQNO)