Merge #957: make selection of ioauth input more clever

8c3ae11 fix maker selection of ioauth input with expired timelocked addresses (undeath)
c2312a4 clean up timestamp_to_time_number() (undeath)

jmclient/jmclient/maker.py

@@ -107,9 +107,8 @@ def reject(msg):
         self.wallet_service.save_wallet()
         # Construct data for auth request back to taker.
         # Need to choose an input utxo pubkey to sign with
-        # (no longer using the coinjoin pubkey from 0.2.0)
-        # Just choose the first utxo in self.utxos and retrieve key from wallet.
-        auth_address = utxos[list(utxos.keys())[0]]['address']
+        # Just choose the first utxo in utxos and retrieve key from wallet.
+        auth_address = next(iter(utxos.values()))['address']
         auth_key = self.wallet_service.get_key_from_addr(auth_address)
         auth_pub = btc.privkey_to_pubkey(auth_key)
         # kphex was auto-converted by @hexbin but we actually need to sign the
@@ -262,7 +261,7 @@ def create_my_orders(self):
         """
 
     @abc.abstractmethod
-    def oid_to_order(self, cjorder, oid, amount):
+    def oid_to_order(self, cjorder, amount):
         """Must convert an order with an offer/order id
         into a set of utxos to fill the order.
         Also provides the output addresses for the Taker.

jmclient/jmclient/wallet.py

@@ -23,7 +23,7 @@
 from .configure import jm_single
 from .blockchaininterface import INF_HEIGHT
 from .support import select_gradual, select_greedy, select_greediest, \
-    select
+    select, NotEnoughFundsException
 from .cryptoengine import TYPE_P2PKH, TYPE_P2SH_P2WPKH,\
     TYPE_P2WPKH, TYPE_TIMELOCK_P2WSH, TYPE_SEGWIT_WALLET_FIDELITY_BONDS,\
     TYPE_WATCHONLY_FIDELITY_BONDS, TYPE_WATCHONLY_TIMELOCK_P2WSH, TYPE_WATCHONLY_P2WPKH,\
@@ -662,7 +662,8 @@ def process_new_tx(self, txd, height=None):
         return (removed_utxos, added_utxos)
 
     def select_utxos(self, mixdepth, amount, utxo_filter=None,
-                      select_fn=None, maxheight=None, includeaddr=False):
+                     select_fn=None, maxheight=None, includeaddr=False,
+                     require_auth_address=False):
         """
         Select a subset of available UTXOS for a given mixdepth whose value is
         greater or equal to amount. If `includeaddr` is True, adds an `address`
@@ -674,10 +675,17 @@ def select_utxos(self, mixdepth, amount, utxo_filter=None,
             amount: int, total minimum amount of all selected utxos
             utxo_filter: list of (txid, index), utxos not to select
             maxheight: only select utxos with blockheight <= this.
+            require_auth_address: if True, output utxos must include a
+                standard wallet address. The first item of the output dict is
+                guaranteed to be a suitable utxo. Result will be empty if no
+                such utxo set could be found.
 
         returns:
             {(txid, index): {'script': bytes, 'path': tuple, 'value': int}}
 
+        raises:
+            NotEnoughFundsException: if mixdepth does not have utxos with
+                enough value to satisfy amount
         """
         assert isinstance(mixdepth, numbers.Integral)
         assert isinstance(amount, numbers.Integral)
@@ -688,14 +696,33 @@ def select_utxos(self, mixdepth, amount, utxo_filter=None,
             assert len(i) == 2
             assert isinstance(i[0], bytes)
             assert isinstance(i[1], numbers.Integral)
-        ret = self._utxos.select_utxos(
+        utxos = self._utxos.select_utxos(
             mixdepth, amount, utxo_filter, select_fn, maxheight=maxheight)
 
-        for data in ret.values():
+        total_value = 0
+        standard_utxo = None
+        for key, data in utxos.items():
+            if self.is_standard_wallet_script(data['path']):
+                standard_utxo = key
+            total_value += data['value']
             data['script'] = self.get_script_from_path(data['path'])
             if includeaddr:
                 data["address"] = self.get_address_from_path(data["path"])
-        return ret
+
+        if require_auth_address and not standard_utxo:
+            # try to select more utxos, hoping for a standard one
+            try:
+                return self.select_utxos(
+                    mixdepth, total_value + 1, utxo_filter, select_fn,
+                    maxheight, includeaddr, require_auth_address)
+            except NotEnoughFundsException:
+                # recursive utxo selection was unsuccessful, give up
+                return {}
+        elif require_auth_address:
+            utxos = collections.OrderedDict(utxos)
+            utxos.move_to_end(standard_utxo, last=False)
+
+        return utxos
 
     def disable_utxo(self, txid, index, disable=True):
         self._utxos.disable_utxo(txid, index, disable)
@@ -889,6 +916,16 @@ def yield_imported_paths(self, mixdepth):
         """
         return iter([])
 
+    def is_standard_wallet_script(self, path):
+        """
+        Check if the path's script is of the same type as the standard wallet
+        key type.
+
+        return:
+            bool
+        """
+        raise NotImplementedError()
+
     def is_known_addr(self, addr):
         """
         Check if address is known to belong to this wallet.
@@ -1725,6 +1762,12 @@ def _get_key_from_path(self, path):
     def _is_imported_path(cls, path):
         return len(path) == 3 and path[0] == cls._IMPORTED_ROOT_PATH
 
+    def is_standard_wallet_script(self, path):
+        if self._is_imported_path(path):
+            engine = self._get_key_from_path(path)[1]
+            return engine == self._ENGINE
+        return super().is_standard_wallet_script(path)
+
     def path_repr_to_path(self, pathstr):
         spath = pathstr.encode('ascii').split(b'/')
         if not self._is_imported_path(spath):
@@ -2030,6 +2073,9 @@ def _get_key_from_path(self, path):
     def _is_my_bip32_path(self, path):
         return path[0] == self._key_ident
 
+    def is_standard_wallet_script(self, path):
+        return self._is_my_bip32_path(path)
+
     def get_new_script(self, mixdepth, address_type):
         if self.disable_new_scripts:
             raise RuntimeError("Obtaining new wallet addresses "
@@ -2221,13 +2267,10 @@ def _time_number_to_timestamp(cls, timenumber):
         return timegm(datetime(year, month, *cls.TIMELOCK_DAY_AND_SHORTER).timetuple())
 
     @classmethod
-    def timestamp_to_time_number(cls, timestamp):
+    def datetime_to_time_number(cls, dt):
         """
         converts a datetime object to a time number
         """
-        #workaround for the year 2038 problem on 32 bit systems
-        #see https://stackoverflow.com/questions/10588027/converting-timestamps-larger-than-maxint-into-datetime-objects
-        dt = datetime.utcfromtimestamp(0) + timedelta(seconds=timestamp)
         if (dt.month - cls.TIMELOCK_EPOCH_MONTH) % cls.TIMENUMBER_UNIT != 0:
             raise ValueError()
         day_and_shorter_tuple = (dt.day, dt.hour, dt.minute, dt.second, dt.microsecond)
@@ -2239,6 +2282,16 @@ def timestamp_to_time_number(cls, timestamp):
             raise ValueError("datetime out of range")
         return timenumber
 
+    @classmethod
+    def timestamp_to_time_number(cls, timestamp):
+        """
+        converts a unix timestamp to a time number
+        """
+        #workaround for the year 2038 problem on 32 bit systems
+        #see https://stackoverflow.com/questions/10588027/converting-timestamps-larger-than-maxint-into-datetime-objects
+        dt = datetime.utcfromtimestamp(0) + timedelta(seconds=timestamp)
+        return cls.datetime_to_time_number(dt)
+
     @classmethod
     def is_timelocked_path(cls, path):
         return len(path) > 4 and path[4] == cls.BIP32_TIMELOCK_ID
@@ -2249,6 +2302,11 @@ def _get_key_ident(self):
         pub = engine.privkey_to_pubkey(priv)
         return sha256(sha256(pub).digest()).digest()[:3]
 
+    def is_standard_wallet_script(self, path):
+        if self.is_timelocked_path(path):
+            return False
+        return super().is_standard_wallet_script(path)
+
     @classmethod
     def get_xpub_from_fidelity_bond_master_pub_key(cls, mpk):
         if mpk.startswith(cls._BIP32_PUBKEY_PREFIX):

jmclient/jmclient/wallet_service.py

@@ -864,16 +864,17 @@ def minconfs_to_maxheight(self, minconfs):
             return self.current_blockheight - minconfs + 1
 
     def select_utxos(self, mixdepth, amount, utxo_filter=None, select_fn=None,
-                     minconfs=None, includeaddr=False):
+                     minconfs=None, includeaddr=False, require_auth_address=False):
         """ Request utxos from the wallet in a particular mixdepth to satisfy
         a certain total amount, optionally set the selector function (or use
         the currently configured function set by the wallet, and optionally
         require a minimum of minconfs confirmations (default none means
         unconfirmed are allowed).
         """
-        return self.wallet.select_utxos(mixdepth, amount, utxo_filter=utxo_filter,
-                select_fn=select_fn, maxheight=self.minconfs_to_maxheight(minconfs),
-                includeaddr=includeaddr)
+        return self.wallet.select_utxos(
+            mixdepth, amount, utxo_filter=utxo_filter, select_fn=select_fn,
+            maxheight=self.minconfs_to_maxheight(minconfs),
+            includeaddr=includeaddr, require_auth_address=require_auth_address)
 
     def get_balance_by_mixdepth(self, verbose=True,
                                 include_disabled=False,

jmclient/jmclient/wallet_utils.py

@@ -5,7 +5,6 @@
 import sqlite3
 import sys
 from datetime import datetime, timedelta
-from calendar import timegm
 from optparse import OptionParser
 from numbers import Integral
 from collections import Counter
@@ -1226,8 +1225,7 @@ def wallet_gettimelockaddress(wallet, locktime_string):
     m = FidelityBondMixin.FIDELITY_BOND_MIXDEPTH
     address_type = FidelityBondMixin.BIP32_TIMELOCK_ID
     lock_datetime = datetime.strptime(locktime_string, "%Y-%m")
-    timenumber = FidelityBondMixin.timestamp_to_time_number(timegm(
-        lock_datetime.timetuple()))
+    timenumber = FidelityBondMixin.datetime_to_time_number(lock_datetime)
     index = timenumber
 
     path = wallet.get_path(m, address_type, index, timenumber)

jmclient/jmclient/yieldgenerator.py

@@ -22,6 +22,11 @@
 
 MAX_MIX_DEPTH = 5
 
+
+class NoIoauthInputException(Exception):
+    pass
+
+
 class YieldGenerator(Maker):
     """A maker for the purposes of generating a yield from held
     bitcoins, offering from the maximum mixdepth and trying to offer
@@ -172,9 +177,18 @@ def oid_to_order(self, offer, amount):
         if not filtered_mix_balance:
             return None, None, None
         jlog.debug('mix depths that have enough = ' + str(filtered_mix_balance))
-        mixdepth = self.select_input_mixdepth(filtered_mix_balance, offer, amount)
-        if mixdepth is None:
+
+        try:
+            mixdepth, utxos = self._get_order_inputs(
+                filtered_mix_balance, offer, required_amount)
+        except NoIoauthInputException:
+            jlog.error(
+                'unable to fill order, no suitable IOAUTH UTXO found. In '
+                'order to spend coins (UTXOs) from a mixdepth using coinjoin,'
+                ' there needs to be at least one standard wallet UTXO (not '
+                'fidelity bond or different address type).')
             return None, None, None
+
         jlog.info('filling offer, mixdepth=' + str(mixdepth) + ', amount=' + str(amount))
 
         cj_addr = self.select_output_address(mixdepth, offer, amount)
@@ -183,12 +197,35 @@ def oid_to_order(self, offer, amount):
         jlog.info('sending output to address=' + str(cj_addr))
 
         change_addr = self.wallet_service.get_internal_addr(mixdepth)
-
-        utxos = self.wallet_service.select_utxos(mixdepth, required_amount,
-                                        minconfs=1, includeaddr=True)
-
         return utxos, cj_addr, change_addr
 
+    def _get_order_inputs(self, filtered_mix_balance, offer, required_amount):
+        """
+        Select inputs from some applicable mixdepth that has a utxo suitable
+        for ioauth.
+
+        params:
+            filtered_mix_balance: see get_available_mixdepths() output
+            offer: offer dict
+            required_amount: int, total inputs value in sat
+
+        returns:
+            mixdepth, utxos (int, dict)
+
+        raises:
+            NoIoauthInputException: if no provided mixdepth has a suitable utxo
+        """
+        while filtered_mix_balance:
+            mixdepth = self.select_input_mixdepth(
+                filtered_mix_balance, offer, required_amount)
+            utxos = self.wallet_service.select_utxos(
+                mixdepth, required_amount, minconfs=1, includeaddr=True,
+                require_auth_address=True)
+            if utxos:
+                return mixdepth, utxos
+            filtered_mix_balance.pop(mixdepth)
+        raise NoIoauthInputException()
+
     def on_tx_confirmed(self, offer, txid, confirmations):
         if offer["cjaddr"] in self.tx_unconfirm_timestamp:
             confirm_time = int(time.time()) - self.tx_unconfirm_timestamp[

jmclient/test/test_client_protocol.py

@@ -123,7 +123,7 @@ def create_my_orders(self):
             'txfee': 0
         }]
 
-    def oid_to_order(self, cjorder, oid, amount):
+    def oid_to_order(self, cjorder, amount):
         # utxos, cj_addr, change_addr
         return [], '', ''
 

jmclient/test/test_taker.py

@@ -79,7 +79,8 @@ def get_utxos_by_mixdepth(self, include_disabled=False, verbose=True,
         return retval
 
     def select_utxos(self, mixdepth, amount, utxo_filter=None, select_fn=None,
-                     maxheight=None, includeaddr=False):
+                     maxheight=None, includeaddr=False,
+                     require_auth_address=False):
         if amount > self.get_balance_by_mixdepth()[mixdepth]:
             raise NotEnoughFundsException(amount, self.get_balance_by_mixdepth()[mixdepth])
         # comment as for get_utxos_by_mixdepth:

jmclient/test/test_wallet.py

@@ -1,5 +1,5 @@
 '''Wallet functionality tests.'''
-
+import datetime
 import os
 import json
 from binascii import hexlify, unhexlify
@@ -906,6 +906,35 @@ def test_create_wallet(setup_wallet, password, wallet_cls):
     os.remove(wallet_name)
     btc.select_chain_params("bitcoin/regtest")
 
+
+@pytest.mark.parametrize('wallet_cls', [
+    SegwitLegacyWallet, SegwitWallet, SegwitWalletFidelityBonds
+])
+def test_is_standard_wallet_script(setup_wallet, wallet_cls):
+    storage = VolatileStorage()
+    wallet_cls.initialize(
+        storage, get_network(), max_mixdepth=0)
+    wallet = wallet_cls(storage)
+    script = wallet.get_new_script(0, 1)
+    assert wallet.is_known_script(script)
+    path = wallet.script_to_path(script)
+    assert wallet.is_standard_wallet_script(path)
+
+
+def test_is_standard_wallet_script_nonstandard(setup_wallet):
+    storage = VolatileStorage()
+    SegwitWalletFidelityBonds.initialize(
+        storage, get_network(), max_mixdepth=0)
+    wallet = SegwitWalletFidelityBonds(storage)
+    import_path = wallet.import_private_key(
+        0, 'cRAGLvPmhpzJNgdMT4W2gVwEW3fusfaDqdQWM2vnWLgXKzCWKtcM')
+    assert wallet.is_standard_wallet_script(import_path)
+    ts = wallet.datetime_to_time_number(
+        datetime.datetime.strptime("2021-07", "%Y-%m"))
+    tl_path = wallet.get_path(0, wallet.BIP32_TIMELOCK_ID, 0, ts)
+    assert not wallet.is_standard_wallet_script(tl_path)
+
+
 @pytest.fixture(scope='module')
 def setup_wallet(request):
     load_test_config()