integrate PSBT support natively. WIP

electrum/address_synchronizer.py

@@ -29,9 +29,9 @@
 from typing import TYPE_CHECKING, Dict, Optional, Set, Tuple, NamedTuple, Sequence
 
 from . import bitcoin
-from .bitcoin import COINBASE_MATURITY, TYPE_ADDRESS, TYPE_PUBKEY
+from .bitcoin import COINBASE_MATURITY
 from .util import profiler, bfh, TxMinedInfo
-from .transaction import Transaction, TxOutput
+from .transaction import Transaction, TxOutput, TxInput, PartialTxInput, TxOutpoint
 from .synchronizer import Synchronizer
 from .verifier import SPV
 from .blockchain import hash_header
@@ -125,27 +125,21 @@ def get_address_history_len(self, addr: str) -> int:
         """Return number of transactions where address is involved."""
         return len(self._history_local.get(addr, ()))
 
-    def get_txin_address(self, txi) -> Optional[str]:
-        addr = txi.get('address')
-        if addr and addr != "(pubkey)":
-            return addr
-        prevout_hash = txi.get('prevout_hash')
-        prevout_n = txi.get('prevout_n')
+    def get_txin_address(self, txin: TxInput) -> Optional[str]:
+        if isinstance(txin, PartialTxInput):
+            if txin.address:
+                return txin.address
+        prevout_hash = txin.prevout.txid.hex()
+        prevout_n = txin.prevout.out_idx
         for addr in self.db.get_txo_addresses(prevout_hash):
             l = self.db.get_txo_addr(prevout_hash, addr)
             for n, v, is_cb in l:
                 if n == prevout_n:
                     return addr
         return None
 
-    def get_txout_address(self, txo: TxOutput):
-        if txo.type == TYPE_ADDRESS:
-            addr = txo.address
-        elif txo.type == TYPE_PUBKEY:
-            addr = bitcoin.public_key_to_p2pkh(bfh(txo.address))
-        else:
-            addr = None
-        return addr
+    def get_txout_address(self, txo: TxOutput) -> Optional[str]:
+        return txo.address
 
     def load_unverified_transactions(self):
         # review transactions that are in the history
@@ -183,7 +177,7 @@ def add_address(self, address):
         if self.synchronizer:
             self.synchronizer.add(address)
 
-    def get_conflicting_transactions(self, tx_hash, tx, include_self=False):
+    def get_conflicting_transactions(self, tx_hash, tx: Transaction, include_self=False):
         """Returns a set of transaction hashes from the wallet history that are
         directly conflicting with tx, i.e. they have common outpoints being
         spent with tx.
@@ -194,10 +188,10 @@ def get_conflicting_transactions(self, tx_hash, tx, include_self=False):
         conflicting_txns = set()
         with self.transaction_lock:
             for txin in tx.inputs():
-                if txin['type'] == 'coinbase':
+                if txin.is_coinbase():
                     continue
-                prevout_hash = txin['prevout_hash']
-                prevout_n = txin['prevout_n']
+                prevout_hash = txin.prevout.txid.hex()
+                prevout_n = txin.prevout.out_idx
                 spending_tx_hash = self.db.get_spent_outpoint(prevout_hash, prevout_n)
                 if spending_tx_hash is None:
                     continue
@@ -213,7 +207,7 @@ def get_conflicting_transactions(self, tx_hash, tx, include_self=False):
                     conflicting_txns -= {tx_hash}
             return conflicting_txns
 
-    def add_transaction(self, tx_hash, tx, allow_unrelated=False) -> bool:
+    def add_transaction(self, tx_hash, tx: Transaction, allow_unrelated=False) -> bool:
         """Returns whether the tx was successfully added to the wallet history."""
         assert tx_hash, tx_hash
         assert tx, tx
@@ -226,7 +220,7 @@ def add_transaction(self, tx_hash, tx, allow_unrelated=False) -> bool:
             # BUT we track is_mine inputs in a txn, and during subsequent calls
             # of add_transaction tx, we might learn of more-and-more inputs of
             # being is_mine, as we roll the gap_limit forward
-            is_coinbase = tx.inputs()[0]['type'] == 'coinbase'
+            is_coinbase = tx.inputs()[0].is_coinbase()
             tx_height = self.get_tx_height(tx_hash).height
             if not allow_unrelated:
                 # note that during sync, if the transactions are not properly sorted,
@@ -277,11 +271,11 @@ def add_value_from_prev_output():
                                 self._get_addr_balance_cache.pop(addr, None)  # invalidate cache
                             return
             for txi in tx.inputs():
-                if txi['type'] == 'coinbase':
+                if txi.is_coinbase():
                     continue
-                prevout_hash = txi['prevout_hash']
-                prevout_n = txi['prevout_n']
-                ser = prevout_hash + ':%d' % prevout_n
+                prevout_hash = txi.prevout.txid.hex()
+                prevout_n = txi.prevout.out_idx
+                ser = txi.prevout.to_str()
                 self.db.set_spent_outpoint(prevout_hash, prevout_n, tx_hash)
                 add_value_from_prev_output()
             # add outputs
@@ -310,10 +304,10 @@ def remove_from_spent_outpoints():
             if tx is not None:
                 # if we have the tx, this branch is faster
                 for txin in tx.inputs():
-                    if txin['type'] == 'coinbase':
+                    if txin.is_coinbase():
                         continue
-                    prevout_hash = txin['prevout_hash']
-                    prevout_n = txin['prevout_n']
+                    prevout_hash = txin.prevout.txid.hex()
+                    prevout_n = txin.prevout.out_idx
                     self.db.remove_spent_outpoint(prevout_hash, prevout_n)
             else:
                 # expensive but always works
@@ -572,7 +566,7 @@ def get_local_height(self) -> int:
             return cached_local_height
         return self.network.get_local_height() if self.network else self.db.get('stored_height', 0)
 
-    def add_future_tx(self, tx, num_blocks):
+    def add_future_tx(self, tx: Transaction, num_blocks):
         with self.lock:
             self.add_transaction(tx.txid(), tx)
             self.future_tx[tx.txid()] = num_blocks
@@ -649,9 +643,9 @@ def get_wallet_delta(self, tx: Transaction):
             if self.is_mine(addr):
                 is_mine = True
                 is_relevant = True
-                d = self.db.get_txo_addr(txin['prevout_hash'], addr)
+                d = self.db.get_txo_addr(txin.prevout.txid.hex(), addr)
                 for n, v, cb in d:
-                    if n == txin['prevout_n']:
+                    if n == txin.prevout.out_idx:
                         value = v
                         break
                 else:
@@ -736,23 +730,19 @@ def get_addr_io(self, address):
                     sent[txi] = height
         return received, sent
 
-    def get_addr_utxo(self, address):
+    def get_addr_utxo(self, address: str) -> Dict[TxOutpoint, PartialTxInput]:
         coins, spent = self.get_addr_io(address)
         for txi in spent:
             coins.pop(txi)
         out = {}
-        for txo, v in coins.items():
+        for prevout_str, v in coins.items():
             tx_height, value, is_cb = v
-            prevout_hash, prevout_n = txo.split(':')
-            x = {
-                'address':address,
-                'value':value,
-                'prevout_n':int(prevout_n),
-                'prevout_hash':prevout_hash,
-                'height':tx_height,
-                'coinbase':is_cb
-            }
-            out[txo] = x
+            prevout = TxOutpoint.from_str(prevout_str)
+            utxo = PartialTxInput(prevout=prevout)
+            utxo._trusted_address = address
+            utxo._trusted_value_sats = value
+            utxo.block_height = tx_height
+            out[prevout] = utxo
         return out
 
     # return the total amount ever received by an address
@@ -799,7 +789,8 @@ def get_addr_balance(self, address, *, excluded_coins: Set[str] = None):
 
     @with_local_height_cached
     def get_utxos(self, domain=None, *, excluded_addresses=None,
-                  mature_only: bool = False, confirmed_only: bool = False, nonlocal_only: bool = False):
+                  mature_only: bool = False, confirmed_only: bool = False,
+                  nonlocal_only: bool = False) -> Sequence[PartialTxInput]:
         coins = []
         if domain is None:
             domain = self.get_addresses()
@@ -809,14 +800,15 @@ def get_utxos(self, domain=None, *, excluded_addresses=None,
         mempool_height = self.get_local_height() + 1  # height of next block
         for addr in domain:
             utxos = self.get_addr_utxo(addr)
-            for x in utxos.values():
-                if confirmed_only and x['height'] <= 0:
+            for utxo in utxos.values():
+                if confirmed_only and utxo.block_height <= 0:
                     continue
-                if nonlocal_only and x['height'] == TX_HEIGHT_LOCAL:
+                if nonlocal_only and utxo.block_height == TX_HEIGHT_LOCAL:
                     continue
-                if mature_only and x['coinbase'] and x['height'] + COINBASE_MATURITY > mempool_height:
+                if (mature_only and utxo.prevout.is_coinbase()
+                        and utxo.block_height + COINBASE_MATURITY > mempool_height):
                     continue
-                coins.append(x)
+                coins.append(utxo)
                 continue
         return coins
 

electrum/base_wizard.py

@@ -33,7 +33,7 @@
 from . import bitcoin
 from . import keystore
 from . import mnemonic
-from .bip32 import is_bip32_derivation, xpub_type, normalize_bip32_derivation
+from .bip32 import is_bip32_derivation, xpub_type, normalize_bip32_derivation, BIP32Node
 from .keystore import bip44_derivation, purpose48_derivation
 from .wallet import (Imported_Wallet, Standard_Wallet, Multisig_Wallet,
                      wallet_types, Wallet, Abstract_Wallet)
@@ -230,7 +230,7 @@ def on_import(self, text):
                 assert bitcoin.is_private_key(pk)
                 txin_type, pubkey = k.import_privkey(pk, None)
                 addr = bitcoin.pubkey_to_address(txin_type, pubkey)
-                self.data['addresses'][addr] = {'type':txin_type, 'pubkey':pubkey, 'redeem_script':None}
+                self.data['addresses'][addr] = {'type':txin_type, 'pubkey':pubkey}
             self.keystores.append(k)
         else:
             return self.terminate()
@@ -420,16 +420,19 @@ def on_hw_derivation(self, name, device_info, derivation, xtype):
         from .keystore import hardware_keystore
         try:
             xpub = self.plugin.get_xpub(device_info.device.id_, derivation, xtype, self)
+            root_xpub = self.plugin.get_xpub(device_info.device.id_, 'm', 'standard', self)
         except ScriptTypeNotSupported:
             raise  # this is handled in derivation_dialog
         except BaseException as e:
             self.logger.exception('')
             self.show_error(e)
             return
+        xfp = BIP32Node.from_xkey(root_xpub).calc_fingerprint_of_this_node().hex().lower()
         d = {
             'type': 'hardware',
             'hw_type': name,
             'derivation': derivation,
+            'root_fingerprint': xfp,
             'xpub': xpub,
             'label': device_info.label,
         }

electrum/bip32.py

@@ -116,7 +116,7 @@ class BIP32Node(NamedTuple):
     eckey: Union[ecc.ECPubkey, ecc.ECPrivkey]
     chaincode: bytes
     depth: int = 0
-    fingerprint: bytes = b'\x00'*4
+    fingerprint: bytes = b'\x00'*4  # as in serialized format, this is the *parent's* fingerprint
     child_number: bytes = b'\x00'*4
 
     @classmethod
@@ -161,7 +161,18 @@ def from_rootseed(cls, seed: bytes, *, xtype: str) -> 'BIP32Node':
                          eckey=ecc.ECPrivkey(master_k),
                          chaincode=master_c)
 
+    @classmethod
+    def from_bytes(cls, b: bytes) -> 'BIP32Node':
+        if len(b) != 78:
+            raise Exception(f"unexpected xkey raw bytes len {len(b)} != 78")
+        xkey = EncodeBase58Check(b)
+        return cls.from_xkey(xkey)
+
     def to_xprv(self, *, net=None) -> str:
+        payload = self.to_xprv_bytes(net=net)
+        return EncodeBase58Check(payload)
+
+    def to_xprv_bytes(self, *, net=None) -> bytes:
         if not self.is_private():
             raise Exception("cannot serialize as xprv; private key missing")
         payload = (xprv_header(self.xtype, net=net) +
@@ -172,24 +183,34 @@ def to_xprv(self, *, net=None) -> str:
                    bytes([0]) +
                    self.eckey.get_secret_bytes())
         assert len(payload) == 78, f"unexpected xprv payload len {len(payload)}"
-        return EncodeBase58Check(payload)
+        return payload
 
     def to_xpub(self, *, net=None) -> str:
+        payload = self.to_xpub_bytes(net=net)
+        return EncodeBase58Check(payload)
+
+    def to_xpub_bytes(self, *, net=None) -> bytes:
         payload = (xpub_header(self.xtype, net=net) +
                    bytes([self.depth]) +
                    self.fingerprint +
                    self.child_number +
                    self.chaincode +
                    self.eckey.get_public_key_bytes(compressed=True))
         assert len(payload) == 78, f"unexpected xpub payload len {len(payload)}"
-        return EncodeBase58Check(payload)
+        return payload
 
     def to_xkey(self, *, net=None) -> str:
         if self.is_private():
             return self.to_xprv(net=net)
         else:
             return self.to_xpub(net=net)
 
+    def to_bytes(self, *, net=None) -> bytes:
+        if self.is_private():
+            return self.to_xprv_bytes(net=net)
+        else:
+            return self.to_xpub_bytes(net=net)
+
     def convert_to_public(self) -> 'BIP32Node':
         if not self.is_private():
             return self
@@ -248,6 +269,12 @@ def subkey_at_public_derivation(self, path: Union[str, Iterable[int]]) -> 'BIP32
                          fingerprint=fingerprint,
                          child_number=child_number)
 
+    def calc_fingerprint_of_this_node(self) -> bytes:
+        """Returns the fingerprint of this node.
+        Note that self.fingerprint is of the *parent*.
+        """
+        return hash_160(self.eckey.get_public_key_bytes(compressed=True))[0:4]
+
 
 def xpub_type(x):
     return BIP32Node.from_xkey(x).xtype

electrum/bitcoin.py

@@ -45,6 +45,7 @@
 TOTAL_COIN_SUPPLY_LIMIT_IN_BTC = 21000000
 
 # supported types of transaction outputs
+# TODO kill these with fire
 TYPE_ADDRESS = 0
 TYPE_PUBKEY  = 1
 TYPE_SCRIPT  = 2
@@ -237,6 +238,8 @@ def script_num_to_hex(i: int) -> str:
 
 def var_int(i: int) -> str:
     # https://en.bitcoin.it/wiki/Protocol_specification#Variable_length_integer
+    # https://github.com/bitcoin/bitcoin/blob/efe1ee0d8d7f82150789f1f6840f139289628a2b/src/serialize.h#L247
+    # "CompactSize"
     if i<0xfd:
         return int_to_hex(i)
     elif i<=0xffff:
@@ -372,24 +375,28 @@ def pubkey_to_address(txin_type: str, pubkey: str, *, net=None) -> str:
     else:
         raise NotImplementedError(txin_type)
 
-def redeem_script_to_address(txin_type: str, redeem_script: str, *, net=None) -> str:
+
+# TODO this method is confusingly named
+def redeem_script_to_address(txin_type: str, scriptcode: str, *, net=None) -> str:
     if net is None: net = constants.net
     if txin_type == 'p2sh':
-        return hash160_to_p2sh(hash_160(bfh(redeem_script)), net=net)
+        # given scriptcode is a redeem_script
+        return hash160_to_p2sh(hash_160(bfh(scriptcode)), net=net)
     elif txin_type == 'p2wsh':
-        return script_to_p2wsh(redeem_script, net=net)
+        # given scriptcode is a witness_script
+        return script_to_p2wsh(scriptcode, net=net)
     elif txin_type == 'p2wsh-p2sh':
-        scriptSig = p2wsh_nested_script(redeem_script)
-        return hash160_to_p2sh(hash_160(bfh(scriptSig)), net=net)
+        # given scriptcode is a witness_script
+        redeem_script = p2wsh_nested_script(scriptcode)
+        return hash160_to_p2sh(hash_160(bfh(redeem_script)), net=net)
     else:
         raise NotImplementedError(txin_type)
 
 
 def script_to_address(script: str, *, net=None) -> str:
     from .transaction import get_address_from_output_script
-    t, addr = get_address_from_output_script(bfh(script), net=net)
-    assert t == TYPE_ADDRESS
-    return addr
+    return get_address_from_output_script(bfh(script), net=net)
+
 
 def address_to_script(addr: str, *, net=None) -> str:
     if net is None: net = constants.net