From 070c5bf9b9ddc19d34e8e824c0e7903b73b06a06 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Mon, 17 Feb 2020 14:40:31 +0000 Subject: [PATCH 01/15] python-bitcointx backend for jmbitcoin. Replaces core transaction, address, serialization and sign functionality for Bitcoin with python-bitcointx backend. Removes bech32 and btscript modules from jmbitcoin. Removes all string, hex, binary conversion routines. A generic hex/binary conversion now is added to jmbase. Removes all transaction serialization and deserialization routines. Removes the now irrelevant test modules. Remaining functions in jmbitcoin remove any parsing of hex format, requiring callers to use binary only. One additional test added, testing the remaining function in secp256k1_transaction.py: the signing of transactions. Deserialized form is now bitcointx.CMutableTransaction. For jmbase, in addition to the above, generic conversions for utxos to and from strings is added, and a dynamic conversion for AMP messages to binary-only. Within the code, utxos are now only in (binarytxid, int) form, except where converted for communcation. Tthe largest part of the changes are the modifications to jmbitcoin calls in jmclient; as well as different encapsulation with CMutableTransaction, there is also a removal of some but not all hex parsing; it remains for rpc calls to Core and for AMP message parsing. Backwards compatibility must be ensured so some joinmarket protocol messages still use hex, and it is also preserved in persistence of PoDLE data. As part of this, some significant simplification of certain legacy functions within the wallet has been done. jmdaemon is entirely unaltered (save for one test which simulates jmclient code). --- jmbase/jmbase/__init__.py | 7 +- jmbase/jmbase/support.py | 132 ++- jmbitcoin/jmbitcoin/__init__.py | 10 +- jmbitcoin/jmbitcoin/bech32.py | 122 -- jmbitcoin/jmbitcoin/btscript.py | 142 --- .../jmbitcoin/secp256k1_deterministic.py | 36 +- jmbitcoin/jmbitcoin/secp256k1_main.py | 519 +-------- jmbitcoin/jmbitcoin/secp256k1_transaction.py | 1012 +++-------------- jmbitcoin/setup.py | 2 +- jmbitcoin/test/test_addresses.py | 56 - jmbitcoin/test/test_bech32.py | 131 --- jmbitcoin/test/test_bip32.py | 4 +- jmbitcoin/test/test_btc_formatting.py | 57 - jmbitcoin/test/test_ecc_signing.py | 30 +- jmbitcoin/test/test_main.py | 61 - jmbitcoin/test/test_tx_serialize.py | 234 ---- jmbitcoin/test/test_tx_signing.py | 91 ++ jmclient/jmclient/__init__.py | 3 +- jmclient/jmclient/blockchaininterface.py | 39 +- jmclient/jmclient/client_protocol.py | 63 +- jmclient/jmclient/commitment_utils.py | 33 +- jmclient/jmclient/configure.py | 50 +- jmclient/jmclient/cryptoengine.py | 71 +- jmclient/jmclient/maker.py | 185 ++- jmclient/jmclient/output.py | 2 +- jmclient/jmclient/podle.py | 241 ++-- jmclient/jmclient/taker.py | 302 +++-- jmclient/jmclient/taker_utils.py | 29 +- jmclient/jmclient/wallet.py | 138 +-- jmclient/jmclient/wallet_service.py | 82 +- jmclient/jmclient/wallet_utils.py | 15 +- jmclient/jmclient/yieldgenerator.py | 6 +- jmclient/test/commontest.py | 62 +- jmclient/test/test_client_protocol.py | 17 +- jmclient/test/test_coinjoin.py | 23 +- jmclient/test/test_commitment_utils.py | 9 +- jmclient/test/test_maker.py | 67 +- jmclient/test/test_payjoin.py | 5 +- jmclient/test/test_podle.py | 29 +- .../test/test_privkeys.py | 32 +- jmclient/test/test_taker.py | 160 +-- jmclient/test/test_tx_creation.py | 241 +--- jmclient/test/test_valid_addresses.py | 32 +- jmclient/test/test_wallet.py | 93 +- jmclient/test/test_wallets.py | 16 +- jmclient/test/test_walletservice.py | 1 - jmclient/test/test_yieldgenerator.py | 16 +- jmdaemon/test/test_message_channel.py | 10 +- scripts/add-utxo.py | 9 +- scripts/convert_old_wallet.py | 5 +- scripts/joinmarket-qt.py | 25 +- scripts/sendtomany.py | 23 +- test/test_segwit.py | 28 +- 53 files changed, 1555 insertions(+), 3253 deletions(-) delete mode 100644 jmbitcoin/jmbitcoin/bech32.py delete mode 100644 jmbitcoin/jmbitcoin/btscript.py delete mode 100644 jmbitcoin/test/test_addresses.py delete mode 100644 jmbitcoin/test/test_bech32.py delete mode 100644 jmbitcoin/test/test_btc_formatting.py delete mode 100644 jmbitcoin/test/test_main.py delete mode 100644 jmbitcoin/test/test_tx_serialize.py create mode 100644 jmbitcoin/test/test_tx_signing.py rename jmbitcoin/test/test_keys.py => jmclient/test/test_privkeys.py (75%) diff --git a/jmbase/jmbase/__init__.py b/jmbase/jmbase/__init__.py index ad3931e53..cdbbedb7b 100644 --- a/jmbase/jmbase/__init__.py +++ b/jmbase/jmbase/__init__.py @@ -2,7 +2,10 @@ from .support import (get_log, chunks, debug_silence, jmprint, joinmarket_alert, core_alert, get_password, set_logging_level, set_logging_color, - lookup_appdata_folder, - JM_WALLET_NAME_PREFIX, JM_APP_NAME) + lookup_appdata_folder, bintohex, bintolehex, + hextobin, lehextobin, utxostr_to_utxo, + utxo_to_utxostr, EXIT_ARGERROR, EXIT_FAILURE, + EXIT_SUCCESS, hexbin, dictchanger, listchanger, + cv, JM_WALLET_NAME_PREFIX, JM_APP_NAME) from .commands import * diff --git a/jmbase/jmbase/support.py b/jmbase/jmbase/support.py index 7175bb5c9..6afad0af6 100644 --- a/jmbase/jmbase/support.py +++ b/jmbase/jmbase/support.py @@ -1,8 +1,9 @@ import logging, sys +import binascii from getpass import getpass from os import path, environ - +from functools import wraps # JoinMarket version JM_CORE_VERSION = '0.7.0dev' @@ -80,6 +81,71 @@ def emit(self, record): handler.setFormatter(logFormatter) log.addHandler(handler) +# hex/binary conversion routines used by dependent packages +def hextobin(h): + """Convert a hex string to bytes""" + return binascii.unhexlify(h.encode('utf8')) + + +def bintohex(b): + """Convert bytes to a hex string""" + return binascii.hexlify(b).decode('utf8') + + +def lehextobin(h): + """Convert a little-endian hex string to bytes + + Lets you write uint256's and uint160's the way the Satoshi codebase shows + them. + """ + return binascii.unhexlify(h.encode('utf8'))[::-1] + + +def bintolehex(b): + """Convert bytes to a little-endian hex string + + Lets you show uint256's and uint160's the way the Satoshi codebase shows + them. + """ + return binascii.hexlify(b[::-1]).decode('utf8') + +def utxostr_to_utxo(x): + if not isinstance(x, str): + return (False, "not a string") + y = x.split(":") + if len(y) != 2: + return (False, + "string is not two items separated by :") + try: + n = int(y[1]) + except: + return (False, "utxo index was not an integer.") + if n < 0: + return (False, "utxo index must not be negative.") + if len(y[0]) != 64: + return (False, "txid is not 64 hex characters.") + try: + txid = binascii.unhexlify(y[0]) + except: + return (False, "txid is not hex.") + return (True, (txid, n)) + +def utxo_to_utxostr(u): + if not isinstance(u, tuple): + return (False, "utxo is not a tuple.") + if not len(u) == 2: + return (False, "utxo should have two elements.") + if not isinstance(u[0], bytes): + return (False, "txid should be bytes.") + if not isinstance(u[1], int): + return (False, "index should be int.") + if u[1] < 0: + return (False, "index must be a positive integer.") + if not len(u[0]) == 32: + return (False, "txid must be 32 bytes.") + txid = binascii.hexlify(u[0]).decode("ascii") + return (True, txid + ":" + str(u[1])) + def jmprint(msg, level="info"): """ Provides the ability to print messages with consistent formatting, outside the logging system @@ -150,3 +216,67 @@ def lookup_appdata_folder(appname): def print_jm_version(option, opt_str, value, parser): print("JoinMarket " + JM_CORE_VERSION) sys.exit(EXIT_SUCCESS) + +# helper functions for conversions of format between over-the-wire JM +# and internal. See details in hexbin() docstring. + +def cv(x): + success, utxo = utxostr_to_utxo(x) + if success: + return utxo + else: + try: + b = hextobin(x) + return b + except: + return x + +def listchanger(l): + rlist = [] + for x in l: + if isinstance(x, list): + rlist.append(listchanger(x)) + elif isinstance(x, dict): + rlist.append(dictchanger(x)) + else: + rlist.append(cv(x)) + return rlist + +def dictchanger(d): + rdict = {} + for k, v in d.items(): + if isinstance(v, dict): + rdict[cv(k)] = dictchanger(v) + elif isinstance(v, list): + rdict[cv(k)] = listchanger(v) + else: + rdict[cv(k)] = cv(v) + return rdict + +def hexbin(func): + """ Decorator for functions of taker and maker receiving over + the wire AMP arguments that may be in hex or hextxid:n format + and converting all to binary. + Functions to which this decorator applies should have all arguments + be one of: + - hex string (keys), converted here to binary + - lists of keys or txid:n strings (converted here to binary, or + (txidbytes, n)) + - lists of lists or dicts, to which these rules apply recursively. + - any other string (unchanged) + - dicts with keys as per above; values are altered recursively according + to the rules above. + """ + @wraps(func) + def func_wrapper(inst, *args, **kwargs): + newargs = [] + for arg in args: + if isinstance(arg, (list, tuple)): + newargs.append(listchanger(arg)) + elif isinstance(arg, dict): + newargs.append(dictchanger(arg)) + else: + newargs.append(cv(arg)) + return func(inst, *newargs, **kwargs) + + return func_wrapper \ No newline at end of file diff --git a/jmbitcoin/jmbitcoin/__init__.py b/jmbitcoin/jmbitcoin/__init__.py index 784673ac4..00fc5194f 100644 --- a/jmbitcoin/jmbitcoin/__init__.py +++ b/jmbitcoin/jmbitcoin/__init__.py @@ -2,8 +2,14 @@ from jmbitcoin.secp256k1_main import * from jmbitcoin.secp256k1_transaction import * from jmbitcoin.secp256k1_deterministic import * -from jmbitcoin.btscript import * -from jmbitcoin.bech32 import * from jmbitcoin.amount import * from jmbitcoin.bip21 import * +from bitcointx import select_chain_params +from bitcointx.core import (x, b2x, b2lx, lx, COutPoint, CTxOut, CTxIn, + CTxInWitness, CTxWitness, CMutableTransaction, + Hash160, coins_to_satoshi, satoshi_to_coins) +from bitcointx.core.script import (CScript, OP_0, SignatureHash, SIGHASH_ALL, + SIGVERSION_WITNESS_V0, CScriptWitness) +from bitcointx.wallet import (CBitcoinSecret, P2WPKHBitcoinAddress, CCoinAddress, + P2SHCoinAddress) diff --git a/jmbitcoin/jmbitcoin/bech32.py b/jmbitcoin/jmbitcoin/bech32.py deleted file mode 100644 index 8d3f9bf71..000000000 --- a/jmbitcoin/jmbitcoin/bech32.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright (c) 2017 Pieter Wuille -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -"""Reference implementation for Bech32 and segwit addresses.""" - -CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l" - - -def bech32_polymod(values): - """Internal function that computes the Bech32 checksum.""" - generator = [0x3b6a57b2, 0x26508e6d, 0x1ea119fa, 0x3d4233dd, 0x2a1462b3] - chk = 1 - for value in values: - top = chk >> 25 - chk = (chk & 0x1ffffff) << 5 ^ value - for i in range(5): - chk ^= generator[i] if ((top >> i) & 1) else 0 - return chk - - -def bech32_hrp_expand(hrp): - """Expand the HRP into values for checksum computation.""" - return [ord(x) >> 5 for x in hrp] + [0] + [ord(x) & 31 for x in hrp] - - -def bech32_verify_checksum(hrp, data): - """Verify a checksum given HRP and converted data characters.""" - return bech32_polymod(bech32_hrp_expand(hrp) + data) == 1 - - -def bech32_create_checksum(hrp, data): - """Compute the checksum values given HRP and data.""" - values = bech32_hrp_expand(hrp) + data - polymod = bech32_polymod(values + [0, 0, 0, 0, 0, 0]) ^ 1 - return [(polymod >> 5 * (5 - i)) & 31 for i in range(6)] - - -def bech32_encode(hrp, data): - """Compute a Bech32 string given HRP and data values.""" - combined = data + bech32_create_checksum(hrp, data) - return hrp + '1' + ''.join([CHARSET[d] for d in combined]) - - -def bech32_decode(bech): - """Validate a Bech32 string, and determine HRP and data.""" - if ((any(ord(x) < 33 or ord(x) > 126 for x in bech)) or - (bech.lower() != bech and bech.upper() != bech)): - return (None, None) - bech = bech.lower() - pos = bech.rfind('1') - if pos < 1 or pos + 7 > len(bech) or len(bech) > 90: - return (None, None) - if not all(x in CHARSET for x in bech[pos+1:]): - return (None, None) - hrp = bech[:pos] - data = [CHARSET.find(x) for x in bech[pos+1:]] - if not bech32_verify_checksum(hrp, data): - return (None, None) - return (hrp, data[:-6]) - - -def convertbits(data, frombits, tobits, pad=True): - """General power-of-2 base conversion.""" - acc = 0 - bits = 0 - ret = [] - maxv = (1 << tobits) - 1 - max_acc = (1 << (frombits + tobits - 1)) - 1 - for value in data: - if value < 0 or (value >> frombits): - return None - acc = ((acc << frombits) | value) & max_acc - bits += frombits - while bits >= tobits: - bits -= tobits - ret.append((acc >> bits) & maxv) - if pad: - if bits: - ret.append((acc << (tobits - bits)) & maxv) - elif bits >= frombits or ((acc << (tobits - bits)) & maxv): - return None - return ret - - -def bech32addr_decode(hrp, addr): - """Decode a segwit address.""" - hrpgot, data = bech32_decode(addr) - if hrpgot != hrp: - return (None, None) - decoded = convertbits(data[1:], 5, 8, False) - if decoded is None or len(decoded) < 2 or len(decoded) > 40: - return (None, None) - if data[0] > 16: - return (None, None) - if data[0] == 0 and len(decoded) != 20 and len(decoded) != 32: - return (None, None) - return (data[0], decoded) - - -def bech32addr_encode(hrp, witver, witprog): - """Encode a segwit address.""" - ret = bech32_encode(hrp, [witver] + convertbits(witprog, 8, 5)) - if bech32addr_decode(hrp, ret) == (None, None): - return None - return ret diff --git a/jmbitcoin/jmbitcoin/btscript.py b/jmbitcoin/jmbitcoin/btscript.py deleted file mode 100644 index c658c4e3b..000000000 --- a/jmbitcoin/jmbitcoin/btscript.py +++ /dev/null @@ -1,142 +0,0 @@ -#OP codes; disabled commented. - -# push value -OP_0 = 0x00 -OP_FALSE = OP_0 -OP_PUSHDATA1 = 0x4c -OP_PUSHDATA2 = 0x4d -OP_PUSHDATA4 = 0x4e -OP_1NEGATE = 0x4f -OP_RESERVED = 0x50 -OP_1 = 0x51 -OP_TRUE = OP_1 -OP_2 = 0x52 -OP_3 = 0x53 -OP_4 = 0x54 -OP_5 = 0x55 -OP_6 = 0x56 -OP_7 = 0x57 -OP_8 = 0x58 -OP_9 = 0x59 -OP_10 = 0x5a -OP_11 = 0x5b -OP_12 = 0x5c -OP_13 = 0x5d -OP_14 = 0x5e -OP_15 = 0x5f -OP_16 = 0x60 - -# control -OP_NOP = 0x61 -OP_VER = 0x62 -OP_IF = 0x63 -OP_NOTIF = 0x64 -#OP_VERIF = 0x65 -#OP_VERNOTIF = 0x66 -OP_ELSE = 0x67 -OP_ENDIF = 0x68 -OP_VERIFY = 0x69 -OP_RETURN = 0x6a - -# stack ops -OP_TOALTSTACK = 0x6b -OP_FROMALTSTACK = 0x6c -OP_2DROP = 0x6d -OP_2DUP = 0x6e -OP_3DUP = 0x6f -OP_2OVER = 0x70 -OP_2ROT = 0x71 -OP_2SWAP = 0x72 -OP_IFDUP = 0x73 -OP_DEPTH = 0x74 -OP_DROP = 0x75 -OP_DUP = 0x76 -OP_NIP = 0x77 -OP_OVER = 0x78 -OP_PICK = 0x79 -OP_ROLL = 0x7a -OP_ROT = 0x7b -OP_SWAP = 0x7c -OP_TUCK = 0x7d - -# splice ops -#OP_CAT = 0x7e -#OP_SUBSTR = 0x7f -#OP_LEFT = 0x80 -#OP_RIGHT = 0x81 -OP_SIZE = 0x82 - -# bit logic -#OP_INVERT = 0x83 -#OP_AND = 0x84 -#OP_OR = 0x85 -#OP_XOR = 0x86 -OP_EQUAL = 0x87 -OP_EQUALVERIFY = 0x88 -OP_RESERVED1 = 0x89 -OP_RESERVED2 = 0x8a - -# numeric -OP_1ADD = 0x8b -OP_1SUB = 0x8c -#OP_2MUL = 0x8d -#OP_2DIV = 0x8e -OP_NEGATE = 0x8f -OP_ABS = 0x90 -OP_NOT = 0x91 -OP_0NOTEQUAL = 0x92 - -OP_ADD = 0x93 -OP_SUB = 0x94 -#OP_MUL = 0x95 -#OP_DIV = 0x96 -#OP_MOD = 0x97 -#OP_LSHIFT = 0x98 -#OP_RSHIFT = 0x99 - -OP_BOOLAND = 0x9a -OP_BOOLOR = 0x9b -OP_NUMEQUAL = 0x9c -OP_NUMEQUALVERIFY = 0x9d -OP_NUMNOTEQUAL = 0x9e -OP_LESSTHAN = 0x9f -OP_GREATERTHAN = 0xa0 -OP_LESSTHANOREQUAL = 0xa1 -OP_GREATERTHANOREQUAL = 0xa2 -OP_MIN = 0xa3 -OP_MAX = 0xa4 - -OP_WITHIN = 0xa5 - -# crypto -OP_RIPEMD160 = 0xa6 -OP_SHA1 = 0xa7 -OP_SHA256 = 0xa8 -OP_HASH160 = 0xa9 -OP_HASH256 = 0xaa -OP_CODESEPARATOR = 0xab -OP_CHECKSIG = 0xac -OP_CHECKSIGVERIFY = 0xad -OP_CHECKMULTISIG = 0xae -OP_CHECKMULTISIGVERIFY = 0xaf - -# expansion -OP_NOP1 = 0xb0 -OP_NOP2 = 0xb1 -OP_CHECKLOCKTIMEVERIFY = OP_NOP2 -OP_NOP3 = 0xb2 -OP_NOP4 = 0xb3 -OP_NOP5 = 0xb4 -OP_NOP6 = 0xb5 -OP_NOP7 = 0xb6 -OP_NOP8 = 0xb7 -OP_NOP9 = 0xb8 -OP_NOP10 = 0xb9 - -# template matching params -OP_SMALLINTEGER = 0xfa -OP_PUBKEYS = 0xfb -OP_PUBKEYHASH = 0xfd -OP_PUBKEY = 0xfe - -OP_INVALIDOPCODE = 0xff diff --git a/jmbitcoin/jmbitcoin/secp256k1_deterministic.py b/jmbitcoin/jmbitcoin/secp256k1_deterministic.py index 1baad533d..eb9023de6 100644 --- a/jmbitcoin/jmbitcoin/secp256k1_deterministic.py +++ b/jmbitcoin/jmbitcoin/secp256k1_deterministic.py @@ -2,6 +2,8 @@ import hmac import hashlib import struct +from bitcointx.core import Hash160, Hash +from bitcointx import base58 # Below code ASSUMES binary inputs and compressed pubkeys MAINNET_PRIVATE = b'\x04\x88\xAD\xE4' @@ -11,6 +13,8 @@ PRIVATE = [MAINNET_PRIVATE, TESTNET_PRIVATE] PUBLIC = [MAINNET_PUBLIC, TESTNET_PUBLIC] +privtopub = privkey_to_pubkey + # BIP32 child key derivation def raw_bip32_ckd(rawtuple, i): @@ -19,25 +23,25 @@ def raw_bip32_ckd(rawtuple, i): if vbytes in PRIVATE: priv = key - pub = privtopub(key, False) + pub = privtopub(key) else: pub = key if i >= 2**31: if vbytes in PUBLIC: raise Exception("Can't do private derivation on public key!") - I = hmac.new(chaincode, b'\x00' + priv[:32] + encode(i, 256, 4), + I = hmac.new(chaincode, b'\x00' + priv[:32] + struct.pack(b'>L', i), hashlib.sha512).digest() else: - I = hmac.new(chaincode, pub + encode(i, 256, 4), + I = hmac.new(chaincode, pub + struct.pack(b'>L', i), hashlib.sha512).digest() if vbytes in PRIVATE: - newkey = add_privkeys(I[:32] + b'\x01', priv, False) - fingerprint = bin_hash160(privtopub(key, False))[:4] + newkey = add_privkeys(I[:32] + b'\x01', priv) + fingerprint = Hash160(privtopub(key))[:4] if vbytes in PUBLIC: - newkey = add_pubkeys([privtopub(I[:32] + b'\x01', False), key], False) - fingerprint = bin_hash160(key)[:4] + newkey = add_pubkeys([privtopub(I[:32] + b'\x01'), key]) + fingerprint = Hash160(key)[:4] return (vbytes, depth + 1, fingerprint, i, I[32:], newkey) @@ -47,18 +51,17 @@ def bip32_serialize(rawtuple): i = struct.pack(b'>L', i) chaincode = chaincode keydata = b'\x00' + key[:-1] if vbytes in PRIVATE else key - bindata = vbytes + from_int_to_byte( - depth % 256) + fingerprint + i + chaincode + keydata - return b58encode(bindata + bin_dbl_sha256(bindata)[:4]) + bindata = vbytes + struct.pack(b'B',depth % 256) + fingerprint + i + chaincode + keydata + return base58.encode(bindata + Hash(bindata)[:4]) def bip32_deserialize(data): - dbin = b58decode(data) - if bin_dbl_sha256(dbin[:-4])[:4] != dbin[-4:]: + dbin = base58.decode(data) + if Hash(dbin[:-4])[:4] != dbin[-4:]: raise Exception("Invalid checksum") vbytes = dbin[0:4] depth = dbin[4] fingerprint = dbin[5:9] - i = decode(dbin[9:13], 256) + i = struct.unpack(b'>L',dbin[9:13])[0] chaincode = dbin[13:45] key = dbin[46:78] + b'\x01' if vbytes in PRIVATE else dbin[45:78] return (vbytes, depth, fingerprint, i, chaincode, key) @@ -68,7 +71,7 @@ def raw_bip32_privtopub(rawtuple): if vbytes in PUBLIC: return rawtuple newvbytes = MAINNET_PUBLIC if vbytes == MAINNET_PRIVATE else TESTNET_PUBLIC - return (newvbytes, depth, fingerprint, i, chaincode, privtopub(key, False)) + return (newvbytes, depth, fingerprint, i, chaincode, privtopub(key)) def bip32_privtopub(data): return bip32_serialize(raw_bip32_privtopub(bip32_deserialize(data))) @@ -77,13 +80,12 @@ def bip32_ckd(data, i): return bip32_serialize(raw_bip32_ckd(bip32_deserialize(data), i)) def bip32_master_key(seed, vbytes=MAINNET_PRIVATE): - I = hmac.new( - from_string_to_bytes("Bitcoin seed"), seed, hashlib.sha512).digest() + I = hmac.new("Bitcoin seed".encode("utf-8"), seed, hashlib.sha512).digest() return bip32_serialize((vbytes, 0, b'\x00' * 4, 0, I[32:], I[:32] + b'\x01' )) def bip32_extract_key(data): - return safe_hexlify(bip32_deserialize(data)[-1]) + return bip32_deserialize(data)[-1] def bip32_descend(*args): if len(args) == 2: diff --git a/jmbitcoin/jmbitcoin/secp256k1_main.py b/jmbitcoin/jmbitcoin/secp256k1_main.py index e91fc5915..db0e7cbe3 100644 --- a/jmbitcoin/jmbitcoin/secp256k1_main.py +++ b/jmbitcoin/jmbitcoin/secp256k1_main.py @@ -7,213 +7,16 @@ import struct import coincurve as secp256k1 +from bitcointx import base58 +from bitcointx.core import Hash +from bitcointx.signmessage import BitcoinMessage + #Required only for PoDLE calculation: N = 115792089237316195423570985008687907852837564279074904382605163141518161494337 BTC_P2PK_VBYTE = {"mainnet": b'\x00', "testnet": b'\x6f', "regtest": 100} BTC_P2SH_VBYTE = {"mainnet": b'\x05', "testnet": b'\xc4'} -#Standard prefix for Bitcoin message signing. -BITCOIN_MESSAGE_MAGIC = b'\x18' + b'Bitcoin Signed Message:\n' - -string_types = (str) -string_or_bytes_types = (str, bytes) -int_types = (int, float) - -# Base switching -code_strings = { - 2: '01', - 10: '0123456789', - 16: '0123456789abcdef', - 32: 'abcdefghijklmnopqrstuvwxyz234567', - 58: '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz', - 256: ''.join([chr(x) for x in range(256)]) -} - -def lpad(msg, symbol, length): - if len(msg) >= length: - return msg - return symbol * (length - len(msg)) + msg - -def get_code_string(base): - if base in code_strings: - return code_strings[base] - else: - raise ValueError("Invalid base!") - -def bin_to_b58check(inp, magicbyte=b'\x00'): - if not isinstance(magicbyte, int): - magicbyte = struct.unpack(b'B', magicbyte)[0] - assert(0 <= magicbyte <= 0xff) - if magicbyte == 0: - inp_fmtd = struct.pack(b'B', magicbyte) + inp - while magicbyte > 0: - inp_fmtd = struct.pack(b'B', magicbyte % 256) + inp - magicbyte //= 256 - checksum = bin_dbl_sha256(inp_fmtd)[:4] - return b58encode(inp_fmtd + checksum) - -def safe_from_hex(s): - return binascii.unhexlify(s) - -def from_int_to_byte(a): - return struct.pack(b'B', a) - -def from_byte_to_int(a): - return struct.unpack(b'B', a)[0] - -def from_string_to_bytes(a): - return a if isinstance(a, bytes) else bytes(a, 'utf-8') - -def safe_hexlify(a): - return binascii.hexlify(a).decode('ascii') - -class SerializationError(Exception): - """Base class for serialization errors""" - - -class SerializationTruncationError(SerializationError): - """Serialized data was truncated - Thrown by deserialize() and stream_deserialize() - """ - -def ser_read(f, n): - """Read from a stream safely - Raises SerializationError and SerializationTruncationError appropriately. - Use this instead of f.read() in your classes stream_(de)serialization() - functions. - """ - MAX_SIZE = 0x02000000 - if n > MAX_SIZE: - raise SerializationError('Asked to read 0x%x bytes; MAX_SIZE exceeded' % n) - r = f.read(n) - if len(r) < n: - raise SerializationTruncationError('Asked to read %i bytes, but only got %i' % (n, len(r))) - return r - -B58_DIGITS = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' - -class Base58Error(Exception): - pass - -class InvalidBase58Error(Base58Error): - """Raised on generic invalid base58 data, such as bad characters. - Checksum failures raise Base58ChecksumError specifically. - """ - pass - -def b58encode(b): - """Encode bytes to a base58-encoded string""" - - # Convert big-endian bytes to integer - n = int('0x0' + binascii.hexlify(b).decode('ascii'), 16) - - # Divide that integer into bas58 - res = [] - while n > 0: - n, r = divmod(n, 58) - res.append(B58_DIGITS[r]) - res = ''.join(res[::-1]) - - # Encode leading zeros as base58 zeros - czero = b'\x00' - if sys.version_info >= (3,0): - # In Python3 indexing a bytes returns numbers, not characters. - czero = 0 - pad = 0 - for c in b: - if c == czero: - pad += 1 - else: - break - return B58_DIGITS[0] * pad + res - -def b58decode(s): - """Decode a base58-encoding string, returning bytes""" - if not s: - return b'' - - # Convert the string to an integer - n = 0 - for c in s: - n *= 58 - if c not in B58_DIGITS: - raise InvalidBase58Error('Character %r is not a valid base58 character' % c) - digit = B58_DIGITS.index(c) - n += digit - - # Convert the integer to bytes - h = '%x' % n - if len(h) % 2: - h = '0' + h - res = bytes(binascii.unhexlify(h.encode('utf8'))) - - # Add padding back. - pad = 0 - for c in s[:-1]: - if c == B58_DIGITS[0]: pad += 1 - else: break - return b'\x00' * pad + res - -def uint256encode(s): - """Convert bytes to uint256""" - r = 0 - t = struct.unpack(b"> (i * 32) & 0xffffffff) - return r - -def encode(val, base, minlen=0): - base, minlen = int(base), int(minlen) - code_string = get_code_string(base) - result_bytes = bytes() - while val > 0: - curcode = code_string[val % base] - result_bytes = bytes([ord(curcode)]) + result_bytes - val //= base - - pad_size = minlen - len(result_bytes) - - padding_element = b'\x00' if base == 256 else b'1' \ - if base == 58 else b'0' - if (pad_size > 0): - result_bytes = padding_element*pad_size + result_bytes - - result_string = ''.join([chr(y) for y in result_bytes]) - result = result_bytes if base == 256 else result_string - - return result - -def decode(string, base): - if base == 256 and isinstance(string, str): - string = bytes(bytearray.fromhex(string)) - base = int(base) - code_string = get_code_string(base) - result = 0 - if base == 256: - def extract(d, cs): - if isinstance(d, int): - return d - else: - return struct.unpack(b'B', d)[0] - else: - def extract(d, cs): - return cs.find(d if isinstance(d, str) else chr(d)) - - if base == 16: - string = string.lower() - while len(string) > 0: - result *= base - result += extract(string[0], code_string) - string = string[1:] - return result - """PoDLE related primitives """ def getG(compressed=True): @@ -237,165 +40,6 @@ def podle_PrivateKey(priv): """ return secp256k1.PrivateKey(priv) - -def privkey_to_address(priv, from_hex=True, magicbyte=0): - return pubkey_to_address(privkey_to_pubkey(priv, from_hex), magicbyte) - -privtoaddr = privkey_to_address - -# Hashes -def bin_hash160(string): - intermed = hashlib.sha256(string).digest() - return hashlib.new('ripemd160', intermed).digest() - -def hash160(string): - if not isinstance(string, bytes): - string = string.encode('utf-8') - return safe_hexlify(bin_hash160(string)) - -def bin_sha256(string): - binary_data = string if isinstance(string, bytes) else bytes(string, - 'utf-8') - return hashlib.sha256(binary_data).digest() - -def sha256(string): - return safe_hexlify(bin_sha256(string)) - -def bin_dbl_sha256(bytes_to_hash): - if not isinstance(bytes_to_hash, bytes): - bytes_to_hash = bytes_to_hash.encode('utf-8') - return hashlib.sha256(hashlib.sha256(bytes_to_hash).digest()).digest() - -def dbl_sha256(string): - return hashlib.sha256(hashlib.sha256(string).digest()).hexdigest() - -def hash_to_int(x): - if len(x) in [40, 64]: - return decode(x, 16) - return decode(x, 256) - -def num_to_var_int(x): - if not isinstance(x, int): - if len(x) == 0: - return b'\x00' - x = struct.unpack(b'B', x)[0] - if x < 253: return from_int_to_byte(x) - elif x < 65536: return from_int_to_byte(253) + struct.pack(b'= (3,0): - newpriv = secp256k1.PrivateKey(secret=native_bytes(priv)) - else: - newpriv = secp256k1.PrivateKey(secret=bytes_to_native_str(priv)) + newpriv = secp256k1.PrivateKey(secret=native_bytes(priv)) return newpriv.public_key.format(compressed) -def privkey_to_pubkey(priv, usehex=True): - '''To avoid changing the interface from the legacy system, - allow an *optional* hex argument here (called differently from - maker/taker code to how it's called in bip32 code), then - pass to the standard hexbin decorator under the hood. - ''' - return privkey_to_pubkey_inner(priv, usehex) +# b58check wrapper functions around bitcointx.base58 functions: +# (avoids complexity of key management structure) + +def bin_to_b58check(inp, magicbyte=b'\x00'): + """ The magic byte (prefix byte) should be passed either + as a single byte or an integer. What is returned is a string + in base58 encoding, with the prefix and the checksum. + """ + if not isinstance(magicbyte, int): + magicbyte = struct.unpack(b'B', magicbyte)[0] + assert(0 <= magicbyte <= 0xff) + if magicbyte == 0: + inp_fmtd = struct.pack(b'B', magicbyte) + inp + while magicbyte > 0: + inp_fmtd = struct.pack(b'B', magicbyte % 256) + inp + magicbyte //= 256 + checksum = Hash(inp_fmtd)[:4] + return base58.encode(inp_fmtd + checksum) + +def b58check_to_bin(s): + data = base58.decode(s) + assert Hash(data[:-4])[:4] == data[-4:] + return struct.pack(b"B", data[0]), data[1:-4] -privtopub = privkey_to_pubkey +def get_version_byte(s): + return b58check_to_bin(s)[0] -@hexbin -def is_valid_pubkey(pubkey, usehex, require_compressed=False): +def ecdsa_sign(msg, priv, formsg=False): + hashed_msg = BitcoinMessage(msg).GetHash() + sig = ecdsa_raw_sign(hashed_msg, priv, rawmsg=True, formsg=formsg) + return base64.b64encode(sig).decode('ascii') + +def ecdsa_verify(msg, sig, pub): + hashed_msg = BitcoinMessage(msg).GetHash() + sig = base64.b64decode(sig) + return ecdsa_raw_verify(hashed_msg, pub, sig, rawmsg=True) + +def is_valid_pubkey(pubkey, require_compressed=False): """ Returns True if the serialized pubkey is a valid secp256k1 pubkey serialization or False if not; returns False for an uncompressed encoding if require_compressed is True. @@ -458,8 +124,8 @@ def is_valid_pubkey(pubkey, usehex, require_compressed=False): return False return True -@hexbin -def multiply(s, pub, usehex, rawpub=True, return_serialized=True): + +def multiply(s, pub, return_serialized=True): '''Input binary compressed pubkey P(33 bytes) and scalar s(32 bytes), return s*P. The return value is a binary compressed public key, @@ -470,24 +136,19 @@ def multiply(s, pub, usehex, rawpub=True, return_serialized=True): ''' newpub = secp256k1.PublicKey(pub) #see note to "tweak_mul" function in podle.py - if sys.version_info >= (3,0): - res = newpub.multiply(native_bytes(s)) - else: - res = newpub.multiply(bytes_to_native_str(s)) + res = newpub.multiply(native_bytes(s)) if not return_serialized: return res return res.format() -@hexbin -def add_pubkeys(pubkeys, usehex): +def add_pubkeys(pubkeys): '''Input a list of binary compressed pubkeys and return their sum as a binary compressed pubkey.''' pubkey_list = [secp256k1.PublicKey(x) for x in pubkeys] r = secp256k1.PublicKey.combine_keys(pubkey_list) return r.format() -@hexbin -def add_privkeys(priv1, priv2, usehex): +def add_privkeys(priv1, priv2): '''Add privkey 1 to privkey 2. Input keys must be in binary either compressed or not. Returned key will have the same compression state. @@ -504,58 +165,34 @@ def add_privkeys(priv1, priv2, usehex): res += b'\x01' return res -@hexbin + def ecdsa_raw_sign(msg, priv, - usehex, - rawpriv=True, rawmsg=False, - usenonce=None, formsg=False): '''Take the binary message msg and sign it with the private key priv. - By default priv is just a 32 byte string, if rawpriv is false - it is assumed to be hex encoded (note only works if usehex=False). If rawmsg is True, no sha256 hash is applied to msg before signing. In this case, msg must be a precalculated hash (256 bit). If rawmsg is False, the secp256k1 lib will hash the message as part of the ECDSA-SHA256 signing algo. - If usenonce is not None, its value is passed to the secp256k1 library - sign() function as the ndata value, which is then used in conjunction - with a custom nonce generating function, such that the nonce used in the ECDSA - sign algorithm is exactly that value (ndata there, usenonce here). 32 bytes. Return value: the calculated signature.''' if rawmsg and len(msg) != 32: raise Exception("Invalid hash input to ECDSA raw sign.") - if rawpriv: - compressed, p = read_privkey(priv) - newpriv = secp256k1.PrivateKey(p) - else: - newpriv = secp256k1.PrivateKey.from_hex(priv) + + compressed, p = read_privkey(priv) + newpriv = secp256k1.PrivateKey(p) if formsg: sig = newpriv.sign_recoverable(msg) return sig - #Donations, thus custom nonce, currently disabled, hence not covered. - elif usenonce: #pragma: no cover - raise NotImplementedError - #if len(usenonce) != 32: - # raise ValueError("Invalid nonce passed to ecdsa_sign: " + str( - # usenonce)) - #nf = ffi.addressof(_noncefunc.lib, "nonce_function_rand") - #ndata = ffi.new("char [32]", usenonce) - #usenonce = (nf, ndata) - #sig = newpriv.ecdsa_sign(msg, raw=rawmsg, custom_nonce=usenonce) else: - #partial fix for secp256k1-transient not including customnonce; - #partial because donations will crash on windows in the "if". if rawmsg: sig = newpriv.sign(msg, hasher=None) else: sig = newpriv.sign(msg) return sig -@hexbin -def ecdsa_raw_verify(msg, pub, sig, usehex, rawmsg=False): +def ecdsa_raw_verify(msg, pub, sig, rawmsg=False): '''Take the binary message msg and binary signature sig, and verify it against the pubkey pub. If rawmsg is True, no sha256 hash is applied to msg before verifying. @@ -575,56 +212,6 @@ def ecdsa_raw_verify(msg, pub, sig, usehex, rawmsg=False): retval = newpub.verify(sig, msg, hasher=None) else: retval = newpub.verify(sig, msg) - except: + except Exception as e: return False return retval - -def estimate_tx_size(ins, outs, txtype='p2pkh'): - '''Estimate transaction size. - The txtype field as detailed below is used to distinguish - the type, but there is at least one source of meaningful roughness: - we assume the output types are the same as the input (to be fair, - outputs only contribute a little to the overall total). This combined - with a few bytes variation in signature sizes means we will expect, - say, 10% inaccuracy here. - - Assuming p2pkh: - out: 8+1+3+2+20=34, in: 1+32+4+1+1+~73+1+1+33=147, - ver:4,seq:4, +2 (len in,out) - total ~= 34*len_out + 147*len_in + 10 (sig sizes vary slightly) - Assuming p2sh M of N multisig: - "ins" must contain M, N so ins= (numins, M, N) (crude assuming all same) - 74*M + 34*N + 45 per input, so total ins ~ len_ins * (45+74M+34N) - so total ~ 34*len_out + (45+74M+34N)*len_in + 10 - Assuming p2sh-p2wpkh: - witness are roughly 3+~73+33 for each input - (txid, vin, 4+20 for witness program encoded as scriptsig, 4 for sequence) - non-witness input fields are roughly 32+4+4+20+4=64, so total becomes - n_in * 64 + 4(ver) + 4(locktime) + n_out*34 - Assuming p2wpkh native: - witness as previous case - non-witness loses the 24 witnessprogram, replaced with 1 zero, - in the scriptSig, so becomes: - n_in * 41 + 4(ver) + 4(locktime) +2 (len in, out) + n_out*34 - ''' - if txtype == 'p2pkh': - return 10 + ins * 147 + 34 * outs - elif txtype == 'p2sh-p2wpkh': - #return the estimate for the witness and non-witness - #portions of the transaction, assuming that all the inputs - #are of segwit type p2sh-p2wpkh - # Note as of Jan19: this misses 2 bytes (trivial) for len in, out - # and also overestimates output size by 2 bytes. - witness_estimate = ins*109 - non_witness_estimate = 4 + 4 + outs*34 + ins*64 - return (witness_estimate, non_witness_estimate) - elif txtype == 'p2wpkh': - witness_estimate = ins*109 - non_witness_estimate = 4 + 4 + 2 + outs*31 + ins*41 - return (witness_estimate, non_witness_estimate) - elif txtype == 'p2shMofN': - ins, M, N = ins - return 10 + (45 + 74*M + 34*N) * ins + 34 * outs - else: - raise NotImplementedError("Transaction size estimation not" + - "yet implemented for type: " + txtype) diff --git a/jmbitcoin/jmbitcoin/secp256k1_transaction.py b/jmbitcoin/jmbitcoin/secp256k1_transaction.py index b8982da35..aca9903c1 100644 --- a/jmbitcoin/jmbitcoin/secp256k1_transaction.py +++ b/jmbitcoin/jmbitcoin/secp256k1_transaction.py @@ -9,548 +9,95 @@ # note, only used for non-cryptographic randomness: import random from jmbitcoin.secp256k1_main import * -from jmbitcoin.bech32 import * -import jmbitcoin as btc -P2PKH_PRE, P2PKH_POST = b'\x76\xa9\x14', b'\x88\xac' -P2SH_P2WPKH_PRE, P2SH_P2WPKH_POST = b'\xa9\x14', b'\x87' -P2WPKH_PRE = b'\x00\x14' -P2WSH_PRE = b'\x00\x20' - -# Transaction serialization and deserialization - -def deserialize(txinp): - if isinstance(txinp, basestring) and not isinstance(txinp, bytes): - tx = BytesIO(binascii.unhexlify(txinp)) - hexout = True - else: - tx = BytesIO(txinp) - hexout = False - - def hex_string(scriptbytes, hexout): - if hexout: - return binascii.hexlify(scriptbytes).decode('ascii') - else: - return scriptbytes - - def read_as_int(bytez): - if bytez == 2: - return struct.unpack(b' len(newtx["outs"]): - raise Exception( - "Transactions with sighash single should have len in <= len out") - newtx["outs"] = newtx["outs"][:i+1] - for out in newtx["outs"][:i]: - out['value'] = 2**64 - 1 - out['script'] = "" - for j, inp in enumerate(newtx["ins"]): - if j != i: - inp["sequence"] = 0 - if hashcode & SIGHASH_ANYONECANPAY: - newtx["ins"] = [newtx["ins"][i]] - else: - pass - return newtx - -def segwit_txid(tx, hashcode=None): - #An easy way to construct the old-style hash (which is the real txid, - #the one without witness or marker/flag, is to remove all txinwitness - #entries from the deserialized form of the full tx, then reserialize, - #because serialize uses that as a flag to decide which serialization - #style to apply. - dtx = deserialize(tx) - for vin in dtx["ins"]: - if "txinwitness" in vin: - del vin["txinwitness"] - reserialized_tx = serialize(dtx) - return txhash(reserialized_tx, hashcode) - -def txhash(tx, hashcode=None, check_sw=True): - """ Creates the appropriate sha256 hash as required - either for signing or calculating txids. - The hashcode argument is used to distinguish the case - where we are hashing for signing (sighashing); by default - it is None, and this indicates we are calculating a txid. - If check_sw is True it checks the serialized format for - segwit flag bytes, and produces the correct form for txid (not wtxid). - """ - if not isinstance(tx, basestring): - tx = serialize(tx) - if isinstance(tx, basestring) and not isinstance(tx, bytes): - tx = binascii.unhexlify(tx) - if check_sw and from_byte_to_int(tx[4:5]) == 0: - if not from_byte_to_int(tx[5:6]) == 1: - #This invalid, but a raise is a DOS vector in some contexts. - return None - return segwit_txid(tx, hashcode) - if hashcode: - if not isinstance(hashcode, int): - hashcode = struct.unpack(b'B', hashcode)[0] - return dbl_sha256(from_string_to_bytes(tx) + struct.pack(b'= 0 - if not isinstance(tx, bytes): - tx = binascii.unhexlify(tx) - if not isinstance(script, bytes): - script = binascii.unhexlify(script) - if scriptCode is not None and not isinstance(scriptCode, bytes): - scriptCode = binascii.unhexlify(scriptCode) - if isinstance(sig, bytes): - sig = binascii.hexlify(sig).decode('ascii') - if isinstance(pub, bytes): - pub = binascii.hexlify(pub).decode('ascii') - - hashcode = binascii.unhexlify(sig[-2:]) - - if amount: - modtx = segwit_signature_form(deserialize(tx), i, - scriptCode, amount, hashcode, decoder_func=lambda x: x) - else: - modtx = signature_form(tx, i, script, hashcode) - return ecdsa_tx_verify(modtx, sig, pub, hashcode) - - -def sign(tx, i, priv, hashcode=SIGHASH_ALL, usenonce=None, amount=None, - native=False): - """ - Given a serialized transaction tx, an input index i, and a privkey - in bytes or hex, returns a serialized transaction, in hex always, - into which the signature and/or witness has been inserted. The field - `amount` flags whether segwit signing is to be done, and the field - `native` flags that native segwit p2wpkh signing is to be done. Note - that signing multisig is to be done with the alternative functions - get_p2sh_signature or get_p2wsh_signature (and non N of N multisig - scripthash signing is not currently supported). - """ - if isinstance(tx, basestring) and not isinstance(tx, bytes): - tx = binascii.unhexlify(tx) - if len(priv) <= 33: - priv = safe_hexlify(priv) - if amount: - serobj = p2wpkh_sign(tx, i, priv, amount, hashcode=hashcode, - usenonce=usenonce, native=native) - else: - pub = privkey_to_pubkey(priv, True) - address = pubkey_to_address(pub) - signing_tx = signature_form(tx, i, mk_pubkey_script(address), hashcode) - sig = ecdsa_tx_sign(signing_tx, priv, hashcode, usenonce=usenonce) - txobj = deserialize(tx) - txobj["ins"][i]["script"] = serialize_script([sig, pub]) - serobj = serialize(txobj) - if isinstance(serobj, basestring) and isinstance(serobj, bytes): - return binascii.hexlify(serobj).decode('ascii') - else: - return serobj - -def p2wpkh_sign(tx, i, priv, amount, hashcode=SIGHASH_ALL, native=False, - usenonce=None): - """Given a serialized transaction, index, private key in hex, - amount in satoshis and optionally hashcode, return the serialized - transaction containing a signature and witness for this input; it's - assumed that the input is of type pay-to-witness-pubkey-hash. - If native is False, it's treated as p2sh nested. + Given a transaction tx of type CMutableTransaction, an input index i, + and a raw privkey in bytes, updates the CMutableTransaction to contain + the newly appended signature. + Only three scriptPubKey types supported: p2pkh, p2wpkh, p2sh-p2wpkh. + Note that signing multisig must be done outside this function, using + the wrapped library. + Returns: (signature, "signing succeeded") + or: (None, errormsg) in case of failure """ - pub = privkey_to_pubkey(priv) - # Convert the input tx and script to hex so that the deserialize() - # call creates hex-encoded fields - script = binascii.hexlify(pubkey_to_p2wpkh_script(pub)).decode('ascii') - scriptCode = binascii.hexlify(pubkey_to_p2pkh_script(pub)).decode('ascii') - if isinstance(tx, bytes): - tx = binascii.hexlify(tx).decode('ascii') - signing_tx = segwit_signature_form(deserialize(tx), i, scriptCode, amount, - hashcode=hashcode) - sig = ecdsa_tx_sign(signing_tx, priv, hashcode, usenonce=usenonce) - txobj = deserialize(tx) - if not native: - txobj["ins"][i]["script"] = "16" + script - else: - txobj["ins"][i]["script"] = "" - txobj["ins"][i]["txinwitness"] = [sig, pub] - return serialize(txobj) - -def signall(tx, priv): - # if priv is a dictionary, assume format is - # { 'txinhash:txinidx' : privkey } - if isinstance(priv, dict): - for e, i in enumerate(deserialize(tx)["ins"]): - k = priv["%s:%d" % (i["outpoint"]["hash"], i["outpoint"]["index"])] - tx = sign(tx, e, k) - else: - for i in range(len(deserialize(tx)["ins"])): - tx = sign(tx, i, priv) - return tx + # script verification flags + flags = set() -def get_p2sh_signature(tx, i, redeem_script, pk, amount=None, hashcode=SIGHASH_ALL): - """ - Tx is assumed to be serialized. redeem_script is for example the - output of mk_multisig_script. - pk is the private key, and must be passed in hex. - If amount is not None, the output of get_p2wsh_signature is returned. - What is returned is a single signature. - """ - if isinstance(tx, str): - tx = binascii.unhexlify(tx) - if isinstance(redeem_script, str): - redeem_script = binascii.unhexlify(redeem_script) - if amount: - return get_p2wsh_signature(tx, i, redeem_script, pk, amount, hashcode) - modtx = signature_form(tx, i, redeem_script, hashcode) - return ecdsa_tx_sign(modtx, pk, hashcode) + def return_err(e): + return None, "Error in signing: " + repr(e) -def get_p2wsh_signature(tx, i, redeem_script, pk, amount, hashcode=SIGHASH_ALL): - """ See note to multisign for the value to pass in as `script`. - Tx is assumed to be serialized. - """ - modtx = segwit_signature_form(deserialize(tx), i, redeem_script, amount, - hashcode, decoder_func=lambda x: x) - return ecdsa_tx_sign(modtx, pk, hashcode) + assert isinstance(tx, CMutableTransaction) + # using direct local access to libsecp256k1 binding, because + # python-bitcoinlib uses OpenSSL key management: + pub = privkey_to_pubkey(priv) -def apply_p2wsh_multisignatures(tx, i, script, sigs): - """Sigs must be passed in as a list, and must be a - complete list for this multisig, in the same order - as the list of pubkeys when creating the scriptPubKey. - """ - if isinstance(script, str): - script = binascii.unhexlify(script) - sigs = [binascii.unhexlify(x) if x[:2] == '30' else x for x in sigs] - if isinstance(tx, str): - return safe_hexlify(apply_p2wsh_multisignatures( - binascii.unhexlify(tx), i, script, sigs)) - txobj = deserialize(tx) - txobj["ins"][i]["script"] = "" - txobj["ins"][i]["txinwitness"] = [None] + sigs + [script] - return serialize(txobj) + if not amount: + # p2pkh only supported here: + input_scriptPubKey = pubkey_to_p2pkh_script(pub) + sighash = SignatureHash(input_scriptPubKey, tx, i, hashcode) + try: + sig = ecdsa_raw_sign(sighash, priv, rawmsg=True) + bytes([hashcode]) + except Exception as e: + return return_err(e) + tx.vin[i].scriptSig = CScript([sig, pub]) + # Verify the signature worked. + try: + VerifyScript(tx.vin[i].scriptSig, + input_scriptPubKey, tx, i, flags=flags) + except Exception as e: + return return_err(e) + return sig, "signing succeeded" -def apply_multisignatures(*args): - # tx,i,script,sigs OR tx,i,script,sig1,sig2...,sig[n] - tx, i, script = args[0], int(args[1]), args[2] - sigs = args[3] if isinstance(args[3], list) else list(args[3:]) + else: + # segwit case; we currently support p2wpkh native or under p2sh. + + # see line 1256 of bitcointx.core.scripteval.py: + flags.add(SCRIPT_VERIFY_P2SH) + + input_scriptPubKey = pubkey_to_p2wpkh_script(pub) + # only created for convenience access to scriptCode: + input_address = P2WPKHBitcoinAddress.from_scriptPubKey(input_scriptPubKey) + # function name is misleading here; redeemScript only applies to p2sh. + scriptCode = input_address.to_redeemScript() + + sighash = SignatureHash(scriptCode, tx, i, hashcode, amount=amount, + sigversion=SIGVERSION_WITNESS_V0) + try: + sig = ecdsa_raw_sign(sighash, priv, rawmsg=True) + bytes([hashcode]) + except Exception as e: + return return_err(e) + if native: + flags.add(SCRIPT_VERIFY_WITNESS) + else: + tx.vin[i].scriptSig = CScript([input_scriptPubKey]) - if isinstance(script, str): - script = binascii.unhexlify(script) - sigs = [binascii.unhexlify(x) if x[:2] == '30' else x for x in sigs] - if isinstance(tx, str): - return safe_hexlify(apply_multisignatures( - binascii.unhexlify(tx), i, script, sigs)) + witness = [sig, pub] + ctxwitness = CTxInWitness(CScriptWitness(witness)) + tx.wit.vtxinwit[i] = ctxwitness + # Verify the signature worked. + try: + VerifyScript(tx.vin[i].scriptSig, input_scriptPubKey, tx, i, + flags=flags, amount=amount, witness=tx.wit.vtxinwit[i].scriptWitness) + except ValidationError as e: + return return_err(e) - txobj = deserialize(tx) - txobj["ins"][i]["script"] = serialize_script([None] + sigs + [script]) - return serialize(txobj) + return sig, "signing succeeded" def apply_freeze_signature(tx, i, redeem_script, sig): if isinstance(redeem_script, str): @@ -863,14 +213,16 @@ def apply_freeze_signature(tx, i, redeem_script, sig): return serialize(txobj) def mktx(ins, outs, version=1, locktime=0): - """ Given a list of input dicts with key "output" - which are txid:n strings in hex, and a list of outputs - which are dicts with keys "address", "value", outputs - a hex serialized tranasction encoding this data. + """ Given a list of input tuples (txid(bytes), n(int)), + and a list of outputs which are dicts with + keys "address" (value should be *str* not CCoinAddress), + "value" (value should be integer satoshis), outputs a + CMutableTransaction object. Tx version and locktime are optionally set, for non-default locktimes, inputs are given nSequence as per below comment. """ - txobj = {"locktime": locktime, "version": version, "ins": [], "outs": []} + vin = [] + vout = [] # This does NOT trigger rbf and mimics Core's standard behaviour as of # Jan 2019. # Tx creators wishing to use rbf will need to set it explicitly outside @@ -880,48 +232,36 @@ def mktx(ins, outs, version=1, locktime=0): else: sequence = 0xffffffff for i in ins: - if isinstance(i, dict) and "outpoint" in i: - txobj["ins"].append(i) - else: - if isinstance(i, dict) and "output" in i: - i = i["output"] - txobj["ins"].append({ - "outpoint": {"hash": i[:64], - "index": int(i[65:])}, - "script": "", - "sequence": sequence - }) + outpoint = CMutableOutPoint((i[0][::-1]), i[1]) + inp = CMutableTxIn(prevout=outpoint, nSequence=sequence) + vin.append(inp) for o in outs: - if isinstance(o, str): - addr = o[:o.find(':')] - val = int(o[o.find(':') + 1:]) - o = {} - if re.match('^[0-9a-fA-F]*$', addr): - o["script"] = addr - else: - o["address"] = addr - o["value"] = val - - outobj = {} - if "address" in o: - outobj["script"] = address_to_script(o["address"]) - elif "script" in o: - outobj["script"] = o["script"] - else: - raise Exception("Could not find 'address' or 'script' in output.") - outobj["value"] = o["value"] - txobj["outs"].append(outobj) - return serialize(txobj) - -def make_shuffled_tx(ins, outs, deser=True, version=1, locktime=0): - """ Simple utility to ensure transaction + # note the to_scriptPubKey method is only available for standard + # address types + out = CMutableTxOut(o["value"], + CCoinAddress(o["address"]).to_scriptPubKey()) + vout.append(out) + return CMutableTransaction(vin, vout, nLockTime=locktime) + +def make_shuffled_tx(ins, outs, version=1, locktime=0): + """ Simple wrapper to ensure transaction inputs and outputs are randomly ordered. Can possibly be replaced by BIP69 in future """ random.shuffle(ins) random.shuffle(outs) - tx = mktx(ins, outs, version=version, locktime=locktime) - if deser: - return deserialize(tx) - else: - return tx + return mktx(ins, outs, version=version, locktime=locktime) + +def verify_tx_input(tx, i, scriptSig, scriptPubKey, amount=None, + witness=None, native=False): + flags = set() + if witness: + flags.add(SCRIPT_VERIFY_P2SH) + if native: + flags.add(SCRIPT_VERIFY_WITNESS) + try: + VerifyScript(scriptSig, scriptPubKey, tx, i, + flags=flags, amount=amount, witness=witness) + except ValidationError as e: + return False + return True diff --git a/jmbitcoin/setup.py b/jmbitcoin/setup.py index 540938295..018025f3a 100644 --- a/jmbitcoin/setup.py +++ b/jmbitcoin/setup.py @@ -9,6 +9,6 @@ author_email='', license='GPL', packages=['jmbitcoin'], - install_requires=['future', 'coincurve', 'urldecode'], + install_requires=['future', 'coincurve', 'python-bitcointx', 'urldecode'], python_requires='>=3.5', zip_safe=False) diff --git a/jmbitcoin/test/test_addresses.py b/jmbitcoin/test/test_addresses.py deleted file mode 100644 index d7723b1f9..000000000 --- a/jmbitcoin/test/test_addresses.py +++ /dev/null @@ -1,56 +0,0 @@ -import jmbitcoin as btc -import json -import pytest -import os -testdir = os.path.dirname(os.path.realpath(__file__)) - -def validate_address(addr, nettype): - """A mock of jmclient.validate_address - """ - BTC_P2PK_VBYTE = {"mainnet": b'\x00', "testnet": b'\x6f'} - BTC_P2SH_VBYTE = {"mainnet": b'\x05', "testnet": b'\xc4'} - try: - ver = btc.get_version_byte(addr) - except AssertionError as e: - return False, 'Checksum wrong. Typo in address?' - except Exception as e: - return False, "Invalid bitcoin address" - if ver not in [BTC_P2PK_VBYTE[nettype], BTC_P2SH_VBYTE[nettype]]: - return False, 'Wrong address version. Testnet/mainnet confused?' - if len(btc.b58check_to_bin(addr)) != 20: - return False, "Address has correct checksum but wrong length." - return True, 'address validated' - -@pytest.mark.parametrize( - "net", - [ - # 1 - ("mainnet"), - # 2 - ("testnet") - ]) -def test_b58_invalid_addresses(net): - #none of these are valid as any kind of key or address - with open(os.path.join(testdir,"base58_keys_invalid.json"), "r") as f: - json_data = f.read() - invalid_key_list = json.loads(json_data) - for k in invalid_key_list: - bad_key = k[0] - res, message = validate_address(bad_key, nettype=net) - assert res == False, "Incorrectly validated address: " + bad_key + " with message: " + message - -def test_b58_valid_addresses(): - with open(os.path.join(testdir,"base58_keys_valid.json"), "r") as f: - json_data = f.read() - valid_keys_list = json.loads(json_data) - for a in valid_keys_list: - addr, pubkey, prop_dict = a - if not prop_dict["isPrivkey"]: - if prop_dict["isTestnet"]: - net = "testnet" - else: - net = "mainnet" - #if using pytest -s ; sanity check to see what's actually being tested - res, message = validate_address(addr, net) - assert res == True, "Incorrectly failed to validate address: " + addr + " with message: " + message - diff --git a/jmbitcoin/test/test_bech32.py b/jmbitcoin/test/test_bech32.py deleted file mode 100644 index ec5de0d09..000000000 --- a/jmbitcoin/test/test_bech32.py +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/python - -# Copyright (c) 2017 Pieter Wuille -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - - -"""Reference tests for segwit adresses""" - -import binascii -import unittest -import jmbitcoin as btc - -VALID_CHECKSUM = [ - "A12UEL5L", - "an83characterlonghumanreadablepartthatcontainsthenumber1andtheexcludedcharactersbio1tt5tgs", - "abcdef1qpzry9x8gf2tvdw0s3jn54khce6mua7lmqqqxw", - "11qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqc8247j", - "split1checkupstagehandshakeupstreamerranterredcaperred2y9e3w", -] - -INVALID_CHECKSUM = [ - " 1nwldj5", - "\x7F" + "1axkwrx", - "an84characterslonghumanreadablepartthatcontainsthenumber1andtheexcludedcharactersbio1569pvx", - "pzry9x0s0muk", - "1pzry9x0s0muk", - "x1b4n0q5v", - "li1dgmt3", - "de1lg7wt\xff", -] - -VALID_ADDRESS = [ - ["BC1QW508D6QEJXTDG4Y5R3ZARVARY0C5XW7KV8F3T4", "0014751e76e8199196d454941c45d1b3a323f1433bd6"], - ["tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q0sl5k7", - "00201863143c14c5166804bd19203356da136c985678cd4d27a1b8c6329604903262"], - ["bc1pw508d6qejxtdg4y5r3zarvary0c5xw7kw508d6qejxtdg4y5r3zarvary0c5xw7k7grplx", - "5128751e76e8199196d454941c45d1b3a323f1433bd6751e76e8199196d454941c45d1b3a323f1433bd6"], - ["BC1SW50QA3JX3S", "6002751e"], - ["bc1zw508d6qejxtdg4y5r3zarvaryvg6kdaj", "5210751e76e8199196d454941c45d1b3a323"], - ["tb1qqqqqp399et2xygdj5xreqhjjvcmzhxw4aywxecjdzew6hylgvsesrxh6hy", - "0020000000c4a5cad46221b2a187905e5266362b99d5e91c6ce24d165dab93e86433"], -] - -INVALID_ADDRESS = [ - "tc1qw508d6qejxtdg4y5r3zarvary0c5xw7kg3g4ty", - "bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t5", - "BC13W508D6QEJXTDG4Y5R3ZARVARY0C5XW7KN40WF2", - "bc1rw5uspcuh", - "bc10w508d6qejxtdg4y5r3zarvary0c5xw7kw508d6qejxtdg4y5r3zarvary0c5xw7kw5rljs90", - "BC1QR508D6QEJXTDG4Y5R3ZARVARYV98GJ9P", - "tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q0sL5k7", - "bc1zw508d6qejxtdg4y5r3zarvaryvqyzf3du", - "tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3pjxtptv", - "bc1gmk9yu", - -] - -INVALID_ADDRESS_ENC = [ - ("BC", 0, 20), - ("bc", 0, 21), - ("bc", 17, 32), - ("bc", 1, 1), - ("bc", 16, 41), -] - -class TestSegwitAddress(unittest.TestCase): - """Unit test class for segwit addressess.""" - - def test_valid_checksum(self): - """Test checksum creation and validation.""" - for test in VALID_CHECKSUM: - hrp, _ = btc.bech32_decode(test) - self.assertIsNotNone(hrp) - pos = test.rfind('1') - test = test[:pos+1] + chr(ord(test[pos + 1]) ^ 1) + test[pos+2:] - hrp, _ = btc.bech32_decode(test) - self.assertIsNone(hrp) - - def test_invalid_checksum(self): - """Test validation of invalid checksums.""" - for test in INVALID_CHECKSUM: - hrp, _ = btc.bech32_decode(test) - self.assertIsNone(hrp) - - def test_valid_address(self): - """Test whether valid addresses decode to the correct output.""" - for (address, hexscript) in VALID_ADDRESS: - hrp = "bc" - witver, witprog = btc.bech32addr_decode(hrp, address) - if witver is None: - hrp = "tb" - witver, witprog = btc.bech32addr_decode(hrp, address) - self.assertIsNotNone(witver) - scriptpubkey = btc.segwit_scriptpubkey(witver, witprog) - self.assertEqual(scriptpubkey, binascii.unhexlify(hexscript)) - addr = btc.bech32addr_encode(hrp, witver, witprog) - self.assertEqual(address.lower(), addr) - - def test_invalid_address(self): - """Test whether invalid addresses fail to decode.""" - for test in INVALID_ADDRESS: - witver, _ = btc.bech32addr_decode("bc", test) - self.assertIsNone(witver) - witver, _ = btc.bech32addr_decode("tb", test) - self.assertIsNone(witver) - - def test_invalid_address_enc(self): - """Test whether address encoding fails on invalid input.""" - for hrp, version, length in INVALID_ADDRESS_ENC: - code = btc.bech32addr_encode(hrp, version, [0] * length) - self.assertIsNone(code) - -if __name__ == "__main__": - unittest.main() diff --git a/jmbitcoin/test/test_bip32.py b/jmbitcoin/test/test_bip32.py index d0eb5b11b..f8b6c5340 100644 --- a/jmbitcoin/test/test_bip32.py +++ b/jmbitcoin/test/test_bip32.py @@ -103,9 +103,9 @@ def test_ckd_pubkeys(): def test_bip32_descend(): master = btc.bip32_master_key(b'\x07'*32) end_key = btc.bip32_descend(master, [2, 3, 10000]) - assert end_key=="6856ef965940a1a7b1311dc041050ac0013e326c7ff4e2c677a7694b4f0405c901" + assert end_key==binascii.unhexlify("6856ef965940a1a7b1311dc041050ac0013e326c7ff4e2c677a7694b4f0405c901") end_key = btc.bip32_descend(master, 2, 5, 4, 5) - assert end_key=="d2d816b6485103c0d7ff95482788f0e8e73fa11817079e006d47979d8196c4b101" + assert end_key==binascii.unhexlify("d2d816b6485103c0d7ff95482788f0e8e73fa11817079e006d47979d8196c4b101") diff --git a/jmbitcoin/test/test_btc_formatting.py b/jmbitcoin/test/test_btc_formatting.py deleted file mode 100644 index 66475d54a..000000000 --- a/jmbitcoin/test/test_btc_formatting.py +++ /dev/null @@ -1,57 +0,0 @@ -#! /usr/bin/env python -'''Test bitcoin module data handling''' - -import jmbitcoin as btc -import pytest -import binascii - -#used in p2sh addresses -def test_hash160(): - assert '0e3397b4abc7a382b3ea2365883c3c7ca5f07600' == \ - btc.hash160('The quick brown fox jumps over the lazy dog') - -def test_bad_code_string(): - for i in [1,9,257,-3,"256"]: - with pytest.raises(ValueError) as e_info: - btc.get_code_string(i) - -#Tests of compactsize encoding, see: -#https://bitcoin.org/en/developer-reference#compactsize-unsigned-integers -#note that little endian is used. -@pytest.mark.parametrize( - "num, compactsize", - [ - (252, "fc"), - (253, "fdfd00"), - (254, "fdfe00"), - (515, "fd0302"), - (65535, "fdffff"), - (65536, "fe00000100"), - (65537, "fe01000100"), - (4294967295, "feffffffff"), - (4294967296, "ff0000000001000000"), - - ]) -def test_compact_size(num, compactsize): - assert btc.num_to_var_int(num) == binascii.unhexlify(compactsize) - -@pytest.mark.parametrize("frm, to", [ - (("16001405481b7f1d90c5a167a15b00e8af76eb6984ea59"), - ["001405481b7f1d90c5a167a15b00e8af76eb6984ea59"]), - (("483045022100ad0dda327945e581a5effd83d75d76a9f07c3128f4dc6d25a54" - "e5ad5dd629bd00220487a992959bd540dbc335c655e6485ebfb394129eb48038f" - "0a2d319782f7cb690121039319452b6abafb5fcf06096196d0c141b8bd18a3de7" - "e9b9352da800d671ccd84"), - ["3045022100ad0dda327945e581a5effd83d75d76a9f07c3128f4dc6d25a54e5ad5dd629bd00220487a992959bd540dbc335c655e6485ebfb394129eb48038f0a2d319782f7cb6901", - "039319452b6abafb5fcf06096196d0c141b8bd18a3de7e9b9352da800d671ccd84"]), - (("51"), [1]), - (("00"), [None]), - (("510000"), [1, None, None]), - (("636505aaaaaaaaaa53"), [99, 101, "aaaaaaaaaa", 3]), - (("51" + "4d0101" + "aa"*257), [1, "aa"*257]), - (("4e" + "03000100" + "aa"*65539), ["aa"*65539]), -]) -def test_deserialize_script(frm, to): - #print(len(btc.deserialize_script(frm)[0])) - assert btc.deserialize_script(frm) == to - assert btc.serialize_script(to) == frm diff --git a/jmbitcoin/test/test_ecc_signing.py b/jmbitcoin/test/test_ecc_signing.py index 570a29262..e3e9a7399 100644 --- a/jmbitcoin/test/test_ecc_signing.py +++ b/jmbitcoin/test/test_ecc_signing.py @@ -12,25 +12,23 @@ def test_valid_sigs(setup_ecc): for v in vectors['vectors']: - msg = v['msg'] - sig = v['sig'] - priv = v['privkey'] - assert sig == btc.ecdsa_raw_sign(msg, priv, True, rawmsg=True)+'01' - #check that the signature verifies against the key(pair) - pubkey = btc.privtopub(priv) - assert btc.ecdsa_raw_verify(msg, pubkey, sig[:-2], True, rawmsg=True) - #check that it fails to verify against corrupted signatures + msg, sig, priv = (binascii.unhexlify( + v[a]) for a in ["msg", "sig", "privkey"]) + assert sig == btc.ecdsa_raw_sign(msg, priv, rawmsg=True)+ b'\x01' + # check that the signature verifies against the key(pair) + pubkey = btc.privkey_to_pubkey(priv) + assert btc.ecdsa_raw_verify(msg, pubkey, sig[:-1], rawmsg=True) + # check that it fails to verify against corrupted signatures for i in [0,1,2,4,7,25,55]: - #corrupt one byte - binsig = binascii.unhexlify(sig) - checksig = binascii.hexlify(binsig[:i] + btc.from_string_to_bytes(chr( - (ord(binsig[i:i+1])+1) %256)) + binsig[i+1:-1]).decode('ascii') + # corrupt one byte + checksig = sig[:i] + chr( + (ord(sig[i:i+1])+1) %256).encode() + sig[i+1:-1] - #this kind of corruption will sometimes lead to an assert - #failure (if the DER format is corrupted) and sometimes lead - #to a signature verification failure. + # this kind of corruption will sometimes lead to an assert + # failure (if the DER format is corrupted) and sometimes lead + # to a signature verification failure. try: - res = btc.ecdsa_raw_verify(msg, pubkey, checksig, True, rawmsg=True) + res = btc.ecdsa_raw_verify(msg, pubkey, checksig, rawmsg=True) except: continue assert res==False diff --git a/jmbitcoin/test/test_main.py b/jmbitcoin/test/test_main.py deleted file mode 100644 index 5f2bf106d..000000000 --- a/jmbitcoin/test/test_main.py +++ /dev/null @@ -1,61 +0,0 @@ -#! /usr/bin/env python -'''Testing mostly exceptional cases in secp256k1_main. - Some of these may represent code that should be removed, TODO.''' - -import jmbitcoin as btc -import binascii -import pytest -import os -testdir = os.path.dirname(os.path.realpath(__file__)) - -def test_hex2b58check(): - assert btc.hex_to_b58check("aa"*32) == "12JAT9y2EcnV6DPUGikLJYjWwk5UmUEFXRiQVmTbfSLbL3njFzp" - -def test_bindblsha(): - assert btc.bin_dbl_sha256("abc") == binascii.unhexlify( - "4f8b42c22dd3729b519ba6f68d2da7cc5b2d606d05daed5ad5128cc03e6c6358") - -def test_lpad(): - assert btc.lpad("aaaa", "b", 5) == "baaaa" - assert btc.lpad("aaaa", "b", 4) == "aaaa" - assert btc.lpad("aaaa", "b", 3) == "aaaa" - -def test_safe_from_hex(): - assert btc.safe_from_hex('ff0100') == b'\xff\x01\x00' - -def test_hash2int(): - assert btc.hash_to_int("aa"*32) == \ - 77194726158210796949047323339125271902179989777093709359638389338608753093290 - -@btc.hexbin -def dummyforwrap(a, b, c, d="foo", e="bar"): - newa = a+b"\x01" - x, y = b - newb = [x+b"\x02", y+b"\x03"] - if d == "foo": - return newb[1] - else: - return newb[0] - -def test_hexbin(): - assert dummyforwrap("aa", ["bb", "cc"], True) == "cc03" - assert dummyforwrap("aa", ["bb", "cc"], True, d="baz") == "bb02" - assert dummyforwrap(b"\xaa", [b"\xbb", b"\xcc"], False) == b"\xcc\x03" - -def test_add_privkeys(): - with pytest.raises(Exception) as e_info: - btc.add_privkeys("aa"*32, "bb"*32+"01", True) - -def test_ecdsa_raw_sign(): - msg = "aa"*31 - with pytest.raises(Exception) as e_info: - btc.ecdsa_raw_sign(msg, None, None, rawmsg=True) - assert e_info.match("Invalid hash input") - #build non-raw priv object as input - privraw = "aa"*32 - msghash = b"\xbb"*32 - sig = binascii.hexlify(btc.ecdsa_raw_sign(msghash, privraw, False, rawpriv=False, rawmsg=True)).decode('ascii') - assert sig == "3045022100b81960b4969b423199dea555f562a66b7f49dea5836a0168361f1a5f8a3c8298022003eea7d7ee4462e3e9d6d59220f950564caeb77f7b1cdb42af3c83b013ff3b2f" - - - \ No newline at end of file diff --git a/jmbitcoin/test/test_tx_serialize.py b/jmbitcoin/test/test_tx_serialize.py deleted file mode 100644 index 225ee5d62..000000000 --- a/jmbitcoin/test/test_tx_serialize.py +++ /dev/null @@ -1,234 +0,0 @@ -import jmbitcoin as btc -import pytest -import json -import os -testdir = os.path.dirname(os.path.realpath(__file__)) - -#TODO: fold these examples into the tx_valid.json file -@pytest.mark.parametrize( - "tx_type, tx_id, tx_hex", - [("simple-tx", - "f31916a1d398a4ec18d56a311c942bb6db934cee6aa8ac30af0b30aad9efb841", - "0100000001c74265f31fc5e24895fdc83f7157cc40045235f3a71ae326a219de9de873" + - "0d8b010000006a473044022076055917470b7ec4f4bb008096266cf816ebb089ad983e" + - "6a0f63340ba0e6a6cb022059ec938b996a75db10504e46830e13d399f28191b9832bd5" + - "f61df097b9e0d47801210291941334a00959af4aa5757abf81d2a7d1aca8adb3431c67" + - "e89419271ba71cb4feffffff023cdeeb03000000001976a914a2426748f14eba44b3f6" + - "abba3e8bce216ea233f388acf4ebf303000000001976a914bfa366464a464005ba0df8" + - "6024a6c3ed859f03ac88ac33280600"), - ("3partycoinjoin", - "d91278e125673f5b9201456b0c36efac3b2b6700fdd04fc2227352a63f941170", - "010000000cedf1756d1d679de268a533c2d009f1d06ba8918908369c5e3b818678603a" + - "4bcb030000006b483045022100a775815dcfaf706ee9b6e6015f162d20d227eb44cf88" + - "93fb3ac95f910058fbd50220500f1f3de75b3bdf7f6a0949528f836c694a5bd1effedb" + - "b02156bcb292a78e5d012102a726879b9d663467ab71726ff5469c8b0a4213a93643f6" + - "ed739fdbb1e81ca5b9ffffffff3e0ebec9b42f30657ef187bd263c57849541014700cc" + - "5480268dc63ce6997552070000006b4830450221008ec354dcb5d661a60651e988356a" + - "215ccaaa688be1e78861b3b89f35e0c1bb7b02203d2117791b74ad2aa594e4dc806d03" + - "ec2a1241c4955b013dc365b6eacc4ee6c9012102632c8b13d9b94f2226c28295b52686" + - "4bbece7ea283a90ca001e498e2a08dbebcffffffff841e4ed900a6592687d7684ec0dd" + - "0893d7a3b86df07bb2626a1f51ae74ee0ba6010000006a473044022017f96effaf9812" + - "0745323859e1b52699baf0993675d028882e42310deac6921102200daaab3008e9bce1" + - "2a3e136a7e0472a12da31121237ac382a6a64e4ab5ed2e5201210365a1491d9c10f866" + - "0a02f47a54e5c0285f186a9f5b4cb75405548cef3427c4eeffffffff7e131188b66001" + - "c0f5a4b45bd3503d2ce17f40e43e3803a7f30c20a8214abc95080000006a4730440220" + - "6c21a02e4bfcce3f9a5775ce7b9033bebb354b15167dec1a1b3f0c7255633234022071" + - "994111c455b1ddd4cd093d0d4f2ecbbf1283a0a1f982962127768e3c4e251301210394" + - "1f5e1834e8b1c1503f2d20b3f3e06db2b80de47f7f31b4cb39f75525753822ffffffff" + - "955544430628ea06493caa541d59374d7708dbef0e59a6653ac025f341c888c8040000" + - "006a473044022022c2afb23a7d401cdfff735ba390fac8dd2cbde2dcce4c3a8f988dfd" + - "a08b956c02204ba4ca6dc72993da6bf3d020530f01d7fc931ef28c78c2fa9b70657e50" + - "8873c801210240dc515cf540c43575615f18670d3f511f6f47b719650be60183ad9f61" + - "f805cdffffffffedf1756d1d679de268a533c2d009f1d06ba8918908369c5e3b818678" + - "603a4bcb010000006b483045022100ee3b322aab4c77debc6ebf48967f7677c03cd5fb" + - "df775885691f1162cca1f04f0220031a8c4017618562f4a2f4f0ece98bf13063c9e03b" + - "720b2eb3ead0ef024bee1f012103ce920bc0f19adf76e0f8b36eb62873d3cba991ee3e" + - "6504e0dbf114a0d5865827ffffffffa8b4085018a3e8848531eecf3164ec2b89ed5ed2" + - "172e22a19e3343f29faa3c03050000006a47304402202ff247fb0ec1eb9fa24903957d" + - "8973eb9a3a8e2dc4d6e6dbbb366c7563b49e680220301a0df3f1dbff1f939f14b3bf78" + - "687d8a2da26dafcb310ade85efd9cd37bd050121038cfc01875ccaacd4863515259e52" + - "b96188e83ca30358bce4111c4021bfc66343fffffffff5e7b4d41b53a1fce37b349d02" + - "33aed80be2b5da0ff45c59014ed5e4ff09b0a5030000006a473044022015035665d467" + - "19031a1e09fffb92484365a38f8c51171046bedc0eea1385cb7402200a296a8315f9a6" + - "3fe9a7020d4fa3be20a2c0587f98ae0d8d0560985d9cfb0748012103d9537b8ccb7206" + - "383edff317fcb1e4188036a18a84a6727104dd9b4d00638037ffffffffedf1756d1d67" + - "9de268a533c2d009f1d06ba8918908369c5e3b818678603a4bcb000000006a47304402" + - "200742b54bb0d8e62dbb87eaa00823152ff46106824d791bcb538e47b9f52e9df90220" + - "705de19cdc96c9dcb9f8dfff070e780c129524a56f369dc3633f69b1472c41d2012102" + - "90428148e18a34190de8a0b9a7ab7e9353feac7bd7bb57dde4acb2a03d1dfb94ffffff" + - "ff841e4ed900a6592687d7684ec0dd0893d7a3b86df07bb2626a1f51ae74ee0ba60c00" + - "00006a4730440220719b2baf0422e98d7462d120c3406e7e3657cceb8fabe84ddece55" + - "d34affb4620220168f87c8bcd57f903104b5effc0feffe3bd776dce6c09bfda8a3a36c" + - "95788ec701210263c387f7b141cf649a5296096976b6b622cd44a611c0e485724f9432" + - "9cf22b79ffffffff3e0ebec9b42f30657ef187bd263c57849541014700cc5480268dc6" + - "3ce6997552010000006a47304402206e62849a47494e9294bd7e47d87b9d32a6b69097" + - "067184b551350b7b34291d46022017ca461f6bcceb97432e9890700a68a4df4f4c5cfd" + - "272926af00a05dfdb3c07b0121024decad000f3aabfdac5ea8f132e457f0056b1f2cab" + - "0c946884d2ff0afbdd68b8ffffffff7e131188b66001c0f5a4b45bd3503d2ce17f40e4" + - "3e3803a7f30c20a8214abc95090000006b483045022100fa9c274892db835066eeac6c" + - "8f25d9507b43c93f4557b16560745e627387d7780220461025024a97816ff0567dcdca" + - "4702b5238894ee7028d04af48a7eb11bd3520301210391c94422cd25ee416178827513" + - "365ee49e4b2f17cdf0b07aed04353c10e34f6dffffffff06008c8647000000001976a9" + - "14e924c7d01f0201df5b1465aed50012c4dffa0ff988ac89ac7800000000001976a914" + - "3dc5ae7471acbba55107b651ce20177b3b44721388ac9f258509000000001976a91400" + - "656d12c4b48359f8637bcf96f9f02ceb2fad1c88ac4de9ec04000000001976a914f7de" + - "454e8b402cbde90f4a325de69236040f144088ac008c8647000000001976a914b7d9f8" + - "29f1bd6919728f91f8a10ce0ca87bde1aa88ac008c8647000000001976a914b877256a" + - "588a315e30b2c5aec38ff70cea71dd0c88ac00000000"), - ("4partycoinjoin", - "55eac9d4a4159d4ba355122c6c18f85293c19ae358306a3773ec3a5d053e2f1b", - "01000000108048bbbc26c394d45b514835d998ae2679c31cecd7623041372703f469aa" + - "ff8e040000006b4830450220293480c5975676ddbb8b66fc6e4bc53668ae4cc59d0da5" + - "377e520f3f360ad15e022100fe6aaf3d890570d66728210afb13e221f0405bbbf43262" + - "16371459877531a2d80121033392a42f2a8e4e93a5d7b50b4d26eabda42e04e110e77e" + - "22b514982a832dde92ffffffff587af588bbe3f911f8234a858cdc62a1c04c12cf09f5" + - "95f92a7292fd3f54d17e080000006b483045022100c47a07c3dd537e1e11216dcf3440" + - "619ec80a88d25f91f8b9de74b348947ba47902205e5a335e2c75b202b75ff8a50a2560" + - "f49542fe4e96598be6fe77b00bc7c33484012103e21d83d80666a020c3e4657f7155ac" + - "4572edf9fc79d59b5b69e27f23ff5f8df5ffffffff8048bbbc26c394d45b514835d998" + - "ae2679c31cecd7623041372703f469aaff8e0b0000006b48304502202a5403bfed1826" + - "c0d5b4143229518d6c0dc6b0881ad04e26039cef4981aa9487022100cecad7bd8fd476" + - "2257f83080bf2f56249bb36554137e9af9c1588f8f2ed170a9012103fd52e4f1e64cdd" + - "003528cf5f388ad455d001ea691bd02125d694117a1d481bbcffffffff8048bbbc26c3" + - "94d45b514835d998ae2679c31cecd7623041372703f469aaff8e110000006b48304502" + - "204e024b69b4a78ea9daa63ad1d8dba5e8729fbb61749dacc49be20053db8c02710221" + - "00c24555b43e6e838fd7cb797cbf96c278531af29493aaf719bdfe3354941d0b320121" + - "02a322a78564a976271153b0e289d631e5c6a217542d540c94262c40a83e48281affff" + - "ffff8048bbbc26c394d45b514835d998ae2679c31cecd7623041372703f469aaff8e05" + - "0000006b483045022100d3c7d326e233b2238576aecf53d5c3dfb88a3b41d815ff2cfd" + - "c3e2c5664aa52102204f0dc1cd110e4317ab435533a4440bbdabee75f360551bb46206" + - "79043058288801210310950ae84cfb02e24960c7227e40dedbc8804397a19e277233b1" + - "f3ecc797fdc2ffffffff9dd9d7a38ab7f29f17c4f32ae5be1426f80ff58c158fc7c74a" + - "afb9d53659cd05050000006b483045022100cc46ea21d4a1622cf7507ab9c96863b5fa" + - "3744c8c4b34a74c7bfdb92af41f40702200def9598dfcae25abaac047969f231c565c0" + - "6df6f7cb8f3c588e390049b0217801210202d1c96a5b459ca4fd430afc4d1b0e468a2a" + - "ba84c48ab5f5ad906ec09f55cfb3ffffffff84dc724e49f71d2a82cc46d781b1f47a21" + - "18c4056ab6dad2a83f03c7ed4a08c6040000006b483045022100ee5db129b8264b8a12" + - "e85fbf29566ad7edb84704214ced4dbea3e628149f08af02201149ab3c4e0aa6fa48ec" + - "4816ecefda6105a6df5838e9a6dbe235d156898d3b40012102da5240065576c7567c3e" + - "cc8dc09f3fda64a0c6b89c3974a7506bdeb7fb440704ffffffff68e787d07a48d3ba00" + - "d8160311ff0533684dd66c7ddba8f7d74af3862d98bd8a020000006b483045022100a8" + - "0edec0b7a6946fbcb877f95c48e22e26710f3f7a7ec9e5dd0e7622832fca3002204a8f" + - "ecc644f674cfa8c5ba0682318f5b6cd8cc5f0c15b70e5e423e3e0888e6550121032708" + - "198b4c046db76f9e0ff7350d05243950738b39c431f55d85d5ec886a1389ffffffff21" + - "57aecaef0b44114b1035d6630a436ee3a6ecbeebfca48b5ce8fa6dbc5e100700000000" + - "6b483045022100df2085a721c026328ea6dec9307c45d5991d51f696c15896ed14d40c" + - "dc69a3d302205a598476fa84a37c5eb70bab35d4a414fce63bc8bedec7883a724ea1f7" + - "d4b98401210271f768f5b16806df11ee81619c1371864707de25b27df94bd4516b6680" + - "2f0709ffffffff7cabc836092178f7325571627777476d1796e446ac8039dd706754b6" + - "824ee723040000006a473044022036eedb5259b74f8a99cba9f4866fafce94b5b7ebf7" + - "d248e973bdba27ac1161b202203d58951892c1680fafc77dd6ee1a9c27754018b96030" + - "a51c8891c1f5273c077f01210271f768f5b16806df11ee81619c1371864707de25b27d" + - "f94bd4516b66802f0709ffffffff84dc724e49f71d2a82cc46d781b1f47a2118c4056a" + - "b6dad2a83f03c7ed4a08c6000000006a473044022014a918294f28157288ab1e66e22b" + - "fe5e5cb92dde076e133b5d1d807bfd252048022020f8ef9d0c094b476b31f177fb083c" + - "b6e65f5167a086fd25bebe5012e46ae8b601210271f768f5b16806df11ee81619c1371" + - "864707de25b27df94bd4516b66802f0709ffffffff4c8670675a981c70935cbd0a0de4" + - "98c9b28c41784d590f1b9e59930e64dd53f5020000006a473044022067b08694963116" + - "f7d53522c2327b46150084b7b9cf5b279daa1461492dffd7010220453b1fd98cdc26fe" + - "d9385ec578da1dd11d1ca7367ebfd5586f4ddb913feabb6c01210397db984e478086ca" + - "f0bf90181c3c597905a52416e152bff007dffd8edf06e2deffffffff8048bbbc26c394" + - "d45b514835d998ae2679c31cecd7623041372703f469aaff8e070000006c4930460221" + - "0094e5123e70426ff6732bebbcaaac2d871b873880d77c71a97507a074fe1046000221" + - "00c67b7d7663817507dd03f00fd2b690d33daa645a13b6966ec1c76e5a8df0b0a40121" + - "035a9e5cdd37824ad390d73abf2c7bc902de6dd7e4ab92fcf4e84506747e8c6cdaffff" + - "ffff8048bbbc26c394d45b514835d998ae2679c31cecd7623041372703f469aaff8e01" + - "0000006b483045022100cfab7cb7b645e201515b756439ba943fecb77e68c76c482de5" + - "225bf7a80ca94a022061600e67eaf33b59c5863de617bdb79f63e69b310710cb2b7f4f" + - "63d9d64fb3bd012102dabd952ef6ebfe396dfd66436089f5df731dd8300ecb6d88eefe" + - "458bc73a4c8fffffffff11565b2030be339919eaa5d1c1b47f7d265485c2e230fe44d9" + - "5ce41f3f6d5bdc030000006b483045022021d350b215077c4b1aee83c747a0cabbe148" + - "2fc4ccc97657f07d481df3f89d45022100b16d629e35e857506af6e8dd74373f370f4b" + - "965f332a488878dc983f554378c40121022cde2859fb62f3b671a887fcce054626cdb2" + - "8ed4d20d0dfcf8e0b090e2d7fe4effffffff8048bbbc26c394d45b514835d998ae2679" + - "c31cecd7623041372703f469aaff8e0c0000006a47304402203adf6153783989ca7ec1" + - "fa12785e5abae5c6bd8a4e26bb0c2ae0cbc4635af981022056faa34a713bede42d0efe" + - "af84c35506f47d487bed59fba2de53167fdad64c570121025d3368bbb24f5980fc6081" + - "d2a1e0e04675d11751a53d13d896cff7b5940ac205ffffffff1cb00400000000000019" + - "76a9146418ea7a9cbcfbd9dfb4f552c00b834072ac833a88ac40420f00000000001976" + - "a91492f8428eae5228083b588b330f9dfc1cccd300fc88ac40420f00000000001976a9" + - "14871e08ec7dd94ef87fd7722ee703007badb7020d88ace8030000000000001976a914" + - "2ced0ea9992d7e5492b9053437702c83ea0750c088ac40420f00000000001976a91449" + - "464457f02609ba1d54153eaba614ab84d593cb88ac877fc30f000000001976a9141c20" + - "8af75037715bdca6c42f29e104af62204e4788ac540b0000000000001976a914cfccd5" + - "185a3c4ced7514fa584eee6ebd895c984888ac73037000000000001976a9149f0ddeb2" + - "473d013452bce71973d7515c0b4cd5cf88ac40420f00000000001976a9142cb4e342c4" + - "398a08397a8cedfd270e432e977da488ac09c2e005000000001976a914ca9ec96314a5" + - "f4842e3fe33a0cf6d047a4398d0a88ac40420f00000000001976a914cd0124b0969f16" + - "4132301032a0f99de5c8d8471e88ace9320400000000001976a9145c5f87677c7b50bc" + - "4fe88bdec9f2221a0ca3c35088ac40420f00000000001976a914fa40beb4e557e14d5e" + - "461c7aa66d674ea6b7ce5988ac40420f00000000001976a914564bdac0f31da4f35ce5" + - "3c9aaef7e796ed7acdcd88ace33c2b00000000001976a9142ef77132983b843b07396b" + - "731d213e3a60b4c6e488ac40420f00000000001976a9144aa2c6a81b4ad8c9414b1039" + - "d5b2c51d3416233888ac40420f00000000001976a914713667aa801d3d7cbf9c3d1667" + - "626131a9937a6288ac40420f00000000001976a9142fa24ea7272cbb3ee06beeb274b3" + - "486c3d83f60188ac40420f00000000001976a9147c2e0ee7484234b0a20fab40855dff" + - "93c3b195da88ac40420f00000000001976a914123d15de9b64aea49a9cf81d781fdfad" + - "00287acf88ac40420f00000000001976a91463f456082de5dbd63c7ab9cd511813d836" + - "a2a7cd88ac306a2e00000000001976a9144103b28ac18a4ad0ce075fe323ee3d38a95c" + - "4d2188acc0541d00000000001976a91484d1b7dadb4638895fc5eb79145dca39daea86" + - "ca88acb80b0000000000001976a9146baff4a7a65316aed38c032c6eded2e48111fb62" + - "88ac34080000000000001976a914a5e54586ea1626fc90359f55fa4a291c99cffae488" + - "accdc30100000000001976a914d547e4e5b7eb382f9309622bdf0271836083936988ac" + - "40420f00000000001976a914a0b17374b3085de303202b29439957ccd71ba13088acde" + - "af2600000000001976a914af4d27ced320ae73068d0e2bb2c3abfe6fbe2c6988ac0000" + - "0000"), - ("p2sh-in-to-p2sh-out", - "355a6090bfdb77d1d7fd7c67ba6a711b9f493344e372928fe79ca0206b34796b", - "0100000001b5bbd7adb1f9f6f599d8e547f7f789ba9b63da9f036e8b739477bb8295fd" + - "4a2600000000fdfd0000473044022062a3ca46f976d4719d94fa67c9360c65b0946e3a" + - "7877f7f2c4cc37b8c77dede302203af47e6c6ccd804726295eb99f90dc198c86414d85" + - "7e21bc617f26903d380b0901483045022100af39dc96cf32aed7165a7644dcdfa939cb" + - "7e6af8236126644be937ff8a10b3a10220204b03b1a6b8acc7748e3c76ee968f9e9997" + - "a81c6e944910d2dc626133aa82bf014c695221021de4261f30b149c3f14f93b1a06a5e" + - "96aa24780c789343f80c341043d46d700a21036daab402e66c56470eda26d981f80a1b" + - "8f224f21fdbe64329b0481b1b37d8a0b2103c72347824e9099dbdfad5a2dac56e5d056" + - "8c9c100caa7490f62f2731c23af27353aeffffffff028a7faf00000000001976a91472" + - "2a53477336bd4960bed86be7fc47b8ece07ba688acbda438010000000017a9148a6425" + - "ccc0cb64097c29579a40e733819d4e07c58700000000"), - ("op-return-output", - "5be05925b8ef04f326b19e2c57f9ce1f3e2024aa992dd0d744f7942e92a200a5", - "0100000001067c0bd822cdcfabf0978bdede3a7b395a5c175219fd704b1ee47f137858" + - "6362000000006b483045022100dffb89e46c734b6429b6322bce9b6343c98d0d9eed12" + - "059ac2b9dfcffaa5577702204350bc86aea9a7e3b2868a643d27c5a36bd3105c8a2a2c" + - "5daf6e8441c83c870f012102a1260627f8845765759454a2ee47603312bbd7fdad0fa0" + - "0f3ad0cea0c5602d2cffffffff03e9690700000000001976a91497d34e7a0c8082f180" + - "546040030afb925c0f2dd888ac0000000000000000166a146f6d6e6900000000000000" + - "0300000000000186a0aa0a0000000000001976a91488d924f51033b74a895863a5fb57" + - "fd545529df7d88ac00000000")]) -def test_serialization_roundtrip(tx_type, tx_id, tx_hex): - assert tx_hex == btc.serialize(btc.deserialize(tx_hex)) - -@pytest.mark.parametrize( - "ins, outs, txtype, valid", - [ - (4, 3, "p2pkh", True), - (4, 3, "p2sh", False), - ]) -def test_estimate_tx_size(ins, outs, txtype, valid): - #TODO: this function should throw on invalid number of ins or outs - if valid: - assert btc.estimate_tx_size(ins, outs, txtype)== 10 + 147*ins + 34*outs - else: - with pytest.raises(NotImplementedError) as e_info: - btc.estimate_tx_size(ins, outs, txtype) - - -def test_serialization_roundtrip2(): - #Data extracted from: - #https://github.com/bitcoin/bitcoin/blob/master/src/test/data/tx_valid.json - #These are a variety of rather strange edge case transactions, which are - #still valid. - #Note that of course this is only a serialization, not validity test, so - #only currently of very limited significance - with open(os.path.join(testdir,"tx_valid.json"), "r") as f: - json_data = f.read() - valid_txs = json.loads(json_data) - for j in valid_txs: - #ignore comment entries - if len(j) < 2: - continue - print(j) - deserialized = btc.deserialize(str(j[0])) - print(deserialized) - assert j[0] == btc.serialize(deserialized) diff --git a/jmbitcoin/test/test_tx_signing.py b/jmbitcoin/test/test_tx_signing.py new file mode 100644 index 000000000..4db7f3c6e --- /dev/null +++ b/jmbitcoin/test/test_tx_signing.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 + +import sys +import pytest + +import binascii +import hashlib +import jmbitcoin as btc + +@pytest.mark.parametrize( + "addrtype", + [("p2wpkh"), + ("p2sh-p2wpkh"), + ("p2pkh"), + ]) +def test_sign_standard_txs(addrtype): + # liberally copied from python-bitcoinlib tests, + # in particular see: + # https://github.com/petertodd/python-bitcoinlib/pull/227 + + # Create the (in)famous correct brainwallet secret key. + priv = hashlib.sha256(b'correct horse battery staple').digest() + b"\x01" + pub = btc.privkey_to_pubkey(priv) + + # Create an address from that private key. + # (note that the input utxo is fake so we are really only creating + # a destination here). + scriptPubKey = btc.CScript([btc.OP_0, btc.Hash160(pub)]) + address = btc.P2WPKHBitcoinAddress.from_scriptPubKey(scriptPubKey) + + # Create a dummy outpoint; use same 32 bytes for convenience + txid = priv[:32] + vout = 2 + amount = btc.coins_to_satoshi(float('0.12345')) + + # Calculate an amount for the upcoming new UTXO. Set a high fee to bypass + # bitcoind minfee setting. + amount_less_fee = int(amount - btc.coins_to_satoshi(0.01)) + + # Create a destination to send the coins. + destination_address = address + target_scriptPubKey = scriptPubKey + + # Create the unsigned transaction. + txin = btc.CTxIn(btc.COutPoint(txid[::-1], vout)) + txout = btc.CTxOut(amount_less_fee, target_scriptPubKey) + tx = btc.CMutableTransaction([txin], [txout]) + + # Calculate the signature hash for the transaction. This is then signed by the + # private key that controls the UTXO being spent here at this txin_index. + if addrtype == "p2wpkh": + sig, msg = btc.sign(tx, 0, priv, amount=amount, native=True) + elif addrtype == "p2sh-p2wpkh": + sig, msg = btc.sign(tx, 0, priv, amount=amount, native=False) + elif addrtype == "p2pkh": + sig, msg = btc.sign(tx, 0, priv) + else: + assert False + if not sig: + print(msg) + raise + print("created signature: ", binascii.hexlify(sig)) + print("serialized transaction: {}".format(btc.b2x(tx.serialize()))) + +def test_mk_shuffled_tx(): + # prepare two addresses for the outputs + pub = btc.privkey_to_pubkey(btc.Hash(b"priv") + b"\x01") + scriptPubKey = btc.CScript([btc.OP_0, btc.Hash160(pub)]) + addr1 = btc.P2WPKHBitcoinAddress.from_scriptPubKey(scriptPubKey) + scriptPubKey_p2sh = scriptPubKey.to_p2sh_scriptPubKey() + addr2 = btc.CCoinAddress.from_scriptPubKey(scriptPubKey_p2sh) + + ins = [(btc.Hash(b"blah"), 7), (btc.Hash(b"foo"), 15)] + # note the casts str() ; most calls to mktx will have addresses fed + # as strings, so this is enforced for simplicity. + outs = [{"address": str(addr1), "value": btc.coins_to_satoshi(float("0.1"))}, + {"address": str(addr2), "value": btc.coins_to_satoshi(float("45981.23331234"))}] + tx = btc.make_shuffled_tx(ins, outs, version=2, locktime=500000) + +def test_bip143_tv(): + # p2sh-p2wpkh case: + rawtx_hex = "0100000001db6b1b20aa0fd7b23880be2ecbd4a98130974cf4748fb66092ac4d3ceb1a54770100000000feffffff02b8b4eb0b000000001976a914a457b684d7f0d539a46a45bbc043f35b59d0d96388ac0008af2f000000001976a914fd270b1ee6abcaea97fea7ad0402e8bd8ad6d77c88ac92040000" + inp_spk_hex = "a9144733f37cf4db86fbc2efed2500b4f4e49f31202387" + value = 10 + redeemScript = "001479091972186c449eb1ded22b78e40d009bdf0089" + privkey_hex = "eb696a065ef48a2192da5b28b694f87544b30fae8327c4510137a922f32c6dcf01" + pubkey_hex = "03ad1d8e89212f0b92c74d23bb710c00662ad1470198ac48c43f7d6f93a2a26873" + tx = btc.CMutableTransaction.deserialize(btc.x(rawtx_hex)) + btc.sign(tx, 0, btc.x(privkey_hex), amount=btc.coins_to_satoshi(10), native=False) + expectedsignedtx = "01000000000101db6b1b20aa0fd7b23880be2ecbd4a98130974cf4748fb66092ac4d3ceb1a5477010000001716001479091972186c449eb1ded22b78e40d009bdf0089feffffff02b8b4eb0b000000001976a914a457b684d7f0d539a46a45bbc043f35b59d0d96388ac0008af2f000000001976a914fd270b1ee6abcaea97fea7ad0402e8bd8ad6d77c88ac02473044022047ac8e878352d3ebbde1c94ce3a10d057c24175747116f8288e5d794d12d482f0220217f36a485cae903c713331d877c1f64677e3622ad4010726870540656fe9dcb012103ad1d8e89212f0b92c74d23bb710c00662ad1470198ac48c43f7d6f93a2a2687392040000" + assert btc.b2x(tx.serialize()) == expectedsignedtx \ No newline at end of file diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index f4dd264e5..0b73d682e 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -18,7 +18,8 @@ UTXOManager, WALLET_IMPLEMENTATIONS, compute_tx_locktime) from .storage import (Argon2Hash, Storage, StorageError, RetryableStorageError, StoragePasswordError, VolatileStorage) -from .cryptoengine import BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH, EngineError +from .cryptoengine import (BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH, EngineError, + TYPE_P2PKH, TYPE_P2SH_P2WPKH, TYPE_P2WPKH) from .configure import (load_test_config, load_program_config, get_p2pk_vbyte, jm_single, get_network, update_persist_config, validate_address, is_burn_destination, get_irc_mchannels, diff --git a/jmclient/jmclient/blockchaininterface.py b/jmclient/jmclient/blockchaininterface.py index 777530a72..56ddff0bd 100644 --- a/jmclient/jmclient/blockchaininterface.py +++ b/jmclient/jmclient/blockchaininterface.py @@ -6,7 +6,7 @@ from decimal import Decimal import binascii from twisted.internet import reactor, task - +from jmbase import bintohex, hextobin import jmbitcoin as btc from jmclient.jsonrpc import JsonRpcConnectionError, JsonRpcError @@ -297,9 +297,8 @@ def get_deser_from_gettransaction(self, rpcretval): if not "hex" in rpcretval: log.info("Malformed gettransaction output") return None - #str cast for unicode - hexval = str(rpcretval["hex"]) - return btc.deserialize(hexval) + return btc.CMutableTransaction.deserialize( + hextobin(rpcretval["hex"])) def list_transactions(self, num, skip=0): """ Return a list of the last `num` transactions seen @@ -309,20 +308,22 @@ def list_transactions(self, num, skip=0): return self.rpc("listtransactions", ["*", num, skip, True]) def get_transaction(self, txid): - """ Returns a serialized transaction for txid txid, + """ Argument txid is passed in binary. + Returns a serialized transaction for txid txid, in hex as returned by Bitcoin Core rpc, or None if no transaction can be retrieved. Works also for watch-only wallets. """ + htxid = bintohex(txid) #changed syntax in 0.14.0; allow both syntaxes try: - res = self.rpc("gettransaction", [txid, True]) - except: + res = self.rpc("gettransaction", [htxid, True]) + except Exception as e: try: - res = self.rpc("gettransaction", [txid, 1]) + res = self.rpc("gettransaction", [htxid, 1]) except JsonRpcError as e: #This should never happen (gettransaction is a wallet rpc). - log.warn("Failed gettransaction call; JsonRpcError") + log.warn("Failed gettransaction call; JsonRpcError: " + repr(e)) return None except Exception as e: log.warn("Failed gettransaction call; unexpected error:") @@ -333,7 +334,11 @@ def get_transaction(self, txid): return None return res - def pushtx(self, txhex): + def pushtx(self, txbin): + """ Given a binary serialized valid bitcoin transaction, + broadcasts it to the network. + """ + txhex = bintohex(txbin) try: txid = self.rpc('sendrawtransaction', [txhex]) except JsonRpcConnectionError as e: @@ -345,9 +350,9 @@ def pushtx(self, txhex): return True def query_utxo_set(self, txout, includeconf=False, includeunconf=False): - """If txout is either (a) a single string in hex encoded txid:n form, + """If txout is either (a) a single utxo in (txidbin, n) form, or a list of the same, returns, as a list for each txout item, - the result of gettxout from the bitcoind rpc for those utxs; + the result of gettxout from the bitcoind rpc for those utxos; if any utxo is invalid, None is returned. includeconf: if this is True, the current number of confirmations of the prescribed utxo is included in the returned result dict. @@ -360,16 +365,18 @@ def query_utxo_set(self, txout, includeconf=False, includeunconf=False): txout = [txout] result = [] for txo in txout: - if len(txo) < 66: + txo_hex = bintohex(txo[0]) + if len(txo_hex) != 64: + log.warn("Invalid utxo format, ignoring: {}".format(txo)) result.append(None) continue try: - txo_idx = int(txo[65:]) + txo_idx = int(txo[1]) except ValueError: log.warn("Invalid utxo format, ignoring: {}".format(txo)) result.append(None) continue - ret = self.rpc('gettxout', [txo[:64], txo_idx, includeunconf]) + ret = self.rpc('gettxout', [txo_hex, txo_idx, includeunconf]) if ret is None: result.append(None) else: @@ -380,7 +387,7 @@ def query_utxo_set(self, txout, includeconf=False, includeunconf=False): result_dict = {'value': int(Decimal(str(ret['value'])) * Decimal('1e8')), 'address': address, - 'script': ret['scriptPubKey']['hex']} + 'script': hextobin(ret['scriptPubKey']['hex'])} if includeconf: result_dict['confirms'] = int(ret['confirmations']) result.append(result_dict) diff --git a/jmclient/jmclient/client_protocol.py b/jmclient/jmclient/client_protocol.py index 8783cfc47..9711d9b0d 100644 --- a/jmclient/jmclient/client_protocol.py +++ b/jmclient/jmclient/client_protocol.py @@ -9,16 +9,17 @@ except ImportError: pass from jmbase import commands - +import binascii import json import hashlib import os import sys -from jmbase import get_log +from jmbase import (get_log, EXIT_FAILURE, hextobin, bintohex, + utxo_to_utxostr, dictchanger) from jmclient import (jm_single, get_irc_mchannels, RegtestBitcoinCoreInterface) import jmbitcoin as btc -from jmbase.support import EXIT_FAILURE + jlog = get_log() @@ -30,7 +31,8 @@ def __init__(self, factory, client, nick_priv=None): self.client = client self.factory = factory if not nick_priv: - self.nick_priv = hashlib.sha256(os.urandom(16)).hexdigest() + '01' + self.nick_priv = hashlib.sha256( + os.urandom(16)).digest() + b"\x01" else: self.nick_priv = nick_priv @@ -59,10 +61,18 @@ def connectionMade(self): self.clientStart() def set_nick(self): - self.nick_pubkey = btc.privtopub(self.nick_priv) - self.nick_pkh_raw = btc.bin_sha256(self.nick_pubkey)[ - :self.nick_hashlen] - self.nick_pkh = btc.b58encode(self.nick_pkh_raw) + """ Algorithm: take pubkey and hex-serialized it; + then SHA2(hexpub) but truncate output to nick_hashlen. + Then encode to a base58 string (no check). + Then prepend J and version char (e.g. '5'). + Finally append padding to nick_maxencoded (+2). + """ + self.nick_pubkey = btc.privkey_to_pubkey(self.nick_priv) + # note we use binascii hexlify directly here because input + # to hashing must be encoded. + self.nick_pkh_raw = hashlib.sha256(binascii.hexlify( + self.nick_pubkey)).digest()[:self.nick_hashlen] + self.nick_pkh = btc.base58.encode(self.nick_pkh_raw) #right pad to maximum possible; b58 is not fixed length. #Use 'O' as one of the 4 not included chars in base58. self.nick_pkh += 'O' * (self.nick_maxencoded - len(self.nick_pkh)) @@ -93,7 +103,7 @@ def on_JM_INIT_PROTO(self, nick_hash_length, nick_max_encoded, @commands.JMRequestMsgSig.responder def on_JM_REQUEST_MSGSIG(self, nick, cmd, msg, msg_to_be_signed, hostid): sig = btc.ecdsa_sign(str(msg_to_be_signed), self.nick_priv) - msg_to_return = str(msg) + " " + self.nick_pubkey + " " + sig + msg_to_return = str(msg) + " " + bintohex(self.nick_pubkey) + " " + sig d = self.callRemote(commands.JMMsgSignature, nick=nick, cmd=cmd, @@ -105,20 +115,21 @@ def on_JM_REQUEST_MSGSIG(self, nick, cmd, msg, msg_to_be_signed, hostid): @commands.JMRequestMsgSigVerify.responder def on_JM_REQUEST_MSGSIG_VERIFY(self, msg, fullmsg, sig, pubkey, nick, hashlen, max_encoded, hostid): + pubkey_bin = hextobin(pubkey) verif_result = True - if not btc.ecdsa_verify(str(msg), sig, pubkey): + if not btc.ecdsa_verify(str(msg), sig, pubkey_bin): # workaround for hostid, which sometimes is lowercase-only for some IRC connections - if not btc.ecdsa_verify(str(msg[:-len(hostid)] + hostid.lower()), sig, pubkey): + if not btc.ecdsa_verify(str(msg[:-len(hostid)] + hostid.lower()), sig, pubkey_bin): jlog.debug("nick signature verification failed, ignoring: " + str(nick)) verif_result = False #check that nick matches hash of pubkey - nick_pkh_raw = btc.bin_sha256(pubkey)[:hashlen] + nick_pkh_raw = hashlib.sha256(pubkey.encode("ascii")).digest()[:hashlen] nick_stripped = nick[2:2 + max_encoded] #strip right padding nick_unpadded = ''.join([x for x in nick_stripped if x != 'O']) - if not nick_unpadded == btc.b58encode(nick_pkh_raw): + if not nick_unpadded == btc.base58.encode(nick_pkh_raw): jlog.debug("Nick hash check failed, expected: " + str(nick_unpadded) - + ", got: " + str(btc.b58encode(nick_pkh_raw))) + + ", got: " + str(btc.base58.encode(nick_pkh_raw))) verif_result = False d = self.callRemote(commands.JMMsgSignatureVerify, verif_result=verif_result, @@ -234,10 +245,18 @@ def on_JM_AUTH_RECEIVED(self, nick, offer, commitment, revelation, amount, jlog.info("Maker refuses to continue on receiving auth.") else: utxos, auth_pub, cj_addr, change_addr, btc_sig = retval[1:] + # json does not allow non-string keys: + utxos_strkeyed = {} + for k in utxos: + success, u = utxo_to_utxostr(k) + assert success + utxos_strkeyed[u] = {"value": utxos[k]["value"], + "address": utxos[k]["address"]} + auth_pub_hex = bintohex(auth_pub) d = self.callRemote(commands.JMIOAuth, nick=nick, - utxolist=json.dumps(utxos), - pubkey=auth_pub, + utxolist=json.dumps(utxos_strkeyed), + pubkey=auth_pub_hex, cjaddr=cj_addr, changeaddr=change_addr, pubkeysig=btc_sig) @@ -258,14 +277,14 @@ def on_JM_TX_RECEIVED(self, nick, txhex, offer): else: sigs = retval[1] self.finalized_offers[nick] = offer - tx = btc.deserialize(txhex) + tx = btc.CMutableTransaction.deserialize(hextobin(txhex)) self.finalized_offers[nick]["txd"] = tx - txid = btc.txhash(btc.serialize(tx)) + txid = tx.GetTxid()[::-1] # we index the callback by the out-set of the transaction, # because the txid is not known until all scriptSigs collected # (hence this is required for Makers, but not Takers). # For more info see WalletService.transaction_monitor(): - txinfo = tuple((x["script"], x["value"]) for x in tx["outs"]) + txinfo = tuple((x.scriptPubKey, x.nValue) for x in tx.vout) self.client.wallet_service.register_callbacks([self.unconfirm_callback], txinfo, "unconfirmed") self.client.wallet_service.register_callbacks([self.confirm_callback], @@ -285,8 +304,8 @@ def on_JM_TX_RECEIVED(self, nick, txhex, offer): def tx_match(self, txd): for k,v in iteritems(self.finalized_offers): - #Tx considered defined by its output set - if v["txd"]["outs"] == txd["outs"]: + # Tx considered defined by its output set + if v["txd"].vout == txd.vout: offerinfo = v break else: @@ -302,7 +321,7 @@ def unconfirm_callback(self, txd, txid): txid) self.client.modify_orders(to_cancel, to_announce) - txinfo = tuple((x["script"], x["value"]) for x in txd["outs"]) + txinfo = tuple((x.scriptPubKey, x.nValue) for x in txd.vout) confirm_timeout_sec = float(jm_single().config.get( "TIMEOUT", "confirm_timeout_hours")) * 3600 task.deferLater(reactor, confirm_timeout_sec, diff --git a/jmclient/jmclient/commitment_utils.py b/jmclient/jmclient/commitment_utils.py index 64a80c4a0..ab598adba 100644 --- a/jmclient/jmclient/commitment_utils.py +++ b/jmclient/jmclient/commitment_utils.py @@ -2,8 +2,10 @@ import sys import jmbitcoin as btc from jmbase import jmprint -from jmclient import jm_single, get_p2pk_vbyte, get_p2sh_vbyte -from jmbase.support import EXIT_FAILURE +from jmclient import (jm_single, get_p2pk_vbyte, get_p2sh_vbyte, + BTCEngine, TYPE_P2PKH, TYPE_P2SH_P2WPKH, + BTC_P2PKH, BTC_P2SH_P2WPKH) +from jmbase.support import EXIT_FAILURE, utxostr_to_utxo def quit(parser, errmsg): #pragma: no cover @@ -18,17 +20,16 @@ def get_utxo_info(upriv): u, priv = upriv.split(',') u = u.strip() priv = priv.strip() - txid, n = u.split(':') - assert len(txid)==64 - assert len(n) in range(1, 4) - n = int(n) - assert n in range(256) + success, utxo = utxostr_to_utxo(u) + assert success, utxo except: #not sending data to stdout in case privkey info jmprint("Failed to parse utxo information for utxo", "error") raise try: - hexpriv = btc.from_wif_privkey(priv, vbyte=get_p2pk_vbyte()) + # see note below for why keytype is ignored, and note that + # this calls read_privkey to validate. + raw, _ = BTCEngine.wif_to_privkey(priv) except: jmprint("failed to parse privkey, make sure it's WIF compressed format.", "error") raise @@ -40,16 +41,20 @@ def validate_utxo_data(utxo_datas, retrieve=False, segwit=False): then use the blockchain instance to look up the utxo and check that its address field matches. If retrieve is True, return the set of utxos and their values. + If segwit is true, assumes a p2sh wrapped p2wpkh, i.e. + native segwit is NOT currently supported here. If segwit + is false, p2pkh is assumed. """ results = [] for u, priv in utxo_datas: jmprint('validating this utxo: ' + str(u), "info") - hexpriv = btc.from_wif_privkey(priv, vbyte=get_p2pk_vbyte()) - if segwit: - addr = btc.pubkey_to_p2sh_p2wpkh_address( - btc.privkey_to_pubkey(hexpriv), get_p2sh_vbyte()) - else: - addr = btc.privkey_to_address(hexpriv, magicbyte=get_p2pk_vbyte()) + # as noted in `ImportWalletMixin` code comments, there is not + # yet a functional auto-detection of key type from WIF, so the + # second argument is ignored; we assume p2sh-p2wpkh if segwit, + # else we assume p2pkh. + engine = BTC_P2SH_P2WPKH if segwit else BTC_P2PKH + rawpriv, _ = BTCEngine.wif_to_privkey(priv) + addr = engine.privkey_to_address(rawpriv) jmprint('claimed address: ' + addr, "info") res = jm_single().bc_interface.query_utxo_set([u]) if len(res) != 1 or None in res: diff --git a/jmclient/jmclient/configure.py b/jmclient/jmclient/configure.py index 48372b296..45f5df239 100644 --- a/jmclient/jmclient/configure.py +++ b/jmclient/jmclient/configure.py @@ -377,33 +377,18 @@ def get_p2pk_vbyte(): def validate_address(addr): try: - assert len(addr) > 2 - if addr[:2].lower() in ['bc', 'tb']: - # Regtest special case - if addr[:4] == 'bcrt': - if btc.bech32addr_decode('bcrt', addr)[1]: - return True, 'address validated' - return False, 'Invalid bech32 regtest address' - #Else, enforce testnet/mainnet per config - if get_network() == "testnet": - hrpreq = 'tb' - else: - hrpreq = 'bc' - if btc.bech32addr_decode(hrpreq, addr)[1]: - return True, 'address validated' - return False, 'Invalid bech32 address' - #Not bech32; assume b58 from here - ver = btc.get_version_byte(addr) - except AssertionError: - return False, 'Checksum wrong. Typo in address?' - except Exception: - return False, "Invalid bitcoin address" - if ver != get_p2pk_vbyte() and ver != get_p2sh_vbyte(): - return False, 'Wrong address version. Testnet/mainnet confused?' - if len(btc.b58check_to_bin(addr)) != 20: - return False, "Address has correct checksum but wrong length." - return True, 'address validated' - + # automatically respects the network + # as set in btc.select_chain_params(...) + x = btc.CCoinAddress(addr) + except Exception as e: + return False, repr(e) + # additional check necessary because python-bitcointx + # does not check hash length on p2sh construction. + try: + x.to_scriptPubKey() + except Exception as e: + return False, repr(e) + return True, "address validated" _BURN_DESTINATION = "BURN" @@ -565,6 +550,7 @@ def get_blockchain_interface_instance(_config): source = _config.get("BLOCKCHAIN", "blockchain_source") network = get_network() testnet = network == 'testnet' + if source in ('bitcoin-rpc', 'regtest', 'bitcoin-rpc-no-history'): rpc_host = _config.get("BLOCKCHAIN", "rpc_host") rpc_port = _config.get("BLOCKCHAIN", "rpc_port") @@ -574,10 +560,20 @@ def get_blockchain_interface_instance(_config): rpc_wallet_file) if source == 'bitcoin-rpc': #pragma: no cover bc_interface = BitcoinCoreInterface(rpc, network) + if testnet: + btc.select_chain_params("bitcoin/testnet") + else: + btc.select_chain_params("bitcoin") elif source == 'regtest': bc_interface = RegtestBitcoinCoreInterface(rpc) + btc.select_chain_params("bitcoin/regtest") elif source == "bitcoin-rpc-no-history": bc_interface = BitcoinCoreNoHistoryInterface(rpc, network) + if testnet or network == "regtest": + # TODO will not work for bech32 regtest addresses: + btc.select_chain_params("bitcoin/testnet") + else: + btc.select_chain_params("bitcoin") else: assert 0 elif source == 'electrum': diff --git a/jmclient/jmclient/cryptoengine.py b/jmclient/jmclient/cryptoengine.py index 9b5d9f153..c24d1085c 100644 --- a/jmclient/jmclient/cryptoengine.py +++ b/jmclient/jmclient/cryptoengine.py @@ -19,20 +19,30 @@ WIF_PREFIX_MAP = {'mainnet': b'\x80', 'testnet': b'\xef'} BIP44_COIN_MAP = {'mainnet': 2**31, 'testnet': 2**31 + 1} -def detect_script_type(script): - if script.startswith(btc.P2PKH_PRE) and script.endswith(btc.P2PKH_POST) and\ - len(script) == 0x14 + len(btc.P2PKH_PRE) + len(btc.P2PKH_POST): +def detect_script_type(script_str): + """ Given a scriptPubKey, decide which engine + to use, one of: p2pkh, p2sh-p2wpkh, p2wpkh. + Note that for the p2sh case, we are assuming the nature + of the redeem script (p2wpkh wrapped) because that is what + we support; but we can't know for sure, from the sPK only. + Raises EngineError if the type cannot be detected, so + callers MUST handle this exception to avoid crashes. + """ + script = btc.CScript(script_str) + if not script.is_valid(): + raise EngineError("Unknown script type for script '{}'" + .format(hexlify(script_str))) + if script.is_p2pkh(): return TYPE_P2PKH - elif (script.startswith(btc.P2SH_P2WPKH_PRE) and - script.endswith(btc.P2SH_P2WPKH_POST) and - len(script) == 0x14 + len(btc.P2SH_P2WPKH_PRE) + len( - btc.P2SH_P2WPKH_POST)): + elif script.is_p2sh(): + # see note above. + # note that is_witness_v0_nested_keyhash does not apply, + # since that picks up scriptSigs not scriptPubKeys. return TYPE_P2SH_P2WPKH - elif script.startswith(btc.P2WPKH_PRE) and\ - len(script) == 0x14 + len(btc.P2WPKH_PRE): + elif script.is_witness_v0_keyhash(): return TYPE_P2WPKH raise EngineError("Unknown script type for script '{}'" - .format(hexlify(script))) + .format(hexlify(script_str))) class classproperty(object): """ @@ -97,28 +107,38 @@ def BIP44_COIN_TYPE(cls): @staticmethod def privkey_to_pubkey(privkey): - return btc.privkey_to_pubkey(privkey, False) + return btc.privkey_to_pubkey(privkey) @staticmethod def address_to_script(addr): - return unhexlify(btc.address_to_script(addr)) + return btc.CCoinAddress(addr).to_scriptPubKey() @classmethod def wif_to_privkey(cls, wif): - raw = btc.b58check_to_bin(wif) + raw = btc.b58check_to_bin(wif)[1] + # see note to `privkey_to_wif`; same applies here. + # We only handle valid private keys, not any byte string. + btc.read_privkey(raw) + vbyte = struct.unpack('B', btc.get_version_byte(wif))[0] - if (struct.unpack('B', btc.BTC_P2PK_VBYTE[get_network()])[0] + struct.unpack('B', cls.WIF_PREFIX)[0]) & 0xff == vbyte: + if (struct.unpack('B', btc.BTC_P2PK_VBYTE[get_network()])[0] + \ + struct.unpack('B', cls.WIF_PREFIX)[0]) & 0xff == vbyte: key_type = TYPE_P2PKH - elif (struct.unpack('B', btc.BTC_P2SH_VBYTE[get_network()])[0] + struct.unpack('B', cls.WIF_PREFIX)[0]) & 0xff == vbyte: + elif (struct.unpack('B', btc.BTC_P2SH_VBYTE[get_network()])[0] + \ + struct.unpack('B', cls.WIF_PREFIX)[0]) & 0xff == vbyte: key_type = TYPE_P2SH_P2WPKH else: key_type = None - return raw, key_type @classmethod def privkey_to_wif(cls, priv): + # refuse to WIF-ify something that we don't recognize + # as a private key; ignoring the return value of this + # function as we only want to raise whatever Exception + # it does: + btc.read_privkey(priv) return btc.bin_to_b58check(priv, cls.WIF_PREFIX) @classmethod @@ -166,12 +186,12 @@ def pubkey_to_script(cls, pubkey): @classmethod def privkey_to_address(cls, privkey): script = cls.key_to_script(privkey) - return btc.script_to_address(script, cls.VBYTE) + return str(btc.CCoinAddress.from_scriptPubKey(script)) @classmethod def pubkey_to_address(cls, pubkey): script = cls.pubkey_to_script(pubkey) - return btc.script_to_address(script, cls.VBYTE) + return str(btc.script_to_address(script, cls.VBYTE)) @classmethod def pubkey_has_address(cls, pubkey, addr): @@ -203,7 +223,12 @@ def sign_message(privkey, message): @classmethod def script_to_address(cls, script): - return btc.script_to_address(script, vbyte=cls.VBYTE) + """ a script passed in as binary converted to a + Bitcoin address of the appropriate type. + """ + s = btc.CScript(script) + assert s.is_valid() + return str(btc.CCoinAddress.from_scriptPubKey(s)) class BTC_P2PKH(BTCEngine): @@ -213,6 +238,7 @@ def VBYTE(cls): @classmethod def pubkey_to_script(cls, pubkey): + # this call does not enforce compressed: return btc.pubkey_to_p2pkh_script(pubkey) @classmethod @@ -222,7 +248,7 @@ def pubkey_to_script_code(cls, pubkey): @classmethod def sign_transaction(cls, tx, index, privkey, *args, **kwargs): hashcode = kwargs.get('hashcode') or btc.SIGHASH_ALL - return btc.sign(btc.serialize(tx), index, privkey, + return btc.sign(tx, index, privkey, hashcode=hashcode, amount=None, native=False) @@ -250,8 +276,9 @@ def pubkey_to_script_code(cls, pubkey): def sign_transaction(cls, tx, index, privkey, amount, hashcode=btc.SIGHASH_ALL, **kwargs): assert amount is not None - return btc.sign(btc.serialize(tx), index, privkey, + a, b = btc.sign(tx, index, privkey, hashcode=hashcode, amount=amount, native=False) + return a, b class BTC_P2WPKH(BTCEngine): @@ -286,7 +313,7 @@ def pubkey_to_script_code(cls, pubkey): def sign_transaction(cls, tx, index, privkey, amount, hashcode=btc.SIGHASH_ALL, **kwargs): assert amount is not None - return btc.sign(btc.serialize(tx), index, privkey, + return btc.sign(tx, index, privkey, hashcode=hashcode, amount=amount, native=True) class BTC_Timelocked_P2WSH(BTCEngine): diff --git a/jmclient/jmclient/maker.py b/jmclient/jmclient/maker.py index c19c5a497..8ae48342c 100644 --- a/jmclient/jmclient/maker.py +++ b/jmclient/jmclient/maker.py @@ -5,15 +5,13 @@ import random import sys import abc -from binascii import unhexlify - -from jmbitcoin import SerializationError, SerializationTruncationError import jmbitcoin as btc +from jmbase import (bintohex, hextobin, hexbin, + get_log, EXIT_SUCCESS, EXIT_FAILURE) from jmclient.wallet import estimate_tx_fee, compute_tx_locktime from jmclient.wallet_service import WalletService from jmclient.configure import jm_single -from jmbase.support import get_log, EXIT_SUCCESS, EXIT_FAILURE from jmclient.support import calc_cj_fee, select_one_utxo from jmclient.podle import verify_podle, PoDLE, PoDLEError from twisted.internet import task, reactor @@ -48,10 +46,15 @@ def try_to_create_my_orders(self): sys.exit(EXIT_FAILURE) jlog.info('offerlist={}'.format(self.offerlist)) + @hexbin def on_auth_received(self, nick, offer, commitment, cr, amount, kphex): """Receives data on proposed transaction offer from daemon, verifies commitment, returns necessary data to send ioauth message (utxos etc) """ + # special case due to cjfee passed as string: it can accidentally parse + # as hex: + if not isinstance(offer["cjfee"], str): + offer["cjfee"] = bintohex(offer["cjfee"]) #check the validity of the proof of discrete log equivalence tries = jm_single().config.getint("POLICY", "taker_utxo_retries") def reject(msg): @@ -65,9 +68,8 @@ def reject(msg): reason = repr(e) return reject(reason) - if not verify_podle(str(cr_dict['P']), str(cr_dict['P2']), str(cr_dict['sig']), - str(cr_dict['e']), str(commitment), - index_range=range(tries)): + if not verify_podle(cr_dict['P'], cr_dict['P2'], cr_dict['sig'], + cr_dict['e'], commitment, index_range=range(tries)): reason = "verify_podle failed" return reject(reason) #finally, check that the proffered utxo is real, old enough, large enough, @@ -89,14 +91,14 @@ def reject(msg): try: if not self.wallet_service.pubkey_has_script( - unhexlify(cr_dict['P']), unhexlify(res[0]['script'])): + cr_dict['P'], res[0]['script']): raise EngineError() except EngineError: reason = "Invalid podle pubkey: " + str(cr_dict['P']) return reject(reason) # authorisation of taker passed - #Find utxos for the transaction now: + # Find utxos for the transaction now: utxos, cj_addr, change_addr = self.oid_to_order(offer, amount) if not utxos: #could not find funds @@ -109,21 +111,28 @@ def reject(msg): # Just choose the first utxo in self.utxos and retrieve key from wallet. auth_address = utxos[list(utxos.keys())[0]]['address'] auth_key = self.wallet_service.get_key_from_addr(auth_address) - auth_pub = btc.privtopub(auth_key) - btc_sig = btc.ecdsa_sign(kphex, auth_key) + auth_pub = btc.privkey_to_pubkey(auth_key) + # kphex was auto-converted by @hexbin but we actually need to sign the + # hex version to comply with pre-existing JM protocol: + btc_sig = btc.ecdsa_sign(bintohex(kphex), auth_key) return (True, utxos, auth_pub, cj_addr, change_addr, btc_sig) - def on_tx_received(self, nick, txhex, offerinfo): + @hexbin + def on_tx_received(self, nick, tx_from_taker, offerinfo): """Called when the counterparty has sent an unsigned transaction. Sigs are created and returned if and only if the transaction passes verification checks (see verify_unsigned_tx()). """ + # special case due to cjfee passed as string: it can accidentally parse + # as hex: + if not isinstance(offerinfo["offer"]["cjfee"], str): + offerinfo["offer"]["cjfee"] = bintohex(offerinfo["offer"]["cjfee"]) try: - tx = btc.deserialize(txhex) - except (IndexError, SerializationError, SerializationTruncationError) as e: + tx = btc.CMutableTransaction.deserialize(tx_from_taker) + except Exception as e: return (False, 'malformed txhex. ' + repr(e)) - jlog.info('obtained tx\n' + pprint.pformat(tx)) + jlog.info('obtained tx\n' + bintohex(tx.serialize())) goodtx, errmsg = self.verify_unsigned_tx(tx, offerinfo) if not goodtx: jlog.info('not a good tx, reason=' + errmsg) @@ -133,25 +142,26 @@ def on_tx_received(self, nick, txhex, offerinfo): utxos = offerinfo["utxos"] our_inputs = {} - for index, ins in enumerate(tx['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) + for index, ins in enumerate(tx.vin): + utxo = (ins.prevout.hash[::-1], ins.prevout.n) if utxo not in utxos: continue script = self.wallet_service.addr_to_script(utxos[utxo]['address']) amount = utxos[utxo]['value'] our_inputs[index] = (script, amount) - txs = self.wallet_service.sign_tx(btc.deserialize(unhexlify(txhex)), our_inputs) + success, msg = self.wallet_service.sign_tx(tx, our_inputs) + assert success, msg for index in our_inputs: - sigmsg = unhexlify(txs['ins'][index]['script']) - if 'txinwitness' in txs['ins'][index]: + sigmsg = tx.vin[index].scriptSig + if tx.has_witness(): # Note that this flag only implies that the transaction # *as a whole* is using segwit serialization; it doesn't # imply that this specific input is segwit type (to be # fully general, we allow that even our own wallet's # inputs might be of mixed type). So, we catch the EngineError # which is thrown by non-segwit types. This way the sigmsg - # will only contain the scriptSig field if the wallet object + # will only contain the scriptCode field if the wallet object # decides it's necessary/appropriate for this specific input # If it is segwit, we prepend the witness data since we want # (sig, pub, witnessprogram=scriptSig - note we could, better, @@ -160,16 +170,16 @@ def on_tx_received(self, nick, txhex, offerinfo): # transaction scriptSig), else (non-sw) the !sig message remains # unchanged as (sig, pub). try: - scriptSig = btc.pubkey_to_p2wpkh_script(txs['ins'][index]['txinwitness'][1]) - sigmsg = b''.join(btc.serialize_script_unit( - x) for x in txs['ins'][index]['txinwitness'] + [scriptSig]) - except IndexError: + sig, pub = [a for a in iter(tx.wit.vtxinwit[index].scriptWitness)] + scriptCode = btc.pubkey_to_p2wpkh_script(pub) + sigmsg = btc.CScript([sig]) + btc.CScript(pub) + scriptCode + except Exception as e: #the sigmsg was already set before the segwit check pass sigs.append(base64.b64encode(sigmsg).decode('ascii')) return (True, sigs) - def verify_unsigned_tx(self, txd, offerinfo): + def verify_unsigned_tx(self, tx, offerinfo): """This code is security-critical. Before signing the transaction the Maker must ensure that all details are as expected, and most importantly @@ -177,14 +187,13 @@ def verify_unsigned_tx(self, txd, offerinfo): in total. The data is taken from the offerinfo dict and compared with the serialized txhex. """ - tx_utxo_set = set(ins['outpoint']['hash'] + ':' + str( - ins['outpoint']['index']) for ins in txd['ins']) + tx_utxo_set = set((x.prevout.hash[::-1], x.prevout.n) for x in tx.vin) utxos = offerinfo["utxos"] cjaddr = offerinfo["cjaddr"] - cjaddr_script = btc.address_to_script(cjaddr) + cjaddr_script = btc.CCoinAddress(cjaddr).to_scriptPubKey() changeaddr = offerinfo["changeaddr"] - changeaddr_script = btc.address_to_script(changeaddr) + changeaddr_script = btc.CCoinAddress(changeaddr).to_scriptPubKey() #Note: this value is under the control of the Taker, #see comment below. amount = offerinfo["amount"] @@ -216,14 +225,14 @@ def verify_unsigned_tx(self, txd, offerinfo): #exactly once with the required amts, in the output. times_seen_cj_addr = 0 times_seen_change_addr = 0 - for outs in txd['outs']: - if outs['script'] == cjaddr_script: + for outs in tx.vout: + if outs.scriptPubKey == cjaddr_script: times_seen_cj_addr += 1 - if outs['value'] != amount: + if outs.nValue != amount: return (False, 'Wrong cj_amount. I expect ' + str(amount)) - if outs['script'] == changeaddr_script: + if outs.scriptPubKey == changeaddr_script: times_seen_change_addr += 1 - if outs['value'] != expected_change_value: + if outs.nValue != expected_change_value: return (False, 'wrong change, i expect ' + str( expected_change_value)) if times_seen_cj_addr != 1 or times_seen_change_addr != 1: @@ -392,7 +401,8 @@ def on_tx_confirmed(self, txd, txid, confirmations): # will not be reached except in testing self.on_tx_unconfirmed(txd, txid) - def on_tx_received(self, nick, txhex): + @hexbin + def on_tx_received(self, nick, txser): """ Called when the sender-counterparty has sent a transaction proposal. 1. First we check for the expected destination and amount (this is sufficient to identify our cp, as this info was presumably passed @@ -418,29 +428,30 @@ def on_tx_received(self, nick, txhex): we broadcast the non-coinjoin fallback tx instead. """ try: - tx = btc.deserialize(txhex) - except (IndexError, SerializationError, SerializationTruncationError) as e: + tx = btc.CMutableTransaction.deserialize(txser) + except Exception as e: return (False, 'malformed txhex. ' + repr(e)) self.user_info('obtained proposed fallback (non-coinjoin) ' +\ - 'transaction from sender:\n' + pprint.pformat(tx)) + 'transaction from sender:\n' + str(tx)) - if len(tx["outs"]) != 2: + if len(tx.vout) != 2: return (False, "Transaction has more than 2 outputs; not supported.") dest_found = False destination_index = -1 change_index = -1 proposed_change_value = 0 - for index, out in enumerate(tx["outs"]): - if out["script"] == btc.address_to_script(self.destination_addr): + for index, out in enumerate(tx.vout): + if out.scriptPubKey == btc.CCoinAddress( + self.destination_addr).to_scriptPubKey(): # we found the expected destination; is the amount correct? - if not out["value"] == self.receiving_amount: + if not out.nValue == self.receiving_amount: return (False, "Wrong payout value in proposal from sender.") dest_found = True destination_index = index else: change_found = True - proposed_change_out = out["script"] - proposed_change_value = out["value"] + proposed_change_out = out.scriptPubKey + proposed_change_value = out.nValue change_index = index if not dest_found: @@ -450,9 +461,8 @@ def on_tx_received(self, nick, txhex): # batch retrieval of utxo data utxo = {} ctr = 0 - for index, ins in enumerate(tx['ins']): - utxo_for_checking = ins['outpoint']['hash'] + ':' + str(ins[ - 'outpoint']['index']) + for index, ins in enumerate(tx.vin): + utxo_for_checking = (ins.prevout.hash[::-1], ins.prevout.n) utxo[ctr] = [index, utxo_for_checking] ctr += 1 @@ -471,7 +481,7 @@ def on_tx_received(self, nick, txhex): btc_fee = total_sender_input - self.receiving_amount - proposed_change_value self.user_info("Network transaction fee of fallback tx is: " + str( btc_fee) + " satoshis.") - fee_est = estimate_tx_fee(len(tx['ins']), len(tx['outs']), + fee_est = estimate_tx_fee(len(tx.vin), len(tx.vout), txtype=self.wallet_service.get_txtype()) fee_ok = False if btc_fee > 0.3 * fee_est and btc_fee < 3 * fee_est: @@ -488,51 +498,28 @@ def on_tx_received(self, nick, txhex): # It has the advantage of (a) being simpler and (b) allowing for any # non standard coins. # - #res = jm_single().bc_interface.rpc('testmempoolaccept', [txhex]) + #res = jm_single().bc_interface.rpc('testmempoolaccept', [txser]) #print("Got this result from rpc call: ", res) #if not res["accepted"]: # return (False, "Proposed transaction was rejected from mempool.") - # Manual verification of the transaction signatures. Passing this - # test does imply that the transaction is valid (unless there is - # a double spend during the process), but is restricted to standard - # types: p2pkh, p2wpkh, p2sh-p2wpkh only. Double spend is not counted - # as a risk as this is a payment. + # Manual verification of the transaction signatures. + # TODO handle native segwit properly for i, u in iteritems(utxo): - if "txinwitness" in tx["ins"][u[0]]: - ver_amt = utxo_data[i]["value"] - try: - ver_sig, ver_pub = tx["ins"][u[0]]["txinwitness"] - except Exception as e: - self.user_info("Segwit error: " + repr(e)) - return (False, "Segwit input not of expected type, " - "either p2sh-p2wpkh or p2wpkh") - # note that the scriptCode is the same whether nested or not - # also note that the scriptCode has to be inferred if we are - # only given a transaction serialization. - scriptCode = "76a914" + btc.hash160(unhexlify(ver_pub)) + "88ac" - else: - scriptCode = None - ver_amt = None - scriptSig = btc.deserialize_script(tx["ins"][u[0]]["script"]) - if len(scriptSig) != 2: - return (False, - "Proposed transaction contains unsupported input type") - ver_sig, ver_pub = scriptSig - if not btc.verify_tx_input(txhex, u[0], - utxo_data[i]['script'], - ver_sig, ver_pub, - scriptCode=scriptCode, - amount=ver_amt): + if not btc.verify_tx_input(tx, i, + tx.vin[i].scriptSig, + btc.CScript(utxo_data[i]["script"]), + amount=utxo_data[i]["value"], + witness=tx.wit.vtxinwit[i].scriptWitness): return (False, "Proposed transaction is not correctly signed.") # At this point we are satisfied with the proposal. Record the fallback # in case the sender disappears and the payjoin tx doesn't happen: self.user_info("We'll use this serialized transaction to broadcast if your" " counterparty fails to broadcast the payjoin version:") - self.user_info(txhex) + self.user_info(bintohex(txser)) # Keep a local copy for broadcast fallback: - self.fallback_tx = txhex + self.fallback_tx = tx # Now we add our own inputs: # See the gist comment here: @@ -558,7 +545,7 @@ def on_tx_received(self, nick, txhex): self.user_info("Choosing one coin at random") try: my_utxos = self.wallet_service.select_utxos( - self.mixdepth, jm_single().DUST_THRESHOLD, + self.mixdepth, jm_single().DUST_THRESHOLD, select_fn=select_one_utxo) except: return self.no_coins_fallback() @@ -566,7 +553,7 @@ def on_tx_received(self, nick, txhex): else: # get an approximate required amount assuming 4 inputs, which is # fairly conservative (but guess by necessity). - fee_for_select = estimate_tx_fee(len(tx['ins']) + 4, 2, + fee_for_select = estimate_tx_fee(len(tx.vin) + 4, 2, txtype=self.wallet_service.get_txtype()) approx_sum = max_sender_amt - self.receiving_amount + fee_for_select try: @@ -592,7 +579,7 @@ def on_tx_received(self, nick, txhex): # adjust the output amount at the destination based on our contribution new_destination_amount = self.receiving_amount + my_total_in # estimate the required fee for the new version of the transaction - total_ins = len(tx["ins"]) + len(my_utxos.keys()) + total_ins = len(tx.vin) + len(my_utxos.keys()) est_fee = estimate_tx_fee(total_ins, 2, txtype=self.wallet_service.get_txtype()) self.user_info("We estimated a fee of: " + str(est_fee)) new_change_amount = total_sender_input + my_total_in - \ @@ -604,25 +591,27 @@ def on_tx_received(self, nick, txhex): new_outs = [{"address": self.destination_addr, "value": new_destination_amount}] if new_change_amount >= jm_single().BITCOIN_DUST_THRESHOLD: - new_outs.append({"script": proposed_change_out, - "value": new_change_amount}) + new_outs.append({"address": str(btc.CCoinAddress.from_scriptPubKey( + proposed_change_out)), "value": new_change_amount}) new_ins = [x[1] for x in utxo.values()] new_ins.extend(my_utxos.keys()) - new_tx = btc.make_shuffled_tx(new_ins, new_outs, False, 2, compute_tx_locktime()) - new_tx_deser = btc.deserialize(new_tx) + new_tx = btc.make_shuffled_tx(new_ins, new_outs, 2, compute_tx_locktime()) + # sign our inputs before transfer our_inputs = {} - for index, ins in enumerate(new_tx_deser['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) + for index, ins in enumerate(new_tx.vin): + utxo = (ins.prevout.hash[::-1], ins.prevout.n) if utxo not in my_utxos: continue - script = self.wallet_service.addr_to_script(my_utxos[utxo]['address']) - amount = my_utxos[utxo]['value'] + script = my_utxos[utxo]["script"] + amount = my_utxos[utxo]["value"] our_inputs[index] = (script, amount) - txs = self.wallet_service.sign_tx(btc.deserialize(new_tx), our_inputs) - txinfo = tuple((x["script"], x["value"]) for x in txs["outs"]) + success, msg = self.wallet_service.sign_tx(new_tx, our_inputs) + if not success: + return (False, "Failed to sign new transaction, error: " + msg) + txinfo = tuple((x.scriptPubKey, x.nValue) for x in new_tx.vout) self.wallet_service.register_callbacks([self.on_tx_unconfirmed], txinfo, "unconfirmed") self.wallet_service.register_callbacks([self.on_tx_confirmed], txinfo, "confirmed") # The blockchain interface just abandons monitoring if the transaction @@ -630,7 +619,7 @@ def on_tx_received(self, nick, txhex): # action in this case, so we add an additional callback to the reactor: reactor.callLater(jm_single().config.getint("TIMEOUT", "unconfirm_timeout_sec"), self.broadcast_fallback) - return (True, nick, btc.serialize(txs)) + return (True, nick, bintohex(new_tx.serialize())) def no_coins_fallback(self): """ Broadcast, optionally, the fallback non-coinjoin transaction @@ -646,8 +635,8 @@ def no_coins_fallback(self): def broadcast_fallback(self): self.user_info("Broadcasting non-coinjoin fallback transaction.") - txid = btc.txhash(self.fallback_tx) - success = jm_single().bc_interface.pushtx(self.fallback_tx) + txid = self.fallback_tx.GetTxid()[::-1] + success = jm_single().bc_interface.pushtx(self.fallback_tx.serialize()) if not success: self.user_info("ERROR: the fallback transaction did not broadcast. " "The payment has NOT been made.") diff --git a/jmclient/jmclient/output.py b/jmclient/jmclient/output.py index 7242aec83..a6ffc4f75 100644 --- a/jmclient/jmclient/output.py +++ b/jmclient/jmclient/output.py @@ -61,7 +61,7 @@ def generate_podle_error_string(priv_utxo_pairs, to, ts, wallet_service, cjamoun "with 'python add-utxo.py --help'\n\n") errmsg += ("***\nFor reference, here are the utxos in your wallet:\n") - for md, utxos in wallet_service.get_utxos_by_mixdepth(hexfmt=False).items(): + for md, utxos in wallet_service.get_utxos_by_mixdepth().items(): if not utxos: continue errmsg += ("\nmixdepth {}:\n{}".format( diff --git a/jmclient/jmclient/podle.py b/jmclient/jmclient/podle.py index eff5dcc0f..d7dd26f76 100644 --- a/jmclient/jmclient/podle.py +++ b/jmclient/jmclient/podle.py @@ -7,11 +7,12 @@ import json import binascii import struct +from pprint import pformat from jmbase import jmprint from jmbitcoin import multiply, add_pubkeys, getG, podle_PublicKey,\ - podle_PrivateKey, encode, decode, N, podle_PublicKey_class -from jmbase.support import EXIT_FAILURE - + podle_PrivateKey, N, podle_PublicKey_class +from jmbase import (EXIT_FAILURE, utxostr_to_utxo, + utxo_to_utxostr, hextobin, bintohex) PODLE_COMMIT_FILE = None @@ -45,33 +46,31 @@ def __init__(self, used=False): #This class allows storing of utxo in format "txid:n" only for #convenience of storage/access; it doesn't check or use the data. - #Arguments must be provided in hex. + #Arguments must be provided in binary not hex. self.u = u if not priv: if P: - #Construct a pubkey from raw hex - self.P = podle_PublicKey(binascii.unhexlify(P)) + self.P = podle_PublicKey(P) else: self.P = None else: if P: raise PoDLEError("Pubkey should not be provided with privkey") #any other formatting abnormality will just throw in PrivateKey - if len(priv) == 66 and priv[-2:] == '01': - priv = priv[:-2] - self.priv = podle_PrivateKey(binascii.unhexlify(priv)) + if len(priv) == 33 and priv[-1:] == b"\x01": + priv = priv[:-1] + self.priv = podle_PrivateKey(priv) self.P = self.priv.public_key if P2: - self.P2 = podle_PublicKey(binascii.unhexlify(P2)) + self.P2 = podle_PublicKey(P2) else: self.P2 = None - #These sig values should be passed in hex. self.s = None self.e = None if s: - self.s = binascii.unhexlify(s) + self.s = s if e: - self.e = binascii.unhexlify(e) + self.e = e #Optionally maintain usage state (boolean) self.used = used #the H(P2) value @@ -79,14 +78,13 @@ def __init__(self, def get_commitment(self): """Set the commitment to sha256(serialization of public key P2) - Return in hex to calling function """ if not self.P2: raise PoDLEError("Cannot construct commitment, no P2 available") if not isinstance(self.P2, podle_PublicKey_class): raise PoDLEError("Cannot construct commitment, P2 is not a pubkey") self.commitment = hashlib.sha256(self.P2.format()).digest() - return binascii.hexlify(self.commitment).decode('ascii') + return self.commitment def generate_podle(self, index=0, k=None): """Given a raw private key, in hex format, @@ -117,79 +115,85 @@ def generate_podle(self, index=0, k=None): and the associated signature data that can be used to open the commitment. """ + self.i = index #TODO nonce could be rfc6979? if not k: k = os.urandom(32) - J = getNUMS(index) + J = getNUMS(self.i) KG = podle_PrivateKey(k).public_key - KJ = multiply(k, J.format(), False, return_serialized=False) + KJ = multiply(k, J.format(), return_serialized=False) self.P2 = getP2(self.priv, J) self.get_commitment() self.e = hashlib.sha256(b''.join([x.format( ) for x in [KG, KJ, self.P, self.P2]])).digest() - k_int = decode(k, 256) - priv_int = decode(self.priv.secret, 256) - e_int = decode(self.e, 256) + k_int, priv_int, e_int = (int.from_bytes(x, + byteorder="big") for x in [k, self.priv.secret, self.e]) sig_int = (k_int + priv_int * e_int) % N - self.s = encode(sig_int, 256, minlen=32) + self.s = (sig_int).to_bytes(32, byteorder="big") return self.reveal() def reveal(self): """Encapsulate all the data representing the proof - in a dict for client functions. Data output in hex. + in a dict for client functions. """ if not all([self.u, self.P, self.P2, self.s, self.e]): raise PoDLEError("Cannot generate proof, data is missing") if not self.commitment: self.get_commitment() - Phex, P2hex, shex, ehex, commit = [ - binascii.hexlify(x).decode('ascii') - for x in [self.P.format(), self.P2.format(), self.s, self.e, - self.commitment] - ] - return {'used': str(self.used), + return {'used': self.used, 'utxo': self.u, - 'P': Phex, - 'P2': P2hex, - 'commit': commit, - 'sig': shex, - 'e': ehex} + 'P': self.P.format(), + 'P2': self.P2.format(), + 'commit': self.commitment, + 'sig': self.s, + 'e': self.e} def serialize_revelation(self, separator='|'): + """ Outputs the over-the-wire format as used in + Joinmarket communication protocol. + """ state_dict = self.reveal() - ser_list = [] - for k in ['utxo', 'P', 'P2', 'sig', 'e']: - ser_list += [state_dict[k]] + success, utxo = utxo_to_utxostr(state_dict["utxo"]) + assert success, "invalid utxo in PoDLE" + ser_list = [utxo] + ser_list += [bintohex(state_dict[x]) for x in ["P", "P2", "sig", "e"]] ser_string = separator.join(ser_list) return ser_string @classmethod def deserialize_revelation(cls, ser_rev, separator='|'): + """ Reads the over-the-wire format as used in + Joinmarket communication protocol. + """ ser_list = ser_rev.split(separator) if len(ser_list) != 5: raise PoDLEError("Failed to deserialize, wrong format") - utxo, P, P2, s, e = ser_list - return {'utxo': utxo, 'P': P, 'P2': P2, 'sig': s, 'e': e} + utxostr, P, P2, s, e = ser_list + success, utxo = utxostr_to_utxo(utxostr) + assert success, "invalid utxo format in PoDLE." + return {'utxo': utxo, 'P': hextobin(P), + 'P2': hextobin(P2), 'sig': hextobin(s), 'e': hextobin(e)} def verify(self, commitment, index_range): """For an object created without a private key, check that the opened commitment verifies for at least - one NUMS point as defined by the range in index_range + one NUMS point as defined by the range in index_range. """ if not all([self.P, self.P2, self.s, self.e]): raise PoDLEError("Verify called without sufficient data") if not self.get_commitment() == commitment: return False + for J in [getNUMS(i) for i in index_range]: sig_priv = podle_PrivateKey(self.s) sG = sig_priv.public_key - sJ = multiply(self.s, J.format(), False) - e_int = decode(self.e, 256) - minus_e = encode(-e_int % N, 256, minlen=32) - minus_e_P = multiply(minus_e, self.P.format(), False) - minus_e_P2 = multiply(minus_e, self.P2.format(), False) - KGser = add_pubkeys([sG.format(), minus_e_P], False) - KJser = add_pubkeys([sJ, minus_e_P2], False) + sJ = multiply(self.s, J.format()) + e_int = int.from_bytes(self.e, byteorder="big") + minus_e = (-e_int % N).to_bytes(32, byteorder="big") + minus_e_P = multiply(minus_e, self.P.format()) + minus_e_P2 = multiply(minus_e, self.P2.format()) + KGser = add_pubkeys([sG.format(), minus_e_P]) + KJser = add_pubkeys([sJ, minus_e_P2]) #check 2: e =?= H(K_G || K_J || P || P2) e_check = hashlib.sha256(KGser + KJser + self.P.format() + self.P2.format()).digest() @@ -198,6 +202,20 @@ def verify(self, commitment, index_range): #commitment fails for any NUMS in the provided range return False + def __repr__(self): + """ Specified here to allow logging. + """ + # note: will throw if not fully initalised + r = self.reveal() + success, utxo = utxo_to_utxostr(r["utxo"]) + assert success, "invalid utxo in PoDLE." + return pformat({'used': r["used"], + 'utxo': utxo, + 'P': bintohex(r["P"]), + 'P2': bintohex(r["P2"]), + 'commit': bintohex(r["commit"]), + 'sig': bintohex(r["sig"]), + 'e': bintohex(r["e"])}) def getNUMS(index=0): """Taking secp256k1's G as a seed, @@ -244,7 +262,7 @@ def verify_all_NUMS(write=False): """ nums_points = {} for i in range(256): - nums_points[i] = binascii.hexlify(getNUMS(i).format()).decode('ascii') + nums_points[i] = bintohex(getNUMS(i).format()) if write: with open("nums_basepoints.txt", "wb") as f: from pprint import pformat @@ -263,13 +281,82 @@ def getP2(priv, nums_pt): priv_raw = priv.secret return multiply(priv_raw, nums_pt.format(), - False, return_serialized=False) +# functions which interact with the external persistence of podle data: + +def switch_external_dict_format(ed, utxo_converter, hexbinconverter): + """External dict has structure: + {txid:N:{'P':pubkey, 'reveal':{1:{'P2':P2,'s':s,'e':e}, 2:{..},..}}} + This function switches between readable/writable in file (strings, hex) + and that used internally. + """ + retval = {} + for u in ed: + success, u2 = utxo_converter(u) + assert success, "invalid utxo format in external dict parsing." + retval[u2] = {"P": hexbinconverter(ed[u]["P"])} + retval[u2]["reveal"] = {} + for i in ed[u]["reveal"]: + # hack: python json does not allow int dict keys, they must + # be str, so when reading from file, we convert back to int: + if not isinstance(i, int): + j = int(i) + else: + j = i + retval[u2]["reveal"][j] = { + "P2": hexbinconverter(ed[u]["reveal"][i]["P2"]), + "s": hexbinconverter(ed[u]["reveal"][i]["s"]), + "e": hexbinconverter(ed[u]["reveal"][i]["e"])} + return retval + +def external_dict_to_file(ed): + """ Converts internal format of dict to one writable/readable + in file. + """ + return switch_external_dict_format(ed, utxo_to_utxostr, bintohex) + +def external_dict_from_file(ed): + """ Takes the external dict extracted through json deserialization + from a file and converts it to internal format: + {txid:N:{'P':pubkey, 'reveal':{1:{'P2':P2,'s':s,'e':e}, 2:{..},..}}} + """ + return switch_external_dict_format(ed, utxostr_to_utxo, hextobin) + +def write_to_podle_file(used, external): + """ Update persisted commitment data in PODLE_COMMIT_FILE. + """ + to_write = {} + to_write['used'] = [bintohex(x) for x in used] + externalfmt = external_dict_to_file(external) + to_write['external'] = externalfmt + with open(PODLE_COMMIT_FILE, "wb") as f: + f.write(json.dumps(to_write, indent=4).encode('utf-8')) + +def read_from_podle_file(): + """ Returns used commitment list and external commitments dict + struct currently stored in PODLE_COMMIT_FILE. + """ + if os.path.isfile(PODLE_COMMIT_FILE): + with open(PODLE_COMMIT_FILE, "rb") as f: + try: + c = json.loads(f.read().decode('utf-8')) + except ValueError: #pragma: no cover + #Exit conditions cannot be included in tests. + jmprint("the file: " + PODLE_COMMIT_FILE + " is not valid json.", + "error") + sys.exit(EXIT_FAILURE) + if 'used' not in c.keys() or 'external' not in c.keys(): + raise PoDLEError("Incorrectly formatted file: " + PODLE_COMMIT_FILE) + + used = [hextobin(x) for x in c["used"]] + external = external_dict_from_file(c["external"]) + return (used, external) + return ([], {}) def get_podle_commitments(): """Returns set of commitments used as a list: - [H(P2),..] (hex) and a dict of all existing external commitments. + [H(P2),..] and a dict of all existing external commitments. It is presumed that each H(P2) can be used only once (this may not literally be true, but represents good joinmarket "citizenship"). @@ -280,18 +367,11 @@ def get_podle_commitments(): """ if not os.path.isfile(PODLE_COMMIT_FILE): return ([], {}) - with open(PODLE_COMMIT_FILE, "rb") as f: - c = json.loads(f.read().decode('utf-8')) - if 'used' not in c.keys() or 'external' not in c.keys(): - raise PoDLEError("Incorrectly formatted file: " + PODLE_COMMIT_FILE) - return (c['used'], c['external']) - + return read_from_podle_file() def add_external_commitments(ecs): """To allow external functions to add - PoDLE commitments that were calculated elsewhere; - the format of each entry in ecs must be: - {txid:N:{'P':pubkey, 'reveal':{1:{'P2':P2,'s':s,'e':e}, 2:{..},..}}} + PoDLE commitments that were calculated elsewhere. """ update_commitments(external_to_add=ecs) @@ -304,25 +384,7 @@ def update_commitments(commitment=None, whose key value is the utxo in external_to_remove, persist updated entries to disk. """ - c = {} - if os.path.isfile(PODLE_COMMIT_FILE): - with open(PODLE_COMMIT_FILE, "rb") as f: - try: - c = json.loads(f.read().decode('utf-8')) - except ValueError: #pragma: no cover - #Exit conditions cannot be included in tests. - jmprint("the file: " + PODLE_COMMIT_FILE + " is not valid json.", - "error") - sys.exit(EXIT_FAILURE) - - if 'used' in c: - commitments = c['used'] - else: - commitments = [] - if 'external' in c: - external = c['external'] - else: - external = {} + commitments, external = read_from_podle_file() if commitment: commitments.append(commitment) #remove repeats @@ -334,11 +396,7 @@ def update_commitments(commitment=None, } if external_to_add: external.update(external_to_add) - to_write = {} - to_write['used'] = commitments - to_write['external'] = external - with open(PODLE_COMMIT_FILE, "wb") as f: - f.write(json.dumps(to_write, indent=4).encode('utf-8')) + write_to_podle_file(commitments, external) def get_podle_tries(utxo, priv=None, max_tries=1, external=False): used_commitments, external_commitments = get_podle_commitments() @@ -349,9 +407,8 @@ def get_podle_tries(utxo, priv=None, max_tries=1, external=False): #use as many as were provided in the file, up to a max of max_tries m = min([len(ec['reveal'].keys()), max_tries]) for i in reversed(range(m)): - key = str(i) - p = PoDLE(u=utxo,P=ec['P'],P2=ec['reveal'][key]['P2'], - s=ec['reveal'][key]['s'], e=ec['reveal'][key]['e']) + p = PoDLE(u=utxo, P=ec["P"], P2=ec["reveal"][i]["P2"], + s=ec["reveal"][i]["s"], e=ec["reveal"][i]["e"]) if p.get_commitment() in used_commitments: return i+1 else: @@ -387,10 +444,10 @@ def generate_podle(priv_utxo_pairs, max_tries=1, allow_external=None, k=None): #which is still available. index = tries p = PoDLE(u=utxo, priv=priv) - c = p.generate_podle(index) + p.generate_podle(index) #persist for future checks - update_commitments(commitment=c['commit']) - return c + update_commitments(commitment=p.commitment) + return p if allow_external: for u in allow_external: tries = get_podle_tries(utxo=u, max_tries=max_tries, external=True) @@ -400,19 +457,17 @@ def generate_podle(priv_utxo_pairs, max_tries=1, allow_external=None, k=None): #remove this entry update_commitments(external_to_remove=u) continue - index = str(tries) ec = external_commitments[u] - p = PoDLE(u=u,P=ec['P'],P2=ec['reveal'][index]['P2'], - s=ec['reveal'][index]['s'], e=ec['reveal'][index]['e']) + ecri = ec["reveal"][tries] + p = PoDLE(u=u, P=ec["P"], P2=ecri["P2"], s=ecri["s"], e=ecri["e"]) update_commitments(commitment=p.get_commitment()) - return p.reveal() + return p #Failed to find any non-used valid commitment: return None def verify_podle(Pser, P2ser, sig, e, commitment, index_range=range(10)): verifying_podle = PoDLE(P=Pser, P2=P2ser, s=sig, e=e) - #check 1: Hash(P2ser) =?= commitment if not verifying_podle.verify(commitment, index_range): return False return True diff --git a/jmclient/jmclient/taker.py b/jmclient/jmclient/taker.py index 22d1e4481..9ee185ac4 100644 --- a/jmclient/jmclient/taker.py +++ b/jmclient/jmclient/taker.py @@ -5,12 +5,10 @@ import pprint import random from twisted.internet import reactor, task -from binascii import hexlify, unhexlify -from jmbitcoin import SerializationError, SerializationTruncationError import jmbitcoin as btc from jmclient.configure import jm_single, validate_address -from jmbase.support import get_log +from jmbase import get_log, hextobin, bintohex, hexbin from jmclient.support import (calc_cj_fee, weighted_order_choose, choose_orders, choose_sweep_orders) from jmclient.wallet import estimate_tx_fee, compute_tx_locktime @@ -238,7 +236,6 @@ def initialize(self, orderbook): #Initialization has been successful. We must set the nonrespondants #now to keep track of what changed when we receive the utxo data self.nonrespondants = list(self.orderbook.keys()) - return (True, self.cjamount, commitment, revelation, self.orderbook) def filter_orderbook(self, orderbook, sweep=False): @@ -344,6 +341,7 @@ def prepare_my_bitcoin_data(self): self.utxos = {None: list(self.input_utxos.keys())} return True + @hexbin def receive_utxos(self, ioauth_data): """Triggered when the daemon returns utxo data from makers who responded; this is the completion of phase 1 @@ -354,6 +352,7 @@ def receive_utxos(self, ioauth_data): #Temporary list used to aggregate all ioauth data that must be removed rejected_counterparties = [] + #Need to authorize against the btc pubkey first. for nick, nickdata in iteritems(ioauth_data): utxo_list, auth_pub, cj_addr, change_addr, btc_sig, maker_pk = nickdata @@ -376,10 +375,9 @@ def receive_utxos(self, ioauth_data): self.maker_utxo_data = {} for nick, nickdata in iteritems(ioauth_data): - utxo_list, auth_pub, cj_addr, change_addr, btc_sig, maker_pk = nickdata + utxo_list, auth_pub, cj_addr, change_addr, _, _ = nickdata + utxo_data = jm_single().bc_interface.query_utxo_set(utxo_list) self.utxos[nick] = utxo_list - utxo_data = jm_single().bc_interface.query_utxo_set(self.utxos[ - nick]) if None in utxo_data: jlog.warn(('ERROR outputs unconfirmed or already spent. ' 'utxo_data={}').format(pprint.pformat(utxo_data))) @@ -391,13 +389,12 @@ def receive_utxos(self, ioauth_data): #Extract the address fields from the utxos #Construct the Bitcoin address for the auth_pub field #Ensure that at least one address from utxos corresponds. - auth_pub_bin = unhexlify(auth_pub) for inp in utxo_data: try: if self.wallet_service.pubkey_has_script( - auth_pub_bin, unhexlify(inp['script'])): + auth_pub, inp['script']): break - except EngineError: + except EngineError as e: pass else: jlog.warn("ERROR maker's (" + nick + ")" @@ -500,81 +497,85 @@ def receive_utxos(self, ioauth_data): else: self.outputs.append({'address': self.my_change_addr, 'value': my_change_value}) - self.utxo_tx = [dict([('output', u)]) - for u in sum(self.utxos.values(), [])] + self.utxo_tx = [u for u in sum(self.utxos.values(), [])] self.outputs.append({'address': self.coinjoin_address(), 'value': self.cjamount}) - tx = btc.make_shuffled_tx(self.utxo_tx, self.outputs, False) - jlog.info('obtained tx\n' + pprint.pformat(btc.deserialize(tx))) + self.latest_tx = btc.make_shuffled_tx(self.utxo_tx, self.outputs) + jlog.info('obtained tx\n' + bintohex(self.latest_tx.serialize())) - self.latest_tx = btc.deserialize(tx) - for index, ins in enumerate(self.latest_tx['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) + for index, ins in enumerate(self.latest_tx.vin): + utxo = (ins.prevout.hash[::-1], ins.prevout.n) if utxo not in self.input_utxos.keys(): continue # placeholders required - ins['script'] = 'deadbeef' + ins.scriptSig = btc.CScript.fromhex("deadbeef") self.taker_info_callback("INFO", "Built tx, sending to counterparties.") - return (True, list(self.maker_utxo_data.keys()), tx) + return (True, list(self.maker_utxo_data.keys()), + bintohex(self.latest_tx.serialize())) + @hexbin def auth_counterparty(self, btc_sig, auth_pub, maker_pk): """Validate the counterpartys claim to own the btc address/pubkey that will be used for coinjoining with an ecdsa verification. """ try: - if not btc.ecdsa_verify(maker_pk, btc_sig, auth_pub): + # maker pubkey as message is in hex format: + if not btc.ecdsa_verify(bintohex(maker_pk), btc_sig, auth_pub): jlog.debug('signature didnt match pubkey and message') return False except Exception as e: - jlog.info("Failed ecdsa verify for maker pubkey: " + str(maker_pk)) + jlog.info("Failed ecdsa verify for maker pubkey: " + bintohex(maker_pk)) jlog.info("Exception was: " + repr(e)) return False return True def on_sig(self, nick, sigb64): """Processes transaction signatures from counterparties. - Returns True if all signatures received correctly, else - returns False + If all signatures received correctly, returns the result + of self.self_sign_and_push() (i.e. we complete the signing + and broadcast); else returns False (thus returns False for + all but last signature). """ if self.aborted: return False if nick not in self.nonrespondants: jlog.debug(('add_signature => nick={} ' 'not in nonrespondants {}').format(nick, self.nonrespondants)) - return - sig = hexlify(base64.b64decode(sigb64)).decode('ascii') + return False + sig = base64.b64decode(sigb64) inserted_sig = False - txhex = btc.serialize(self.latest_tx) # batch retrieval of utxo data utxo = {} ctr = 0 - for index, ins in enumerate(self.latest_tx['ins']): - utxo_for_checking = ins['outpoint']['hash'] + ':' + str(ins[ - 'outpoint']['index']) - #'deadbeef' markers mean our own input scripts are not '' - if (ins['script'] != ''): + for index, ins in enumerate(self.latest_tx.vin): + utxo_for_checking = (ins.prevout.hash[::-1], ins.prevout.n) + # 'deadbeef' markers mean our own input scripts are not queried + if ins.scriptSig != b"": continue utxo[ctr] = [index, utxo_for_checking] ctr += 1 utxo_data = jm_single().bc_interface.query_utxo_set([x[ 1] for x in utxo.values()]) - # insert signatures for i, u in iteritems(utxo): if utxo_data[i] is None: continue - #Check if the sender serialize_scripted the scriptCode - #item into the sig message; if so, also pick up the amount - #from the utxo data retrieved from the blockchain to verify - #the segwit-style signature. Note that this allows a mixed - #SW/non-SW transaction as each utxo is interpreted separately. - sig_deserialized = btc.deserialize_script(sig) - #verify_tx_input will not even parse the script if it has integers or None, - #so abort in case we were given a junk sig: - if not all([not isinstance(x, int) and x for x in sig_deserialized]): - jlog.warn("Junk signature: " + str(sig_deserialized) + \ + # Check if the sender included the scriptCode in the sig message; + # if so, also pick up the amount from the utxo data retrieved + # from the blockchain to verify the segwit-style signature. + # Note that this allows a mixed SW/non-SW transaction as each utxo + # is interpreted separately. + try: + sig_deserialized = [a for a in iter(btc.CScript(sig))] + except Exception as e: + jlog.debug("Failed to parse junk sig message, ignoring.") + break + # abort in case we were given a junk sig (note this previously had + # to check to avoid crashes in verify_tx_input, no longer (Feb 2020)): + if not all([x for x in sig_deserialized]): + jlog.debug("Junk signature: " + str(sig_deserialized) + \ ", not attempting to verify") break if len(sig_deserialized) == 2: @@ -583,44 +584,48 @@ def on_sig(self, nick, sigb64): elif len(sig_deserialized) == 3: ver_sig, ver_pub, scriptCode = sig_deserialized else: - jlog.debug("Invalid signature message - more than 3 items") + jlog.debug("Invalid signature message - not 2 or 3 items") break + ver_amt = utxo_data[i]['value'] if scriptCode else None - sig_good = btc.verify_tx_input(txhex, u[0], utxo_data[i]['script'], - ver_sig, ver_pub, scriptCode=scriptCode, amount=ver_amt) - - if ver_amt is not None and not sig_good: - # Special case to deal with legacy bots 0.5.0 or lower: - # the third field in the sigmessage was originally *not* the - # scriptCode, but the contents of tx['ins'][index]['script'], - # i.e. the witness program 0014... ; for this we can verify - # implicitly, as verify_tx_input used to, by reconstructing - # from the public key. For these cases, we can *assume* that - # the input is of type p2sh-p2wpkh; we call the jmbitcoin method - # directly, as we cannot assume that *our* wallet handles this. - scriptCode = hexlify(btc.pubkey_to_p2pkh_script( - ver_pub, True)).decode('ascii') - sig_good = btc.verify_tx_input(txhex, u[0], utxo_data[i]['script'], - ver_sig, ver_pub, scriptCode=scriptCode, amount=ver_amt) + witness = btc.CScriptWitness( + [ver_sig, ver_pub]) if scriptCode else None + + # don't attempt to parse `pub` as pubkey unless it's valid. + if scriptCode: + try: + s = btc.pubkey_to_p2wpkh_script(ver_pub) + except: + jlog.debug("Junk signature message, invalid pubkey, ignoring.") + break + scriptSig = btc.CScript([ver_sig, ver_pub]) if not scriptCode else btc.CScript([s]) + + # Pre-Feb 2020, we used the third field scriptCode differently in + # pre- and post-0.5.0; now the scriptCode is implicit (i.e. calculated + # by underlying library, so that exceptional case is covered. + sig_good = btc.verify_tx_input(self.latest_tx, u[0], scriptSig, + btc.CScript(utxo_data[i]['script']), amount=ver_amt, witness=witness) + + # verification for the native case is functionally identical but + # adds another flag; so we can allow it here: + if not sig_good: + sig_good = btc.verify_tx_input(self.latest_tx, u[0], scriptSig, + btc.CScript(utxo_data[i]['script']), amount=ver_amt, + witness=witness, native=True) + # if passes, below code executes, and we should change for native: + scriptSig = btc.CScript([b""]) if sig_good: jlog.debug('found good sig at index=%d' % (u[0])) + + # Note that, due to the complexity of handling multisig or other + # arbitrary script (considering sending multiple signatures OTW), + # there is an assumption of p2sh-p2wpkh or p2wpkh, for the segwit + # case. + self.latest_tx.vin[u[0]].scriptSig = scriptSig if ver_amt: - # Note that, due to the complexity of handling multisig or other - # arbitrary script (considering sending multiple signatures OTW), - # there is an assumption of p2sh-p2wpkh or p2wpkh, for the segwit - # case. - self.latest_tx["ins"][u[0]]["txinwitness"] = [ver_sig, ver_pub] - if btc.is_segwit_native_script(utxo_data[i]['script']): - scriptSig = "" - else: - scriptSig = btc.serialize_script_unit( - btc.pubkey_to_p2wpkh_script(ver_pub)) - self.latest_tx["ins"][u[0]]["script"] = scriptSig - else: - # Non segwit (as per above comments) is limited only to single key, - # p2pkh case. - self.latest_tx["ins"][u[0]]["script"] = sig + self.latest_tx.wit.vtxinwit[u[0]] = btc.CTxInWitness( + btc.CScriptWitness(witness)) inserted_sig = True # check if maker has sent everything possible @@ -643,8 +648,8 @@ def on_sig(self, nick, sigb64): # other guy sent a failed signature tx_signed = True - for ins in self.latest_tx['ins']: - if ins['script'] == '': + for ins in self.latest_tx.vin: + if ins.scriptSig == b"": tx_signed = False if not tx_signed: return False @@ -667,7 +672,7 @@ def make_commitment(self): def filter_by_coin_age_amt(utxos, age, amt): results = jm_single().bc_interface.query_utxo_set(utxos, - includeconf=True) + includeconf=True) newresults = [] too_old = [] too_small = [] @@ -683,7 +688,6 @@ def filter_by_coin_age_amt(utxos, age, amt): too_small.append(utxos[i]) if valid_age and valid_amt: newresults.append(utxos[i]) - return newresults, too_old, too_small def priv_utxo_pairs_from_utxos(utxos, age, amt): @@ -695,7 +699,7 @@ def priv_utxo_pairs_from_utxos(utxos, age, amt): age, amt) new_utxos_dict = {k: v for k, v in utxos.items() if k in new_utxos} for k, v in iteritems(new_utxos_dict): - addr = v['address'] + addr = self.wallet_service.script_to_addr(v["script"]) priv = self.wallet_service.get_key_from_addr(addr) if priv: #can be null from create-unsigned priv_utxo_pairs.append((priv, k)) @@ -723,7 +727,7 @@ def priv_utxo_pairs_from_utxos(utxos, age, amt): #in the transaction, about to be consumed, rather than use #random utxos that will persist after. At this step we also #allow use of external utxos in the json file. - if any(self.wallet_service.get_utxos_by_mixdepth(hexfmt=False).values()): + if any(self.wallet_service.get_utxos_by_mixdepth().values()): utxos = {} for mdutxo in self.wallet_service.get_utxos_by_mixdepth().values(): utxos.update(mdutxo) @@ -739,13 +743,9 @@ def priv_utxo_pairs_from_utxos(utxos, age, amt): ext_valid = None podle_data = generate_podle(priv_utxo_pairs, tries, ext_valid) if podle_data: - jlog.debug("Generated PoDLE: " + pprint.pformat(podle_data)) - revelation = PoDLE(u=podle_data['utxo'], - P=podle_data['P'], - P2=podle_data['P2'], - s=podle_data['sig'], - e=podle_data['e']).serialize_revelation() - return (commit_type_byte + podle_data["commit"], revelation, + jlog.debug("Generated PoDLE: " + repr(podle_data)) + return (commit_type_byte + bintohex(podle_data.commitment), + podle_data.serialize_revelation(), "Commitment sourced OK") else: errmsgheader, errmsg = generate_podle_error_string(priv_utxo_pairs, @@ -771,25 +771,26 @@ def coinjoin_address(self): def self_sign(self): # now sign it ourselves our_inputs = {} - for index, ins in enumerate(self.latest_tx['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) + for index, ins in enumerate(self.latest_tx.vin): + utxo = (ins.prevout.hash[::-1], ins.prevout.n) if utxo not in self.input_utxos.keys(): continue - script = self.wallet_service.addr_to_script(self.input_utxos[utxo]['address']) + script = self.input_utxos[utxo]["script"] amount = self.input_utxos[utxo]['value'] our_inputs[index] = (script, amount) - self.latest_tx = self.wallet_service.sign_tx(self.latest_tx, our_inputs) + success, msg = self.wallet_service.sign_tx(self.latest_tx, our_inputs) + if not success: + jlog.error("Failed to sign transaction: " + msg) def push(self): - tx = btc.serialize(self.latest_tx) - jlog.debug('\n' + tx) - self.txid = btc.txhash(tx) + jlog.debug('\n' + bintohex(self.latest_tx.serialize())) + self.txid = bintohex(self.latest_tx.GetTxid()[::-1]) jlog.info('txid = ' + self.txid) #If we are sending to a bech32 address, in case of sweep, will #need to use that bech32 for address import, which requires #converting to script (Core does not allow import of bech32) if self.my_cj_addr.lower()[:2] in ['bc', 'tb']: - notify_addr = btc.address_to_script(self.my_cj_addr) + notify_addr = btc.CCoinAddress(self.my_cj_addr).to_scriptPubKey() else: notify_addr = self.my_cj_addr #add the callbacks *before* pushing to ensure triggering; @@ -810,7 +811,7 @@ def push(self): tx_broadcast = jm_single().config.get('POLICY', 'tx_broadcast') nick_to_use = None if tx_broadcast == 'self': - pushed = jm_single().bc_interface.pushtx(tx) + pushed = jm_single().bc_interface.pushtx(self.latest_tx.serialize()) elif tx_broadcast in ['random-peer', 'not-self']: n = len(self.maker_utxo_data) if tx_broadcast == 'random-peer': @@ -818,14 +819,14 @@ def push(self): else: i = random.randrange(n) if i == n: - pushed = jm_single().bc_interface.pushtx(tx) + pushed = jm_single().bc_interface.pushtx(self.latest_tx.serialize()) else: nick_to_use = list(self.maker_utxo_data.keys())[i] pushed = True else: jlog.info("Only self, random-peer and not-self broadcast " "methods supported. Reverting to self-broadcast.") - pushed = jm_single().bc_interface.pushtx(tx) + pushed = jm_single().bc_interface.pushtx(self.latest_tx.serialize()) if not pushed: self.on_finished_callback(False, fromtx=True) else: @@ -841,7 +842,7 @@ def tx_match(self, txd): # Takers process only in series, so this should not occur: assert self.latest_tx is not None # check if the transaction matches our created tx: - if txd['outs'] != self.latest_tx['outs']: + if txd.vout != self.latest_tx.vout: return False return True @@ -928,12 +929,12 @@ def initialize(self, orderbook): # For the p2ep taker, the variable 'my_cj_addr' is the destination: self.my_cj_addr = si[3] if isinstance(self.cjamount, float): - raise JMTakerError("P2EP coinjoin must use amount in satoshis") + raise JMTakerError("Payjoin must use amount in satoshis") if self.cjamount == 0: # Note that we don't allow sweep, currently, since the coin # choosing algo would not apply in that case (we'd have to rewrite # prepare_my_bitcoin_data for that case). - raise JMTakerError("P2EP coinjoin does not currently support sweep") + raise JMTakerError("Payjoin does not currently support sweep") # Next we prepare our coins with the inherited method # for this purpose; for this we must set the @@ -1004,27 +1005,27 @@ def receive_utxos(self, ioauth_data): if self.my_change_addr is not None: self.outputs.append({'address': self.my_change_addr, 'value': my_change_value}) - # As for JM coinjoins, the `None` key is used for our own inputs - # to the transaction; this preparatory version contains only those. - tx = btc.make_shuffled_tx(self.utxos[None], self.outputs, - False, 2, compute_tx_locktime()) - jlog.info('Created proposed fallback tx\n' + pprint.pformat( - btc.deserialize(tx))) + # Oour own inputs to the transaction; this preparatory version + # contains only those. + tx = btc.make_shuffled_tx(self.input_utxos, self.outputs, + version=2, locktime=compute_tx_locktime()) + jlog.info('Created proposed fallback tx\n' + pprint.pformat(str(tx))) # We now sign as a courtesy, because if we disappear the recipient # can still claim his coins with this. # sign our inputs before transfer our_inputs = {} - dtx = btc.deserialize(tx) - for index, ins in enumerate(dtx['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) - script = self.wallet_service.addr_to_script(self.input_utxos[utxo]['address']) - amount = self.input_utxos[utxo]['value'] - our_inputs[index] = (script, amount) - self.signed_noncj_tx = btc.serialize(self.wallet_service.sign_tx(dtx, our_inputs)) + for index, ins in enumerate(tx.vin): + utxo = (ins.prevout.hash[::-1], ins.prevout.n) + our_inputs[index] = (self.input_utxos[utxo]["script"], + self.input_utxos[utxo]['value']) + success, msg = self.wallet_service.sign_tx(tx, our_inputs) + if not success: + jlog.error("Failed to create backup transaction; error: " + msg) self.taker_info_callback("INFO", "Built tx proposal, sending to receiver.") - return (True, [self.p2ep_receiver_nick], self.signed_noncj_tx) + return (True, [self.p2ep_receiver_nick], bintohex(tx.serialize())) - def on_tx_received(self, nick, txhex): + @hexbin + def on_tx_received(self, nick, txser): """ Here the taker (payer) retrieves a version of the transaction from the maker (receiver) which should have the following properties: @@ -1041,12 +1042,14 @@ def on_tx_received(self, nick, txhex): and then broadcast (TODO broadcast delay or don't broadcast). """ try: - tx = btc.deserialize(txhex) - except (IndexError, SerializationError, SerializationTruncationError) as e: + tx = btc.CMutableTransaction.deserialize(txser) + except Exception as e: return (False, "malformed txhex. " + repr(e)) - jlog.info("Obtained tx from receiver:\n" + pprint.pformat(tx)) - cjaddr_script = btc.address_to_script(self.my_cj_addr) - changeaddr_script = btc.address_to_script(self.my_change_addr) + jlog.info("Obtained tx from receiver:\n" + pprint.pformat(str(tx))) + cjaddr_script = btc.CCoinAddress( + self.my_cj_addr).to_scriptPubKey() + changeaddr_script = btc.CCoinAddress( + self.my_change_addr).to_scriptPubKey() # We ensure that the coinjoin address and our expected change # address are still in the outputs, once (with the caveat that @@ -1054,19 +1057,19 @@ def on_tx_received(self, nick, txhex): # of dust change, which we assess after). times_seen_cj_addr = 0 times_seen_change_addr = 0 - for outs in tx['outs']: - if outs['script'] == cjaddr_script: + for outs in tx.vout: + if outs.scriptPubKey == cjaddr_script: times_seen_cj_addr += 1 - new_cj_amount = outs['value'] + new_cj_amount = outs.nValue if new_cj_amount < self.cjamount: # This is a violation of protocol; # receiver must be providing extra bitcoin # as input, so his receiving amount should have increased. return (False, 'Wrong cj_amount. I expect at least' + str(self.cjamount)) - if outs['script'] == changeaddr_script: + if outs.scriptPubKey == changeaddr_script: times_seen_change_addr += 1 - new_change_amount = outs['value'] + new_change_amount = outs.nValue if times_seen_cj_addr != 1: fmt = ('cj addr not in tx outputs once, #cjaddr={}').format return (False, (fmt(times_seen_cj_addr))) @@ -1077,8 +1080,7 @@ def on_tx_received(self, nick, txhex): new_change_amount = 0 # Check that our inputs are present. - tx_utxo_set = set(ins['outpoint']['hash'] + ':' + str( - ins['outpoint']['index']) for ins in tx['ins']) + tx_utxo_set = set((ins.prevout.hash[::-1], ins.prevout.n) for ins in tx.vin) if not tx_utxo_set.issuperset(set(self.utxos[None])): return (False, "my utxos are not contained") # Check that the sequence numbers of all inputs are unaltered @@ -1087,10 +1089,10 @@ def on_tx_received(self, nick, txhex): # Note that this is hacky and is most elegantly addressed by # use of PSBT (although any object encapsulation of tx input # would serve the same purpose). - if tx["locktime"] == 0: + if tx.nLockTime == 0: return (False, "Invalid PayJoin v0 transaction: locktime 0") - for i in tx["ins"]: - if i["sequence"] != 0xffffffff - 1: + for i in tx.vin: + if i.nSequence != 0xffffffff - 1: return (False, "Invalid PayJoin v0 transaction: "+\ "sequence is not 0xffffffff -1") @@ -1099,7 +1101,7 @@ def on_tx_received(self, nick, txhex): # not) of PayJoin to sweep utxos at no cost. # (TODO This is very kludgy, more sophisticated approach # should be used in future): - if len(tx["ins"]) - len (self.utxos[None]) > 5: + if len(tx.vin) - len (self.utxos[None]) > 5: return (False, "proposed tx has more than 5 inputs from " "the recipient, which is too expensive.") @@ -1122,9 +1124,8 @@ def on_tx_received(self, nick, txhex): # checking input validity and transaction balance. retrieve_utxos = {} ctr = 0 - for index, ins in enumerate(tx['ins']): - utxo_for_checking = ins['outpoint']['hash'] + ':' + str( - ins['outpoint']['index']) + for index, ins in enumerate(tx.vin): + utxo_for_checking = (ins.prevout.hash[::-1], ins.prevout.n) if utxo_for_checking in self.utxos[None]: continue retrieve_utxos[ctr] = [index, utxo_for_checking] @@ -1140,32 +1141,11 @@ def on_tx_received(self, nick, txhex): if utxo_data[i] is None: return (False, "Proposed transaction contains invalid utxos") total_receiver_input += utxo_data[i]["value"] - scriptCode = None - ver_amt = None idx = retrieve_utxos[i][0] - if "txinwitness" in tx["ins"][idx]: - ver_amt = utxo_data[i]["value"] - try: - ver_sig, ver_pub = tx["ins"][idx]["txinwitness"] - except Exception as e: - print("Segwit error: ", repr(e)) - return (False, "Segwit input not of expected type, " - "either p2sh-p2wpkh or p2wpkh") - # note that the scriptCode is the same whether nested or not - # also note that the scriptCode has to be inferred if we are - # only given a transaction serialization. - scriptCode = "76a914" + btc.hash160(unhexlify(ver_pub)) + "88ac" - else: - scriptSig = btc.deserialize_script(tx["ins"][idx]["script"]) - if len(scriptSig) != 2: - return (False, - "Proposed transaction contains unsupported input type") - ver_sig, ver_pub = scriptSig - if not btc.verify_tx_input(txhex, idx, - utxo_data[i]['script'], - ver_sig, ver_pub, - scriptCode=scriptCode, - amount=ver_amt): + if not btc.verify_tx_input(tx, idx, tx.vin[idx].scriptSig, + btc.CScript(utxo_data[i]['script']), + amount=utxo_data[i]["value"], + witness=tx.wit.vtxinwit[idx].scriptWitness): return (False, "Proposed transaction is not correctly signed.") payment = new_cj_amount - total_receiver_input @@ -1185,7 +1165,7 @@ def on_tx_received(self, nick, txhex): # our fee estimator. Its return value will be governed by our own fee settings # in joinmarket.cfg; allow either (a) automatic agreement for any value within # a range of 0.3 to 3x this figure, or (b) user to agree on prompt. - fee_est = estimate_tx_fee(len(tx['ins']), len(tx['outs']), + fee_est = estimate_tx_fee(len(tx.vin), len(tx.vout), txtype=self.wallet_service.get_txtype()) fee_ok = False if btc_fee > 0.3 * fee_est and btc_fee < 3 * fee_est: diff --git a/jmclient/jmclient/taker_utils.py b/jmclient/jmclient/taker_utils.py index 852401812..4f325442b 100644 --- a/jmclient/jmclient/taker_utils.py +++ b/jmclient/jmclient/taker_utils.py @@ -5,15 +5,13 @@ import sys import time import numbers - -from jmbase import get_log, jmprint +from jmbase import get_log, jmprint, bintohex from .configure import jm_single, validate_address, is_burn_destination from .schedule import human_readable_schedule_entry, tweak_tumble_schedule,\ schedule_to_text from .wallet import BaseWallet, estimate_tx_fee, compute_tx_locktime, \ FidelityBondMixin -from jmbitcoin import deserialize, make_shuffled_tx, serialize, txhash,\ - amount_to_str, mk_burn_script, bin_hash160 +from jmbitcoin import make_shuffled_tx, amount_to_str, mk_burn_script from jmbase.support import EXIT_SUCCESS log = get_log() @@ -136,13 +134,20 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, log.info("Using a fee of : " + amount_to_str(fee_est) + ".") if amount != 0: log.info("Using a change value of: " + amount_to_str(changeval) + ".") - txsigned = sign_tx(wallet_service, make_shuffled_tx( - list(utxos.keys()), outs, False, 2, tx_locktime), utxos) + tx = make_shuffled_tx(list(utxos.keys()), outs, 2, compute_tx_locktime()) + list(utxos.keys()), outs, 2, tx_locktime), utxos) + inscripts = {} + for i, txinp in enumerate(tx.vin): + u = (txinp.prevout.hash[::-1], txinp.prevout.n) + inscripts[i] = (utxos[u]["script"], utxos[u]["value"]) + success, msg = wallet_service.sign_tx(tx, inscripts) + if not success: + log.error("Failed to sign transaction, quitting. Error msg: " + msg) + return log.info("Got signed transaction:\n") - log.info(pformat(txsigned)) - tx = serialize(txsigned) + log.info(pformat(str(tx))) log.info("In serialized form (for copy-paste):") - log.info(tx) + log.info(bintohex(tx.serialize())) actual_amount = amount if amount != 0 else total_inputs_val - fee_est log.info("Sends: " + amount_to_str(actual_amount) + " to destination: " + destination) if not answeryes: @@ -151,12 +156,12 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, log.info("You chose not to broadcast the transaction, quitting.") return False else: - accepted = accept_callback(pformat(txsigned), destination, actual_amount, + accepted = accept_callback(pformat(str(tx)), destination, actual_amount, fee_est) if not accepted: return False - jm_single().bc_interface.pushtx(tx) - txid = txhash(tx) + jm_single().bc_interface.pushtx(tx.serialize()) + txid = bintohex(tx.GetTxid()[::-1]) successmsg = "Transaction sent: " + txid cb = log.info if not info_callback else info_callback cb(successmsg) diff --git a/jmclient/jmclient/wallet.py b/jmclient/jmclient/wallet.py index 5159a0df3..2a62a28c3 100644 --- a/jmclient/jmclient/wallet.py +++ b/jmclient/jmclient/wallet.py @@ -27,7 +27,7 @@ from .support import get_random_bytes from . import mn_encode, mn_decode import jmbitcoin as btc -from jmbase import JM_WALLET_NAME_PREFIX +from jmbase import JM_WALLET_NAME_PREFIX, bintohex """ @@ -198,7 +198,8 @@ def remove_utxo(self, txid, index, mixdepth): assert isinstance(index, numbers.Integral) assert isinstance(mixdepth, numbers.Integral) - return self._utxo[mixdepth].pop((txid, index)) + x = self._utxo[mixdepth].pop((txid, index)) + return x def add_utxo(self, txid, index, path, value, mixdepth, height=None): # Assumed: that we add a utxo only if we want it enabled, @@ -433,19 +434,22 @@ def sign_tx(self, tx, scripts, **kwargs): Add signatures to transaction for inputs referenced by scripts. args: - tx: transaction dict + tx: CMutableTransaction object scripts: {input_index: (output_script, amount)} kwargs: additional arguments for engine.sign_transaction returns: - input transaction dict with added signatures, hex-encoded. + True, None if success. + False, msg if signing failed, with error msg. """ for index, (script, amount) in scripts.items(): assert amount > 0 path = self.script_to_path(script) privkey, engine = self._get_key_from_path(path) - tx = btc.deserialize(engine.sign_transaction(tx, index, privkey, - amount, **kwargs)) - return tx + sig, msg = engine.sign_transaction(tx, index, privkey, + amount, **kwargs) + if not sig: + return False, msg + return True, None @deprecated def get_key_from_addr(self, addr): @@ -455,7 +459,7 @@ def get_key_from_addr(self, addr): script = self._ENGINE.address_to_script(addr) path = self.script_to_path(script) privkey = self._get_key_from_path(path)[0] - return hexlify(privkey).decode('ascii') + return privkey def _get_addr_int_ext(self, address_type, mixdepth): if address_type == self.ADDRESS_TYPE_EXTERNAL: @@ -578,32 +582,19 @@ def update_cache_index(self): """ self.save() - @deprecated def remove_old_utxos(self, tx): - tx = deepcopy(tx) - for inp in tx['ins']: - inp['outpoint']['hash'] = unhexlify(inp['outpoint']['hash']) - - ret = self.remove_old_utxos_(tx) - - removed_utxos = {} - for (txid, index), val in ret.items(): - val['address'] = self.get_address_from_path(val['path']) - removed_utxos[hexlify(txid).decode('ascii') + ':' + str(index)] = val - return removed_utxos - - def remove_old_utxos_(self, tx): """ Remove all own inputs of tx from internal utxo list. args: - tx: transaction dict + tx: CMutableTransaction returns: {(txid, index): {'script': bytes, 'path': str, 'value': int} for all removed utxos """ removed_utxos = {} - for inp in tx['ins']: - txid, index = inp['outpoint']['hash'], inp['outpoint']['index'] + for inp in tx.vin: + txid = inp.prevout.hash[::-1] + index = inp.prevout.n md = self._utxos.have_utxo(txid, index) if md is False: continue @@ -614,46 +605,32 @@ def remove_old_utxos_(self, tx): 'value': value} return removed_utxos - @deprecated - def add_new_utxos(self, tx, txid, height=None): - tx = deepcopy(tx) - for out in tx['outs']: - out['script'] = unhexlify(out['script']) - - ret = self.add_new_utxos_(tx, unhexlify(txid), height=height) - - added_utxos = {} - for (txid_bin, index), val in ret.items(): - addr = self.get_address_from_path(val['path']) - val['address'] = addr - added_utxos[txid + ':' + str(index)] = val - return added_utxos - - def add_new_utxos_(self, tx, txid, height=None): + def add_new_utxos(self, tx, height=None): """ Add all outputs of tx for this wallet to internal utxo list. - + They are also returned in standard dict form. args: - tx: transaction dict + tx: CMutableTransaction height: blockheight in which tx was included, or None if unconfirmed. returns: - {(txid, index): {'script': bytes, 'path': tuple, 'value': int} + {(txid, index): {'script': bytes, 'path': tuple, 'value': int, + 'address': str} for all added utxos """ - assert isinstance(txid, bytes) and len(txid) == self._utxos.TXID_LEN added_utxos = {} - for index, outs in enumerate(tx['outs']): + txid = tx.GetTxid()[::-1] + for index, outs in enumerate(tx.vout): + spk = outs.scriptPubKey + val = outs.nValue try: - self.add_utxo(txid, index, outs['script'], outs['value'], - height=height) + self.add_utxo(txid, index, spk, val, height=height) except WalletError: continue - path = self.script_to_path(outs['script']) - added_utxos[(txid, index)] = {'script': outs['script'], - 'path': path, - 'value': outs['value']} + path = self.script_to_path(spk) + added_utxos[(txid, index)] = {'script': spk, 'path': path, 'value': val, + 'address': self._ENGINE.script_to_address(spk)} return added_utxos def add_utxo(self, txid, index, script, value, height=None): @@ -669,9 +646,10 @@ def add_utxo(self, txid, index, script, value, height=None): mixdepth = self._get_mixdepth_from_path(path) self._utxos.add_utxo(txid, index, path, value, mixdepth, height=height) - def process_new_tx(self, txd, txid, height=None): - """ Given a newly seen transaction, deserialized as txd and - with transaction id, process its inputs and outputs and update + def process_new_tx(self, txd, height=None): + """ Given a newly seen transaction, deserialized as + CMutableTransaction txd, + process its inputs and outputs and update the utxo contents of this wallet accordingly. NOTE: this should correctly handle transactions that are not actually related to the wallet; it will not add (or remove, @@ -679,30 +657,15 @@ def process_new_tx(self, txd, txid, height=None): functions check this condition. """ removed_utxos = self.remove_old_utxos(txd) - added_utxos = self.add_new_utxos(txd, txid, height=height) + added_utxos = self.add_new_utxos(txd, height=height) return (removed_utxos, added_utxos) - @deprecated - def select_utxos(self, mixdepth, amount, utxo_filter=None, select_fn=None, - maxheight=None): - utxo_filter_new = None - if utxo_filter: - utxo_filter_new = [(unhexlify(utxo[:64]), int(utxo[65:])) - for utxo in utxo_filter] - ret = self.select_utxos_(mixdepth, amount, utxo_filter_new, select_fn, - maxheight=maxheight) - ret_conv = {} - for utxo, data in ret.items(): - addr = self.get_address_from_path(data['path']) - utxo_txt = hexlify(utxo[0]).decode('ascii') + ':' + str(utxo[1]) - ret_conv[utxo_txt] = {'address': addr, 'value': data['value']} - return ret_conv - - def select_utxos_(self, mixdepth, amount, utxo_filter=None, - select_fn=None, maxheight=None): + def select_utxos(self, mixdepth, amount, utxo_filter=None, + select_fn=None, maxheight=None, includeaddr=False): """ Select a subset of available UTXOS for a given mixdepth whose value is - greater or equal to amount. + greater or equal to amount. If `includeaddr` is True, adds an `address` + key to the returned dict. args: mixdepth: int, mixdepth to select utxos from, must be smaller or @@ -713,6 +676,7 @@ def select_utxos_(self, mixdepth, amount, utxo_filter=None, returns: {(txid, index): {'script': bytes, 'path': tuple, 'value': int}} + """ assert isinstance(mixdepth, numbers.Integral) assert isinstance(amount, numbers.Integral) @@ -728,7 +692,8 @@ def select_utxos_(self, mixdepth, amount, utxo_filter=None, for data in ret.values(): data['script'] = self.get_script_from_path(data['path']) - + if includeaddr: + data["address"] = self.get_address_from_path(data["path"]) return ret def disable_utxo(self, txid, index, disable=True): @@ -758,21 +723,7 @@ def get_balance_by_mixdepth(self, verbose=True, include_disabled=include_disabled, maxheight=maxheight) - @deprecated - def get_utxos_by_mixdepth(self, verbose=True, includeheight=False): - # TODO: verbose - ret = self.get_utxos_by_mixdepth_(includeheight=includeheight) - - utxos_conv = collections.defaultdict(dict) - for md, utxos in ret.items(): - for utxo, data in utxos.items(): - utxo_str = hexlify(utxo[0]).decode('ascii') + ':' + str(utxo[1]) - addr = self.get_address_from_path(data['path']) - data['address'] = addr - utxos_conv[md][utxo_str] = data - return utxos_conv - - def get_utxos_by_mixdepth_(self, include_disabled=False, includeheight=False): + def get_utxos_by_mixdepth(self, include_disabled=False, includeheight=False): """ Get all UTXOs for active mixdepths. @@ -791,9 +742,11 @@ def get_utxos_by_mixdepth_(self, include_disabled=False, includeheight=False): if not include_disabled and self._utxos.is_disabled(*utxo): continue script = self.get_script_from_path(path) + addr = self.get_address_from_path(path) script_utxos[md][utxo] = {'script': script, 'path': path, - 'value': value} + 'value': value, + 'address': addr} if includeheight: script_utxos[md][utxo]['height'] = height return script_utxos @@ -1101,7 +1054,6 @@ def import_private_key(self, mixdepth, wif, key_type=None): raise WalletError("Unsupported key type for imported keys.") privkey, key_type_wif = self._ENGINE.wif_to_privkey(wif) - # FIXME: there is no established standard for encoding key type in wif #if key_type is not None and key_type_wif is not None and \ # key_type != key_type_wif: diff --git a/jmclient/jmclient/wallet_service.py b/jmclient/jmclient/wallet_service.py index ef186b286..35521058a 100644 --- a/jmclient/jmclient/wallet_service.py +++ b/jmclient/jmclient/wallet_service.py @@ -16,8 +16,9 @@ from jmclient.blockchaininterface import (INF_HEIGHT, BitcoinCoreInterface, BitcoinCoreNoHistoryInterface) from jmclient.wallet import FidelityBondMixin -from jmbase.support import jmprint, EXIT_SUCCESS -import jmbitcoin as btc +from jmbase.support import jmprint, EXIT_SUCCESS, utxo_to_utxostr, bintohex, hextobin +from jmbitcoin import lx + """Wallet service The purpose of this independent service is to allow @@ -173,9 +174,9 @@ def register_callbacks(self, callbacks, txinfo, cb_type="all"): and return type boolean. `txinfo` - either a txid expected for the transaction, if known, or a tuple of the ordered output set, of the form - (('script': script), ('value': value), ..). This can be - constructed from jmbitcoin.deserialize output, key "outs", - using tuple(). See WalletService.transaction_monitor(). + ((CScript, int), ..). This is be constructed from the + CMutableTransaction vout list. + See WalletService.transaction_monitor(). `cb_type` - must be one of "all", "unconfirmed", "confirmed"; the first type will be called back once for every new transaction, the second only once when the number of @@ -223,7 +224,9 @@ def import_non_wallet_address(self, address): self.bci.import_addresses([address], self.EXTERNAL_WALLET_LABEL, restart_cb=self.restart_callback) - def default_autofreeze_warning_cb(self, utxostr): + def default_autofreeze_warning_cb(self, utxo): + success, utxostr = utxo_to_utxostr(utxo) + assert success, "Autofreeze warning cb called with invalid utxo." jlog.warning("WARNING: new utxo has been automatically " "frozen to prevent forced address reuse: ") jlog.warning(utxostr) @@ -267,9 +270,7 @@ def check_for_reuse(self, added_utxos): utxo]["value"] <= freeze_threshold: # freezing of coins must be communicated to user: self.autofreeze_warning_cb(utxo) - # process_new_tx returns added utxos in str format: - txidstr, idx = utxo.split(":") - self.disable_utxo(binascii.unhexlify(txidstr), int(idx)) + self.disable_utxo(*utxo) def transaction_monitor(self): """Keeps track of any changes in the wallet (new transactions). @@ -298,7 +299,7 @@ def transaction_monitor(self): for tx in new_txs: txid = tx["txid"] - res = self.bci.get_transaction(txid) + res = self.bci.get_transaction(hextobin(txid)) if not res: continue confs = res["confirmations"] @@ -317,7 +318,7 @@ def transaction_monitor(self): txd = self.bci.get_deser_from_gettransaction(res) if txd is None: continue - removed_utxos, added_utxos = self.wallet.process_new_tx(txd, txid, height) + removed_utxos, added_utxos = self.wallet.process_new_tx(txd, height) if txid not in self.processed_txids: # apply checks to disable/freeze utxos to reused addrs if needed: self.check_for_reuse(added_utxos) @@ -339,16 +340,9 @@ def transaction_monitor(self): f(txd, txid) # The tuple given as the second possible key for the dict - # is such because dict keys must be hashable types, so a simple - # replication of the entries in the list tx["outs"], where tx - # was generated via jmbitcoin.deserialize, is unacceptable to - # Python, since they are dicts. However their keyset is deterministic - # so it is sufficient to convert these dicts to tuples with fixed - # ordering, thus it can be used as a key into the self.callbacks - # dicts. (This is needed because txid is not always available + # is such because txid is not always available # at the time of callback registration). - possible_keys = [txid, tuple( - (x["script"], x["value"]) for x in txd["outs"])] + possible_keys = [txid, tuple((x.scriptPubKey, x.nValue) for x in txd.vout)] # note that len(added_utxos) > 0 is not a sufficient condition for # the tx being new, since wallet.add_new_utxos will happily re-add @@ -406,8 +400,9 @@ def log_new_tx(self, removed_utxos, added_utxos, txid): def report_changed(x, utxos): if len(utxos.keys()) > 0: jlog.info(x + ' utxos=\n{}'.format('\n'.join( - '{} - {}'.format(u, fmt_tx_data(tx_data, self)) - for u, tx_data in utxos.items()))) + '{} - {}'.format(utxo_to_utxostr(u)[1], + fmt_tx_data(tx_data, self)) for u, + tx_data in utxos.items()))) report_changed("Removed", removed_utxos) report_changed("Added", added_utxos) @@ -752,14 +747,15 @@ def sync_unspent(self): def _add_unspent_txo(self, utxo, height): """ Add a UTXO as returned by rpc's listunspent call to the wallet. - + Note that these are returned as little endian outpoint txids, so + are converted. params: utxo: single utxo dict as returned by listunspent current_blockheight: blockheight as integer, used to set the block in which a confirmed utxo is included. """ - txid = binascii.unhexlify(utxo['txid']) - script = binascii.unhexlify(utxo['scriptPubKey']) + txid = hextobin(utxo['txid']) + script = hextobin(utxo['scriptPubKey']) value = int(Decimal(str(utxo['amount'])) * Decimal('1e8')) self.add_utxo(txid, int(utxo['vout']), script, value, height) @@ -774,11 +770,9 @@ def save_wallet(self): self.wallet.save() def get_utxos_by_mixdepth(self, include_disabled=False, - verbose=False, hexfmt=True, includeconfs=False): + verbose=False, includeconfs=False): """ Returns utxos by mixdepth in a dict, optionally including information about how many confirmations each utxo has. - TODO clean up underlying wallet.get_utxos_by_mixdepth (including verbosity - and formatting options) to make this less confusing. """ def height_to_confs(x): # convert height entries to confirmations: @@ -793,36 +787,30 @@ def height_to_confs(x): confs = self.current_blockheight - h + 1 ubym_conv[m][u]["confs"] = confs return ubym_conv + ubym = self.wallet.get_utxos_by_mixdepth( + include_disabled=include_disabled, includeheight=includeconfs) + if not includeconfs: + return ubym + else: + return height_to_confs(ubym) - if hexfmt: - ubym = self.wallet.get_utxos_by_mixdepth(verbose=verbose, - includeheight=includeconfs) - if not includeconfs: - return ubym - else: - return height_to_confs(ubym) + def minconfs_to_maxheight(self, minconfs): + if minconfs is None: + return None else: - ubym = self.wallet.get_utxos_by_mixdepth_( - include_disabled=include_disabled, includeheight=includeconfs) - if not includeconfs: - return ubym - else: - return height_to_confs(ubym) + return self.current_blockheight - minconfs + 1 def select_utxos(self, mixdepth, amount, utxo_filter=None, select_fn=None, - minconfs=None): + minconfs=None, includeaddr=False): """ Request utxos from the wallet in a particular mixdepth to satisfy a certain total amount, optionally set the selector function (or use the currently configured function set by the wallet, and optionally require a minimum of minconfs confirmations (default none means unconfirmed are allowed). """ - if minconfs is None: - maxheight = None - else: - maxheight = self.current_blockheight - minconfs + 1 return self.wallet.select_utxos(mixdepth, amount, utxo_filter=utxo_filter, - select_fn=select_fn, maxheight=maxheight) + select_fn=select_fn, maxheight=self.minconfs_to_maxheight(minconfs), + includeaddr=includeaddr) def get_balance_by_mixdepth(self, verbose=True, include_disabled=False, diff --git a/jmclient/jmclient/wallet_utils.py b/jmclient/jmclient/wallet_utils.py index 6770a9eeb..8aa381829 100644 --- a/jmclient/jmclient/wallet_utils.py +++ b/jmclient/jmclient/wallet_utils.py @@ -16,7 +16,8 @@ LegacyWallet, SegwitWallet, FidelityBondMixin, FidelityBondWatchonlyWallet, is_native_segwit_mode, load_program_config, add_base_options, check_regtest) from jmclient.wallet_service import WalletService -from jmbase.support import get_password, jmprint, EXIT_FAILURE, EXIT_ARGERROR +from jmbase.support import (get_password, jmprint, EXIT_FAILURE, + EXIT_ARGERROR, utxo_to_utxostr) from .cryptoengine import TYPE_P2PKH, TYPE_P2SH_P2WPKH, TYPE_P2WPKH, \ TYPE_SEGWIT_LEGACY_WALLET_FIDELITY_BONDS @@ -369,22 +370,26 @@ def wallet_showutxos(wallet, showprivkey): utxos = wallet.get_utxos_by_mixdepth(includeconfs=True) for md in utxos: for u, av in utxos[md].items(): + success, us = utxo_to_utxostr(u) + assert success key = wallet.get_key_from_addr(av['address']) tries = podle.get_podle_tries(u, key, max_tries) tries_remaining = max(0, max_tries - tries) - unsp[u] = {'address': av['address'], 'value': av['value'], + unsp[us] = {'address': av['address'], 'value': av['value'], 'tries': tries, 'tries_remaining': tries_remaining, 'external': False, 'confirmations': av['confs']} if showprivkey: - unsp[u]['privkey'] = wallet.get_wif_path(av['path']) + unsp[us]['privkey'] = wallet.get_wif_path(av['path']) used_commitments, external_commitments = podle.get_podle_commitments() for u, ec in iteritems(external_commitments): + success, us = utxo_to_utxostr(u) + assert success tries = podle.get_podle_tries(utxo=u, max_tries=max_tries, external=True) tries_remaining = max(0, max_tries - tries) - unsp[u] = {'tries': tries, 'tries_remaining': tries_remaining, + unsp[us] = {'tries': tries, 'tries_remaining': tries_remaining, 'external': True} return json.dumps(unsp, indent=4) @@ -431,7 +436,7 @@ def get_addr_status(addr_path, utxos, is_new, is_internal): acctlist = [] # TODO - either optionally not show disabled utxos, or # mark them differently in display (labels; colors) - utxos = wallet_service.get_utxos_by_mixdepth(include_disabled=True, hexfmt=False) + utxos = wallet_service.get_utxos_by_mixdepth(include_disabled=True) for m in range(wallet_service.mixdepth + 1): branchlist = [] for address_type in [0, 1]: diff --git a/jmclient/jmclient/yieldgenerator.py b/jmclient/jmclient/yieldgenerator.py index b18eb9063..f89af3604 100644 --- a/jmclient/jmclient/yieldgenerator.py +++ b/jmclient/jmclient/yieldgenerator.py @@ -132,7 +132,8 @@ def oid_to_order(self, offer, amount): change_addr = self.wallet_service.get_internal_addr(mixdepth) - utxos = self.wallet_service.select_utxos(mixdepth, total_amount, minconfs=1) + utxos = self.wallet_service.select_utxos(mixdepth, total_amount, + minconfs=1, includeaddr=True) my_total_in = sum([va['value'] for va in utxos.values()]) real_cjfee = calc_cj_fee(offer["ordertype"], offer["cjfee"], amount) change_value = my_total_in - amount - offer["txfee"] + real_cjfee @@ -141,7 +142,8 @@ def oid_to_order(self, offer, amount): 'finding new utxos').format(change_value)) try: utxos = self.wallet_service.select_utxos(mixdepth, - total_amount + jm_single().DUST_THRESHOLD, minconfs=1) + total_amount + jm_single().DUST_THRESHOLD, + minconfs=1, includeaddr=True) except Exception: jlog.info('dont have the required UTXOs to make a ' 'output above the dust threshold, quitting') diff --git a/jmclient/test/commontest.py b/jmclient/test/commontest.py index 01ce37a3e..36ed7c444 100644 --- a/jmclient/test/commontest.py +++ b/jmclient/test/commontest.py @@ -6,11 +6,13 @@ import random from decimal import Decimal -from jmbase import get_log +from jmbase import (get_log, hextobin, utxostr_to_utxo, + utxo_to_utxostr, listchanger, dictchanger) + from jmclient import ( jm_single, open_test_wallet_maybe, estimate_tx_fee, BlockchainInterface, get_p2sh_vbyte, BIP32Wallet, - SegwitLegacyWallet, WalletService) + SegwitLegacyWallet, WalletService, BTC_P2SH_P2WPKH) from jmbase.support import chunks import jmbitcoin as btc @@ -45,9 +47,11 @@ def import_addresses(self, addr_list, wallet_name, restart_cb=None): pass def is_address_imported(self, addr): pass - + + def get_current_block_height(self): + return 10**6 + def pushtx(self, txhex): - print("pushing: " + str(txhex)) return True def insert_fake_query_results(self, fqr): @@ -56,7 +60,7 @@ def insert_fake_query_results(self, fqr): def setQUSFail(self, state): self.qusfail = state - def query_utxo_set(self, txouts,includeconf=False): + def query_utxo_set(self, txouts, includeconf=False): if self.qusfail: #simulate failure to find the utxo return [None] @@ -72,12 +76,14 @@ def query_utxo_set(self, txouts,includeconf=False): known_outs = {"03243f4a659e278a1333f8308f6aaf32db4692ee7df0340202750fd6c09150f6:1": "03a2d1cbe977b1feaf8d0d5cc28c686859563d1520b28018be0c2661cf1ebe4857", "498faa8b22534f3b443c6b0ce202f31e12f21668b4f0c7a005146808f250d4c3:0": "02b4b749d54e96b04066b0803e372a43d6ffa16e75a001ae0ed4b235674ab286be", "3f3ea820d706e08ad8dc1d2c392c98facb1b067ae4c671043ae9461057bd2a3c:1": "023bcbafb4f68455e0d1d117c178b0e82a84e66414f0987453d78da034b299c3a9"} + known_outs = dictchanger(known_outs) #our wallet utxos, faked, for podle tests: utxos are doctored (leading 'f'), #and the lists are (amt, age) wallet_outs = {'f34b635ed8891f16c4ec5b8236ae86164783903e8e8bb47fa9ef2ca31f3c2d7a:0': [10000000, 2], 'f780d6e5e381bff01a3519997bb4fcba002493103a198fde334fd264f9835d75:1': [20000000, 6], 'fe574db96a4d43a99786b3ea653cda9e4388f377848f489332577e018380cff1:0': [50000000, 3], 'fd9711a2ef340750db21efb761f5f7d665d94b312332dc354e252c77e9c48349:0': [50000000, 6]} + wallet_outs = dictchanger(wallet_outs) if includeconf and set(txouts).issubset(set(wallet_outs)): #includeconf used as a trigger for a podle check; @@ -88,16 +94,16 @@ def query_utxo_set(self, txouts,includeconf=False): 'confirms': wallet_outs[to][1]}) return results if txouts[0] in known_outs: - addr = btc.pubkey_to_p2sh_p2wpkh_address( - known_outs[txouts[0]], get_p2sh_vbyte()) + scr = BTC_P2SH_P2WPKH.pubkey_to_script(known_outs[txouts[0]]) + addr = btc.CCoinAddress.from_scriptPubKey(scr) return [{'value': 200000000, 'address': addr, - 'script': btc.address_to_script(addr), + 'script': scr, 'confirms': 20}] for t in txouts: result_dict = {'value': 10000000000, 'address': "mrcNu71ztWjAQA6ww9kHiW3zBWSQidHXTQ", - 'script': '76a91479b000887626b294a914501a4cd226b58b23598388ac'} + 'script': hextobin('76a91479b000887626b294a914501a4cd226b58b23598388ac')} if includeconf: result_dict['confirms'] = 20 result.append(result_dict) @@ -110,18 +116,10 @@ def estimate_fee_per_kb(self, N): def create_wallet_for_sync(wallet_structure, a, **kwargs): #We need a distinct seed for each run so as not to step over each other; #make it through a deterministic hash - seedh = btc.sha256("".join([str(x) for x in a]))[:32] + seedh = btc.b2x(btc.Hash("".join([str(x) for x in a]).encode("utf-8")))[:32] return make_wallets( 1, [wallet_structure], fixed_seeds=[seedh], **kwargs)[0]['wallet'] - -def binarize_tx(tx): - for o in tx['outs']: - o['script'] = binascii.unhexlify(o['script']) - for i in tx['ins']: - i['outpoint']['hash'] = binascii.unhexlify(i['outpoint']['hash']) - - def make_sign_and_push(ins_full, wallet_service, amount, @@ -130,7 +128,12 @@ def make_sign_and_push(ins_full, hashcode=btc.SIGHASH_ALL, estimate_fee = False): """Utility function for easily building transactions - from wallets + from wallets. + `ins_full` should be a list of dicts in format returned + by wallet.select_utxos: + {(txid, index): {"script":..,"value":..,"path":..}} + ... although the path is not used. + The "script" and "value" data is used to allow signing. """ assert isinstance(wallet_service, WalletService) total = sum(x['value'] for x in ins_full.values()) @@ -143,22 +146,21 @@ def make_sign_and_push(ins_full, 'address': output_addr}, {'value': total - amount - fee_est, 'address': change_addr}] - de_tx = btc.deserialize(btc.mktx(ins, outs)) + tx = btc.mktx(ins, outs) scripts = {} - for index, ins in enumerate(de_tx['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) - script = wallet_service.addr_to_script(ins_full[utxo]['address']) - scripts[index] = (script, ins_full[utxo]['value']) - binarize_tx(de_tx) - de_tx = wallet_service.sign_tx(de_tx, scripts, hashcode=hashcode) + for i, j in enumerate(ins): + scripts[i] = (ins_full[j]["script"], ins_full[j]["value"]) + + success, msg = wallet_service.sign_tx(tx, scripts, hashcode=hashcode) + if not success: + return False #pushtx returns False on any error - push_succeed = jm_single().bc_interface.pushtx(btc.serialize(de_tx)) + push_succeed = jm_single().bc_interface.pushtx(tx.serialize()) if push_succeed: - txid = btc.txhash(btc.serialize(de_tx)) # in normal operation this happens automatically # but in some tests there is no monitoring loop: - wallet_service.process_new_tx(de_tx, txid) - return txid + wallet_service.process_new_tx(tx) + return tx.GetTxid()[::-1] else: return False diff --git a/jmclient/test/test_client_protocol.py b/jmclient/test/test_client_protocol.py index 0ced71237..7583a555f 100644 --- a/jmclient/test/test_client_protocol.py +++ b/jmclient/test/test_client_protocol.py @@ -1,7 +1,8 @@ #! /usr/bin/env python '''test client-protocol interfacae.''' -from jmbase import get_log +from jmbase import get_log, bintohex, hextobin +from jmbase.commands import * from jmclient import load_test_config, Taker,\ JMClientProtocolFactory, jm_single, Maker, WalletService from jmclient.client_protocol import JMTakerClientProtocol @@ -14,12 +15,10 @@ from twisted.protocols import amp from twisted.trial import unittest from twisted.test import proto_helpers -from jmbase.commands import * from taker_test_data import t_raw_signed_tx from commontest import default_max_cj_fee import json import jmbitcoin as bitcoin - import twisted twisted.internet.base.DelayedCall.debug = True @@ -97,7 +96,7 @@ def try_to_create_my_orders(self): def on_auth_received(self, nick, offer, commitment, cr, amount, kphex): # success, utxos, auth_pub, cj_addr, change_addr, btc_sig - return True, [], '', '', '', '' + return True, [], b"", '', '', '' def on_tx_received(self, nick, txhex, offerinfo): # success, sigs @@ -227,8 +226,8 @@ def on_JM_MAKE_TX(self, nick_list, txhex): self.defaultCallbacks(d2) #To test, this must include a valid ecdsa sig fullmsg = "fullmsgforverify" - priv = "aa"*32 + "01" - pub = bitcoin.privkey_to_pubkey(priv) + priv = b"\xaa"*32 + b"\x01" + pub = bintohex(bitcoin.privkey_to_pubkey(priv)) sig = bitcoin.ecdsa_sign(fullmsg, priv) d3 = self.callRemote(JMRequestMsgSigVerify, msg="msgforverify", @@ -262,7 +261,7 @@ class JMTestServerProtocolFactory(protocol.ServerFactory): class DummyClientProtocolFactory(JMClientProtocolFactory): def buildProtocol(self, addr): - return JMTakerClientProtocol(self, self.client, nick_priv="aa"*32) + return JMTakerClientProtocol(self, self.client, nick_priv=b"\xaa"*32 + b"\x01") class TrialTestJMClientProto(unittest.TestCase): @@ -364,8 +363,8 @@ def test_JMRequestMsgSig(self): @inlineCallbacks def test_JMRequestMsgSigVerify(self): fullmsg = 'fullmsgforverify' - priv = 'aa'*32 + '01' - pub = bitcoin.privkey_to_pubkey(priv) + priv = b"\xaa"*32 + b"\x01" + pub = bintohex(bitcoin.privkey_to_pubkey(priv)) sig = bitcoin.ecdsa_sign(fullmsg, priv) yield self.init_client() yield self.callClient( diff --git a/jmclient/test/test_coinjoin.py b/jmclient/test/test_coinjoin.py index 6428a2b91..810bf7151 100644 --- a/jmclient/test/test_coinjoin.py +++ b/jmclient/test/test_coinjoin.py @@ -6,14 +6,15 @@ import os import sys import pytest +import copy from twisted.internet import reactor -from jmbase import get_log +from jmbase import get_log, hextobin, bintohex from jmclient import load_test_config, jm_single,\ YieldGeneratorBasic, Taker, LegacyWallet, SegwitLegacyWallet,\ NO_ROUNDING from jmclient.podle import set_commitment_file -from commontest import make_wallets, binarize_tx, default_max_cj_fee +from commontest import make_wallets, default_max_cj_fee from test_taker import dummy_filter_orderbook import jmbitcoin as btc @@ -85,14 +86,14 @@ def init_coinjoin(taker, makers, orderbook, cj_amount): ioauth_data = list(response[1:]) ioauth_data[0] = list(ioauth_data[0].keys()) # maker_pk which is set up by jmdaemon - ioauth_data.append(None) + ioauth_data.append("00") maker_data[mid] = ioauth_data # this is handled by jmdaemon active_orders[mid]['utxos'] = response[1] active_orders[mid]['cjaddr'] = ioauth_data[2] active_orders[mid]['changeaddr'] = ioauth_data[3] - active_orders[mid]['offer'] = m.offerlist[0] + active_orders[mid]['offer'] = copy.deepcopy(m.offerlist[0]) active_orders[mid]['amount'] = cj_amount return active_orders, maker_data @@ -194,13 +195,12 @@ def raise_exit(i): taker_final_result = do_tx_signing(taker, makers, active_orders, txdata) assert taker_final_result is not False - tx = btc.deserialize(txdata[2]) - binarize_tx(tx) + tx = btc.CMutableTransaction.deserialize(hextobin(txdata[2])) wallet_service = wallet_services[-1] # TODO change for new tx monitoring: - wallet_service.remove_old_utxos_(tx) - wallet_service.add_new_utxos_(tx, b'\x00' * 32) # fake txid + wallet_service.remove_old_utxos(tx) + wallet_service.add_new_utxos(tx) balances = wallet_service.get_balance_by_mixdepth() assert balances[0] == cj_amount @@ -250,14 +250,13 @@ def raise_exit(i): taker_final_result = do_tx_signing(taker, makers, active_orders, txdata) assert taker_final_result is not False - tx = btc.deserialize(txdata[2]) - binarize_tx(tx) + tx = btc.CMutableTransaction.deserialize(hextobin(txdata[2])) for i in range(MAKER_NUM): wallet_service = wallet_services[i] # TODO as above re: monitoring - wallet_service.remove_old_utxos_(tx) - wallet_service.add_new_utxos_(tx, b'\x00' * 32) # fake txid + wallet_service.remove_old_utxos(tx) + wallet_service.add_new_utxos(tx) balances = wallet_service.get_balance_by_mixdepth() assert balances[0] == cj_amount diff --git a/jmclient/test/test_commitment_utils.py b/jmclient/test/test_commitment_utils.py index 31000ab1a..070e7e8ec 100644 --- a/jmclient/test/test_commitment_utils.py +++ b/jmclient/test/test_commitment_utils.py @@ -4,11 +4,14 @@ from jmclient import (load_test_config, jm_single) from jmclient.commitment_utils import get_utxo_info, validate_utxo_data - +from jmbitcoin import select_chain_params def test_get_utxo_info(): load_test_config() + # this test tests mainnet keys, so temporarily switch network + select_chain_params("bitcoin") jm_single().config.set("BLOCKCHAIN", "network", "mainnet") + dbci = DummyBlockchainInterface() privkey = "L1RrrnXkcKut5DEMwtDthjwRcTTwED36thyL1DebVrKuwvohjMNi" #to verify use from_wif_privkey and privkey_to_address @@ -28,7 +31,7 @@ def test_get_utxo_info(): with pytest.raises(Exception) as e_info: u, priv = get_utxo_info(fakeutxo + privkey) #invalid index - fu2 = "ab"*32 + ":00004" + fu2 = "ab"*32 + ":-1" with pytest.raises(Exception) as e_info: u, priv = get_utxo_info(fu2 + "," + privkey) #invalid privkey @@ -53,3 +56,5 @@ def test_get_utxo_info(): retval = validate_utxo_data(utxodatas, False) assert not retval dbci.setQUSFail(False) + select_chain_params("bitcoin/regtest") + jm_single().config.set("BLOCKCHAIN", "network", "mainnet") diff --git a/jmclient/test/test_maker.py b/jmclient/test/test_maker.py index 1bc53e9ab..a7046f2ef 100644 --- a/jmclient/test/test_maker.py +++ b/jmclient/test/test_maker.py @@ -29,8 +29,7 @@ def construct_tx_offerlist(cjaddr, changeaddr, maker_utxos, maker_utxos_value, 'txfee': 0 } - utxos = { utxo['outpoint']['hash'] + ':' + str(utxo['outpoint']['index']): - {'utxo': utxo, 'value': maker_utxos_value} for utxo in maker_utxos } + utxos = { utxo: {'utxo': utxo, 'value': maker_utxos_value} for utxo in maker_utxos } offerlist = { 'utxos': utxos, @@ -46,59 +45,56 @@ def construct_tx_offerlist(cjaddr, changeaddr, maker_utxos, maker_utxos_value, def create_tx_inputs(count=1): inp = [] for i in range(count): - inp.append({'outpoint': {'hash': '0'*64, 'index': i}, - 'script': '', - 'sequence': 4294967295}) + inp.append((b"\x00"*32, i)) return inp -def create_tx_outputs(*scripts_amount): +def create_tx_outputs(*addrs_amount): outp = [] - for script, amount in scripts_amount: - outp.append({'script': script, 'value': amount}) + for addr, amount in addrs_amount: + outp.append({'address': addr, 'value': amount}) return outp def address_p2pkh_generator(): - return get_address_generator(b'\x76\xa9\x14', b'\x88\xac', get_p2pk_vbyte()) + return get_address_generator(b'\x76\xa9\x14', b'\x88\xac') def address_p2sh_generator(): - return get_address_generator(b'\xa9\x14', b'\x87', get_p2sh_vbyte()) + return get_address_generator(b'\xa9\x14', b'\x87', p2sh=True) -def get_address_generator(script_pre, script_post, vbyte): +def get_address_generator(script_pre, script_post, p2sh=False): counter = 0 while True: script = script_pre + struct.pack(b'=LQQ', 0, 0, counter) + script_post - addr = btc.script_to_address(script, vbyte) - yield addr, binascii.hexlify(script).decode('ascii') + if p2sh: + addr = btc.CCoinAddress.from_scriptPubKey( + btc.CScript(script).to_p2sh_scriptPubKey()) + else: + addr = btc.CCoinAddress.from_scriptPubKey(btc.CScript(script)) + yield str(addr), binascii.hexlify(script).decode('ascii') counter += 1 -def create_tx_and_offerlist(cj_addr, cj_change_addr, other_output_scripts, - cj_script=None, cj_change_script=None, offertype='swreloffer'): - assert len(other_output_scripts) % 2 == 0, "bug in test" +def create_tx_and_offerlist(cj_addr, cj_change_addr, other_output_addrs, + offertype='swreloffer'): + assert len(other_output_addrs) % 2 == 0, "bug in test" cj_value = 100000000 maker_total_value = cj_value*3 - if cj_script is None: - cj_script = btc.address_to_script(cj_addr) - if cj_change_script is None: - cj_change_script = btc.address_to_script(cj_change_addr) - inputs = create_tx_inputs(3) outputs = create_tx_outputs( - (cj_script, cj_value), - (cj_change_script, maker_total_value - cj_value), # cjfee=0, txfee=0 - *((script, cj_value + (i%2)*(50000000+i)) \ - for i, script in enumerate(other_output_scripts)) + (cj_addr, cj_value), + (cj_change_addr, maker_total_value - cj_value), # cjfee=0, txfee=0 + *((addr, cj_value + (i%2)*(50000000+i)) \ + for i, addr in enumerate(other_output_addrs)) ) maker_utxos = [inputs[0]] - tx = btc.deserialize(btc.mktx(inputs, outputs)) + tx = btc.mktx(inputs, outputs) offerlist = construct_tx_offerlist(cj_addr, cj_change_addr, maker_utxos, maker_total_value, cj_value, offertype) @@ -119,21 +115,20 @@ def test_verify_unsigned_tx_sw_valid(setup_env_nodeps): # test standard cj tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr, - [next(p2sh_gen)[1] for s in range(4)], cj_script, cj_change_script) + [next(p2sh_gen)[0] for s in range(4)]) assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "standard sw cj" # test cj with mixed outputs tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr, - list(chain((next(p2sh_gen)[1] for s in range(3)), - (next(p2pkh_gen)[1] for s in range(1)))), - cj_script, cj_change_script) + list(chain((next(p2sh_gen)[0] for s in range(3)), + (next(p2pkh_gen)[0] for s in range(1))))) assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "sw cj with p2pkh output" # test cj with only p2pkh outputs tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr, - [next(p2pkh_gen)[1] for s in range(4)], cj_script, cj_change_script) + [next(p2pkh_gen)[0] for s in range(4)]) assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "sw cj with only p2pkh outputs" @@ -152,21 +147,20 @@ def test_verify_unsigned_tx_nonsw_valid(setup_env_nodeps): # test standard cj tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr, - [next(p2pkh_gen)[1] for s in range(4)], cj_script, cj_change_script, 'reloffer') + [next(p2pkh_gen)[0] for s in range(4)], offertype='reloffer') assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "standard nonsw cj" # test cj with mixed outputs tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr, - list(chain((next(p2sh_gen)[1] for s in range(1)), - (next(p2pkh_gen)[1] for s in range(3)))), - cj_script, cj_change_script, 'reloffer') + list(chain((next(p2sh_gen)[0] for s in range(1)), + (next(p2pkh_gen)[0] for s in range(3)))), offertype='reloffer') assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "nonsw cj with p2sh output" # test cj with only p2sh outputs tx, offerlist = create_tx_and_offerlist(cj_addr, changeaddr, - [next(p2sh_gen)[1] for s in range(4)], cj_script, cj_change_script, 'reloffer') + [next(p2sh_gen)[0] for s in range(4)], offertype='reloffer') assert maker.verify_unsigned_tx(tx, offerlist) == (True, None), "nonsw cj with only p2sh outputs" @@ -175,4 +169,5 @@ def test_verify_unsigned_tx_nonsw_valid(setup_env_nodeps): def setup_env_nodeps(monkeypatch): monkeypatch.setattr(jmclient.configure, 'get_blockchain_interface_instance', lambda x: DummyBlockchainInterface()) + btc.select_chain_params("bitcoin/regtest") load_test_config() diff --git a/jmclient/test/test_payjoin.py b/jmclient/test/test_payjoin.py index 3670100b1..6d0ba142a 100644 --- a/jmclient/test/test_payjoin.py +++ b/jmclient/test/test_payjoin.py @@ -75,14 +75,11 @@ def final_checks(wallet_services, amount, txfee, tsb, msb, source_mixdepth=0): return True @pytest.mark.parametrize('wallet_cls, wallet_structures, mean_amt', - [([LegacyWallet, LegacyWallet], - [[4, 0, 0, 0, 0]] * 2, 1.0), + [ # note we have removed LegacyWallet test cases. ([SegwitLegacyWallet, SegwitLegacyWallet], [[1, 3, 0, 0, 0]] * 2, 2.0), ([SegwitWallet, SegwitWallet], [[1, 0, 0, 0, 0]] * 2, 4.0), - ([LegacyWallet, SegwitWallet], - [[4, 0, 0, 0, 0]] * 2, 1.0), ([SegwitLegacyWallet, SegwitWallet], [[1, 3, 0, 0, 0]] * 2, 2.0), ([SegwitWallet, SegwitLegacyWallet], diff --git a/jmclient/test/test_podle.py b/jmclient/test/test_podle.py index d49fe80cc..a643b7e19 100644 --- a/jmclient/test/test_podle.py +++ b/jmclient/test/test_podle.py @@ -28,8 +28,8 @@ def test_commitment_retries(setup_podle): """ allowed = jm_single().config.getint("POLICY", "taker_utxo_retries") #make some pretend commitments - dummy_priv_utxo_pairs = [(bitcoin.sha256(os.urandom(10)), - bitcoin.sha256(os.urandom(10))+":0") for _ in range(10)] + dummy_priv_utxo_pairs = [(bitcoin.Hash(os.urandom(10)), + bitcoin.b2x(bitcoin.Hash(os.urandom(10)))+":0") for _ in range(10)] #test a single commitment request of all 10 for x in dummy_priv_utxo_pairs: p = generate_podle([x], allowed) @@ -46,15 +46,15 @@ def generate_single_podle_sig(priv, i): This calls the underlying 'raw' code based on the class PoDLE, not the library 'generate_podle' which intelligently searches and updates commitments. """ - dummy_utxo = bitcoin.sha256(priv) + ":3" - podle = PoDLE(dummy_utxo, binascii.hexlify(priv).decode('ascii')) + dummy_utxo = bitcoin.b2x(bitcoin.Hash(priv)) + ":3" + podle = PoDLE(dummy_utxo, priv) r = podle.generate_podle(i) return (r['P'], r['P2'], r['sig'], r['e'], r['commit']) def test_rand_commitments(setup_podle): for i in range(20): - priv = os.urandom(32) + priv = os.urandom(32)+b"\x01" Pser, P2ser, s, e, commitment = generate_single_podle_sig(priv, 1 + i%5) assert verify_podle(Pser, P2ser, s, e, commitment) #tweak commitments to verify failure @@ -90,7 +90,7 @@ def test_external_commitments(setup_podle): tries = jm_single().config.getint("POLICY","taker_utxo_retries") for i in range(10): priv = os.urandom(32) - dummy_utxo = bitcoin.sha256(priv)+":2" + dummy_utxo = (bitcoin.Hash(priv), 2) ecs[dummy_utxo] = {} ecs[dummy_utxo]['reveal']={} for j in range(tries): @@ -104,16 +104,16 @@ def test_external_commitments(setup_podle): assert external[u]['P'] == ecs[u]['P'] for i in range(tries): for x in ['P2', 's', 'e']: - assert external[u]['reveal'][str(i)][x] == ecs[u]['reveal'][i][x] + assert external[u]['reveal'][i][x] == ecs[u]['reveal'][i][x] #add a dummy used commitment, then try again - update_commitments(commitment="ab"*32) + update_commitments(commitment=b"\xab"*32) ecs = {} known_commits = [] known_utxos = [] tries = 3 for i in range(1, 6): - u = binascii.hexlify(struct.pack(b'B', i)*32).decode('ascii') + u = (struct.pack(b'B', i)*32, i+3) known_utxos.append(u) priv = struct.pack(b'B', i)*32+b"\x01" ecs[u] = {} @@ -131,8 +131,9 @@ def test_external_commitments(setup_podle): #this should find the remaining one utxo and return from it assert generate_podle([], max_tries=tries, allow_external=known_utxos) #test commitment removal - to_remove = ecs[binascii.hexlify(struct.pack(b'B', 3)*32).decode('ascii')] - update_commitments(external_to_remove={binascii.hexlify(struct.pack(b'B', 3)*32).decode('ascii'):to_remove}) + tru = (struct.pack(b"B", 3)*32, 3+3) + to_remove = {tru: ecs[tru]} + update_commitments(external_to_remove=to_remove) #test that an incorrectly formatted file raises with open(get_commitment_file(), "rb") as f: validjson = json.loads(f.read().decode('utf-8')) @@ -152,14 +153,14 @@ def test_podle_constructor(setup_podle): """Tests rules about construction of PoDLE object are conformed to. """ - priv = "aa"*32 + priv = b"\xaa"*32 #pub and priv together not allowed with pytest.raises(PoDLEError) as e_info: p = PoDLE(priv=priv, P="dummypub") #no pub or priv is allowed, i forget if this is useful for something p = PoDLE() #create from priv - p = PoDLE(priv=priv+"01", u="dummyutxo") + p = PoDLE(priv=priv+b"\x01", u=(struct.pack(b"B", 7)*32, 4)) pdict = p.generate_podle(2) assert all([k in pdict for k in ['used', 'utxo', 'P', 'P2', 'commit', 'sig', 'e']]) #using the valid data, serialize/deserialize test @@ -181,7 +182,7 @@ def test_podle_constructor(setup_podle): with pytest.raises(PoDLEError) as e_info: p.generate_podle(0) #Test construction from pubkey - pub = bitcoin.privkey_to_pubkey(priv+"01") + pub = bitcoin.privkey_to_pubkey(priv+b"\x01") p = PoDLE(P=pub) with pytest.raises(PoDLEError) as e_info: p.get_commitment() diff --git a/jmbitcoin/test/test_keys.py b/jmclient/test/test_privkeys.py similarity index 75% rename from jmbitcoin/test/test_keys.py rename to jmclient/test/test_privkeys.py index 06c48fd50..2fb6884eb 100644 --- a/jmbitcoin/test/test_keys.py +++ b/jmclient/test/test_privkeys.py @@ -2,13 +2,16 @@ '''Public and private key validity and formatting tests.''' import jmbitcoin as btc +from jmclient import (BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH, + jm_single, load_test_config) import binascii +import struct import json import pytest import os testdir = os.path.dirname(os.path.realpath(__file__)) -def test_read_raw_privkeys(): +def test_read_raw_privkeys(setup_keys): badkeys = [b'', b'\x07'*31,b'\x07'*34, b'\x07'*33] for b in badkeys: with pytest.raises(Exception) as e_info: @@ -18,7 +21,7 @@ def test_read_raw_privkeys(): c, k = btc.read_privkey(g[0]) assert c == g[1] -def test_wif_privkeys_invalid(): +def test_wif_privkeys_invalid(setup_keys): #first try to create wif privkey from key of wrong length bad_privs = [b'\x01\x02'*17] #some silly private key but > 33 bytes @@ -27,7 +30,7 @@ def test_wif_privkeys_invalid(): for priv in bad_privs: with pytest.raises(Exception) as e_info: - fake_wif = btc.wif_compressed_privkey(binascii.hexlify(priv).decode('ascii')) + fake_wif = BTCEngine.privkey_to_wif(priv) #Create a wif with wrong length bad_wif1 = btc.bin_to_b58check(b'\x01\x02'*34, b'\x80') @@ -35,7 +38,7 @@ def test_wif_privkeys_invalid(): bad_wif2 = btc.bin_to_b58check(b'\x07'*33, b'\x80') for bw in [bad_wif1, bad_wif2]: with pytest.raises(Exception) as e_info: - fake_priv = btc.from_wif_privkey(bw) + fake_priv, keytype = BTCEngine.wif_to_privkey(bw) #Some invalid b58 from bitcoin repo; #none of these are valid as any kind of key or address @@ -49,15 +52,14 @@ def test_wif_privkeys_invalid(): print('testing this key: ' + bad_key) #should throw exception with pytest.raises(Exception) as e_info: - from_wif_key = btc.from_wif_privkey(bad_key, - btc.get_version_byte(bad_key)) + from_wif_key, keytype = BTCEngine.wif_to_privkey(bad_key) #in case the b58 check encoding is valid, we should #also check if the leading version byte is in the #expected set, and throw an error if not. if chr(btc.get_version_byte(bad_key)) not in b'\x80\xef': raise Exception("Invalid version byte") -def test_wif_privkeys_valid(): +def test_wif_privkeys_valid(setup_keys): with open(os.path.join(testdir,"base58_keys_valid.json"), "r") as f: json_data = f.read() valid_keys_list = json.loads(json_data) @@ -65,16 +67,20 @@ def test_wif_privkeys_valid(): key, hex_key, prop_dict = a if prop_dict["isPrivkey"]: netval = "testnet" if prop_dict["isTestnet"] else "mainnet" + jm_single().config.set("BLOCKCHAIN", "network", netval) print('testing this key: ' + key) assert btc.get_version_byte( key) in b'\x80\xef', "not valid network byte" comp = prop_dict["isCompressed"] - from_wif_key = btc.from_wif_privkey( - key, - compressed=comp, - vbyte=btc.from_int_to_byte(btc.from_byte_to_int(btc.get_version_byte(key))-128)) - expected_key = hex_key - if comp: expected_key += '01' + if not comp: + # we only handle compressed keys + continue + from_wif_key, keytype = BTCEngine.wif_to_privkey(key) + expected_key = binascii.unhexlify(hex_key) + b"\x01" assert from_wif_key == expected_key, "Incorrect key decoding: " + \ str(from_wif_key) + ", should be: " + str(expected_key) + jm_single().config.set("BLOCKCHAIN", "network", "testnet") +@pytest.fixture(scope='module') +def setup_keys(): + load_test_config() \ No newline at end of file diff --git a/jmclient/test/test_taker.py b/jmclient/test/test_taker.py index bc00d78ba..3cb0bd304 100644 --- a/jmclient/test/test_taker.py +++ b/jmclient/test/test_taker.py @@ -10,13 +10,21 @@ import json import struct from base64 import b64encode +from jmbase import (utxostr_to_utxo, utxo_to_utxostr, hextobin, + dictchanger, listchanger) from jmclient import load_test_config, jm_single, set_commitment_file,\ get_commitment_file, SegwitLegacyWallet, Taker, VolatileStorage,\ - get_network, WalletService, NO_ROUNDING + get_network, WalletService, NO_ROUNDING, BTC_P2PKH from taker_test_data import t_utxos_by_mixdepth, t_orderbook,\ t_maker_response, t_chosen_orders, t_dummy_ext from commontest import default_max_cj_fee +def convert_utxos(utxodict): + return_dict = {} + for uk, val in utxodict.items(): + return_dict[utxostr_to_utxo(uk)[1]] = val + return return_dict + class DummyWallet(SegwitLegacyWallet): def __init__(self): storage = VolatileStorage() @@ -36,32 +44,30 @@ def _add_utxos(self): script = self._ENGINE.address_to_script(data['address']) self._script_map[script] = path - def get_utxos_by_mixdepth(self, verbose=True, includeheight=False): - return t_utxos_by_mixdepth - - def get_utxos_by_mixdepth_(self, verbose=True, include_disabled=False, - includeheight=False): - utxos = self.get_utxos_by_mixdepth(verbose) - - utxos_conv = {} - for md, utxo_data in utxos.items(): - md_utxo = utxos_conv.setdefault(md, {}) - for i, (utxo_hex, data) in enumerate(utxo_data.items()): - utxo, index = utxo_hex.split(':') - data_conv = { - 'script': self._ENGINE.address_to_script(data['address']), - 'path': (b'dummy', md, i), - 'value': data['value'] - } - md_utxo[(binascii.unhexlify(utxo), int(index))] = data_conv - - return utxos_conv + def get_utxos_by_mixdepth(self, include_disabled=False, verbose=True, + includeheight=False): + # utxostr conversion routines because taker_test_data uses hex: + retval = {} + for mixdepth, v in t_utxos_by_mixdepth.items(): + retval[mixdepth] = {} + for i, (utxo, val) in enumerate(v.items()): + retval[mixdepth][utxostr_to_utxo(utxo)[1]] = val + val["script"] = self._ENGINE.address_to_script(val['address']) + val["path"] = (b'dummy', mixdepth, i) + return retval def select_utxos(self, mixdepth, amount, utxo_filter=None, select_fn=None, - maxheight=None): + maxheight=None, includeaddr=False): if amount > self.get_balance_by_mixdepth()[mixdepth]: raise Exception("Not enough funds") - return t_utxos_by_mixdepth[mixdepth] + # comment as for get_utxos_by_mixdepth: + retval = {} + for k, v in t_utxos_by_mixdepth[mixdepth].items(): + success, u = utxostr_to_utxo(k) + assert success + retval[u] = v + retval[u]["script"] = self.addr_to_script(retval[u]["address"]) + return retval def get_internal_addr(self, mixing_depth, bci=None): if self.inject_addr_get_failure: @@ -70,7 +76,7 @@ def get_internal_addr(self, mixing_depth, bci=None): def sign_tx(self, tx, addrs): print("Pretending to sign on addresses: " + str(addrs)) - return tx + return True, None def sign(self, tx, i, priv, amount): """Sign a transaction; the amount field @@ -97,10 +103,10 @@ def get_key_from_addr(self, addr): musGZczug3BAbqobmYherywCwL9REgNaNm """ for p in privs: - addrs[p] = bitcoin.privkey_to_address(p, False, magicbyte=0x6f) + addrs[p] = BTC_P2PKH.privkey_to_address(p) for p, a in iteritems(addrs): if a == addr: - return binascii.hexlify(p).decode('ascii') + return p raise ValueError("No such keypair") def _is_my_bip32_path(self, path): @@ -170,7 +176,7 @@ def clean_up(): amount = 110000000 taker = get_taker([(mixdepth, amount, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", NO_ROUNDING)]) taker.cjamount = amount - taker.input_utxos = t_utxos_by_mixdepth[0] + taker.input_utxos = convert_utxos(t_utxos_by_mixdepth[0]) if failquery: jm_single().bc_interface.setQUSFail(True) taker.make_commitment() @@ -194,12 +200,13 @@ def test_auth_pub_not_found(setup_taker): res = taker.initialize(orderbook) taker.orderbook = copy.deepcopy(t_chosen_orders) #total_cjfee unaffected, all same maker_response = copy.deepcopy(t_maker_response) - utxos = ["03243f4a659e278a1333f8308f6aaf32db4692ee7df0340202750fd6c09150f6:1", - "498faa8b22534f3b443c6b0ce202f31e12f21668b4f0c7a005146808f250d4c3:0", - "3f3ea820d706e08ad8dc1d2c392c98facb1b067ae4c671043ae9461057bd2a3c:1"] + utxos = [utxostr_to_utxo(x)[1] for x in [ + "03243f4a659e278a1333f8308f6aaf32db4692ee7df0340202750fd6c09150f6:1", + "498faa8b22534f3b443c6b0ce202f31e12f21668b4f0c7a005146808f250d4c3:0", + "3f3ea820d706e08ad8dc1d2c392c98facb1b067ae4c671043ae9461057bd2a3c:1"]] fake_query_results = [{'value': 200000000, 'address': "mrKTGvFfYUEqk52qPKUroumZJcpjHLQ6pn", - 'script': '76a914767c956efe6092a775fea39a06d1cac9aae956d788ac', + 'script': hextobin('76a914767c956efe6092a775fea39a06d1cac9aae956d788ac'), 'utxo': utxos[i], 'confirms': 20} for i in range(3)] jm_single().bc_interface.insert_fake_query_results(fake_query_results) @@ -223,8 +230,8 @@ def test_auth_pub_not_found(setup_taker): ([(0, 199850001, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False, 2, False, None, None), #trigger sub dust change for taker #edge case triggers that do fail - ([(0, 199850000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False, - 2, False, None, None), #trigger negative change + ([(0, 199851000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False, + 2, False, None, None), #trigger negative change ([(0, 199599800, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, False, 2, False, None, None), #trigger sub dust change for maker ([(0, 20000000, 3, "INTERNAL", 0, NO_ROUNDING)], True, False, @@ -232,7 +239,7 @@ def test_auth_pub_not_found(setup_taker): ([(0, 20000000, 3, "INTERNAL", 0, NO_ROUNDING)], False, False, 7, False, None, None), #test not enough cp ([(0, 80000000, 3, "INTERNAL", 0, NO_ROUNDING)], False, False, - 2, False, None, "30000"), #test failed commit + 2, False, None, "30000"), #test failed commit ([(0, 20000000, 3, "INTERNAL", 0, NO_ROUNDING)], False, False, 2, True, None, None), #test unauthed response ([(0, 5000000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw", 0, NO_ROUNDING)], False, True, @@ -282,14 +289,15 @@ def clean_up(): if notauthed: #Doctor one of the maker response data fields maker_response["J659UPUSLLjHJpaB"][1] = "xx" #the auth pub - if schedule[0][1] == 199850000: + if schedule[0][1] == 199851000: #triggers negative change - #makers offer 3000 txfee; we estimate ~ 147*10 + 2*34 + 10=1548 bytes - #times 30k = 46440, so we pay 43440, plus maker fees = 3*0.0002*200000000 - #roughly, gives required selected = amt + 163k, hence the above = - #2btc - 150k sats = 199850000 (tweaked because of aggressive coin selection) + #((109 + 4*64)*ins + 34 * outs + 8)/4. plug in 9 ins and 8 outs gives + #tx size estimate = 1101 bytes. Times 30 ~= 33030. + #makers offer 3000 txfee, so we pay 30030, plus maker fees = 3*0.0002*200000000 + #roughly, gives required selected = amt + 120k+30k, hence the above = + #2btc - 140k sats = 199851000 (tweaked because of aggressive coin selection) #simulate the effect of a maker giving us a lot more utxos - taker.utxos["dummy_for_negative_change"] = ["a", "b", "c", "d", "e"] + taker.utxos["dummy_for_negative_change"] = [(struct.pack(b"B", a) *32, a+1) for a in range(7,12)] with pytest.raises(ValueError) as e_info: res = taker.receive_utxos(maker_response) return clean_up() @@ -361,7 +369,7 @@ def clean_up(): [ (7), ]) -def test_unconfirm_confirm(schedule_len): +def test_unconfirm_confirm(setup_taker, schedule_len): """These functions are: do-nothing by default (unconfirm, for Taker), and merely update schedule index for confirm (useful for schedules/tumbles). This tests that the on_finished callback correctly reports the fromtx @@ -369,20 +377,26 @@ def test_unconfirm_confirm(schedule_len): The exception to the above is that the txd passed in must match self.latest_tx, so we use a dummy value here for that. """ + class DummyTx(object): + pass test_unconfirm_confirm.txflag = True def finished_for_confirms(res, fromtx=False, waittime=0, txdetails=None): assert res #confirmed should always send true test_unconfirm_confirm.txflag = fromtx taker = get_taker(schedule_len=schedule_len, on_finished=finished_for_confirms) - taker.latest_tx = {"outs": "blah"} - taker.unconfirm_callback({"ins": "foo", "outs": "blah"}, "b") + taker.latest_tx = DummyTx() + taker.latest_tx.vout = "blah" + fake_txd = DummyTx() + fake_txd.vin = "foo" + fake_txd.vout = "blah" + taker.unconfirm_callback(fake_txd, "b") for i in range(schedule_len-1): taker.schedule_index += 1 - fromtx = taker.confirm_callback({"ins": "foo", "outs": "blah"}, "b", 1) + fromtx = taker.confirm_callback(fake_txd, "b", 1) assert test_unconfirm_confirm.txflag taker.schedule_index += 1 - fromtx = taker.confirm_callback({"ins": "foo", "outs": "blah"}, "b", 1) + fromtx = taker.confirm_callback(fake_txd, "b", 1) assert not test_unconfirm_confirm.txflag @pytest.mark.parametrize( @@ -396,16 +410,13 @@ def test_on_sig(setup_taker, dummyaddr, schedule): #then, create a signature with various inputs, pass in in b64 to on_sig. #in order for it to verify, the DummyBlockchainInterface will have to #return the right values in query_utxo_set - + utxos = [(struct.pack(b"B", x) * 32, 1) for x in range(5)] #create 2 privkey + utxos that are to be ours privs = [x*32 + b"\x01" for x in [struct.pack(b'B', y) for y in range(1,6)]] - utxos = [str(x)*64+":1" for x in range(5)] - fake_query_results = [{'value': 200000000, - 'utxo': utxos[x], - 'address': bitcoin.privkey_to_address(privs[x], False, magicbyte=0x6f), - 'script': bitcoin.mk_pubkey_script( - bitcoin.privkey_to_address(privs[x], False, magicbyte=0x6f)), - 'confirms': 20} for x in range(5)] + scripts = [BTC_P2PKH.privkey_to_script(privs[x]) for x in range(5)] + addrs = [BTC_P2PKH.privkey_to_address(privs[x]) for x in range(5)] + fake_query_results = [{'value': 200000000, 'utxo': utxos[x], 'address': addrs[x], + 'script': scripts[x], 'confirms': 20} for x in range(5)] dbci = DummyBlockchainInterface() dbci.insert_fake_query_results(fake_query_results) @@ -414,47 +425,52 @@ def test_on_sig(setup_taker, dummyaddr, schedule): outs = [{'value': 100000000, 'address': dummyaddr}, {'value': 899990000, 'address': dummyaddr}] tx = bitcoin.mktx(utxos, outs) - - de_tx = bitcoin.deserialize(tx) + # since tx will be updated as it is signed, unlike in real life + # (where maker signing operation doesn't happen here), we'll create + # a second copy without the signatures: + tx2 = bitcoin.mktx(utxos, outs) + #prepare the Taker with the right intermediate data taker = get_taker(schedule=schedule) taker.nonrespondants=["cp1", "cp2", "cp3"] - taker.latest_tx = de_tx + taker.latest_tx = tx #my inputs are the first 2 utxos taker.input_utxos = {utxos[0]: - {'address': bitcoin.privkey_to_address(privs[0], False, magicbyte=0x6f), - 'script': bitcoin.mk_pubkey_script( - bitcoin.privkey_to_address(privs[0], False, magicbyte=0x6f)), + {'address': addrs[0], + 'script': scripts[0], 'value': 200000000}, utxos[1]: - {'address': bitcoin.privkey_to_address(privs[1], False, magicbyte=0x6f), - 'script': bitcoin.mk_pubkey_script( - bitcoin.privkey_to_address(privs[1], False, magicbyte=0x6f)), + {'address': addrs[1], + 'script': scripts[1], 'value': 200000000}} taker.utxos = {None: utxos[:2], "cp1": [utxos[2]], "cp2": [utxos[3]], "cp3":[utxos[4]]} for i in range(2): # placeholders required for my inputs - taker.latest_tx['ins'][i]['script'] = 'deadbeef' + taker.latest_tx.vin[i].scriptSig = bitcoin.CScript(hextobin('deadbeef')) + tx2.vin[i].scriptSig = bitcoin.CScript(hextobin('deadbeef')) #to prepare for my signing, need to mark cjaddr: taker.my_cj_addr = dummyaddr #make signatures for the last 3 fake utxos, considered as "not ours": - tx3 = bitcoin.sign(tx, 2, privs[2]) - sig3 = b64encode(binascii.unhexlify(bitcoin.deserialize(tx3)['ins'][2]['script'])) + sig, msg = bitcoin.sign(tx2, 2, privs[2]) + assert sig, "Failed to sign: " + msg + sig3 = b64encode(tx2.vin[2].scriptSig) taker.on_sig("cp1", sig3) #try sending the same sig again; should be ignored taker.on_sig("cp1", sig3) - tx4 = bitcoin.sign(tx, 3, privs[3]) - sig4 = b64encode(binascii.unhexlify(bitcoin.deserialize(tx4)['ins'][3]['script'])) + sig, msg = bitcoin.sign(tx2, 3, privs[3]) + assert sig, "Failed to sign: " + msg + sig4 = b64encode(tx2.vin[3].scriptSig) #try sending junk instead of cp2's correct sig - taker.on_sig("cp2", str("junk")) + assert not taker.on_sig("cp2", str("junk")), "incorrectly accepted junk signature" taker.on_sig("cp2", sig4) - tx5 = bitcoin.sign(tx, 4, privs[4]) + sig, msg = bitcoin.sign(tx2, 4, privs[4]) + assert sig, "Failed to sign: " + msg #Before completing with the final signature, which will trigger our own #signing, try with an injected failure of query utxo set, which should #prevent this signature being accepted. dbci.setQUSFail(True) - sig5 = b64encode(binascii.unhexlify(bitcoin.deserialize(tx5)['ins'][4]['script'])) - taker.on_sig("cp3", sig5) + sig5 = b64encode(tx2.vin[4].scriptSig) + assert not taker.on_sig("cp3", sig5), "incorrectly accepted sig5" #allow it to succeed, and try again dbci.setQUSFail(False) #this should succeed and trigger the we-sign code @@ -465,7 +481,7 @@ def test_on_sig(setup_taker, dummyaddr, schedule): [ ([(0, 20000000, 3, "mnsquzxrHXpFsZeL42qwbKdCP2y1esN3qw")]), ]) -def test_auth_counterparty(schedule): +def test_auth_counterparty(setup_taker, schedule): taker = get_taker(schedule=schedule) first_maker_response = t_maker_response["J659UPUSLLjHJpaB"] utxo, auth_pub, cjaddr, changeaddr, sig, maker_pub = first_maker_response diff --git a/jmclient/test/test_tx_creation.py b/jmclient/test/test_tx_creation.py index d251b43fc..7c1e75c7c 100644 --- a/jmclient/test/test_tx_creation.py +++ b/jmclient/test/test_tx_creation.py @@ -1,6 +1,9 @@ #! /usr/bin/env python '''Test of unusual transaction types creation and push to -network to check validity.''' + network to check validity. + Note as of Feb 2020: earlier versions included multisig + p2(w)sh tests, these have been removed since Joinmarket + does not use this feature.''' import time import binascii @@ -24,62 +27,12 @@ "028a2f126e3999ff66d01dcb101ab526d3aa1bf5cbdc4bde14950a4cead95f6fcb", "02bea84d70e74f7603746b62d79bf035e16d982b56e6a1ee07dfd3b9130e8a2ad9"] - -@pytest.mark.parametrize( - "nw, wallet_structures, mean_amt, sdev_amt, amount, pubs, k", [ - (1, [[2, 1, 4, 0, 0]], 4, 1.4, 600000000, vpubs[1:4], 2), - (1, [[3, 3, 0, 0, 3]], 4, 1.4, 100000000, vpubs[:4], 3), - ]) -def test_create_p2sh_output_tx(setup_tx_creation, nw, wallet_structures, - mean_amt, sdev_amt, amount, pubs, k): - wallets = make_wallets(nw, wallet_structures, mean_amt, sdev_amt) - for w in wallets.values(): - w['wallet'].sync_wallet(fast=True) - for k, w in enumerate(wallets.values()): - wallet_service = w['wallet'] - ins_full = wallet_service.select_utxos(0, amount) - script = bitcoin.mk_multisig_script(pubs, k) - output_addr = bitcoin.p2sh_scriptaddr(bitcoin.safe_from_hex(script), - magicbyte=196) - txid = make_sign_and_push(ins_full, - wallet_service, - amount, - output_addr=output_addr) - assert txid - -def test_script_to_address(setup_tx_creation): - sample_script = "a914307f099a3bfedec9a09682238db491bade1b467f87" - assert bitcoin.script_to_address( - sample_script, vbyte=5) == "367SYUMqo1Fi4tQsycnmCtB6Ces1Z7EZLH" - assert bitcoin.script_to_address( - sample_script, vbyte=196) == "2MwfecDHsQTm4Gg3RekQdpqAMR15BJrjfRF" - -def test_mktx(setup_tx_creation): - """Testing exceptional conditions; not guaranteed - to create valid tx objects""" - #outpoint structure must be {"outpoint":{"hash":hash, "index": num}} - ins = [{'outpoint': {"hash":x*64, "index":0}, - "script": "", "sequence": 4294967295} for x in ["a", "b", "c"]] - pub = vpubs[0] - addr = bitcoin.pubkey_to_address(pub, magicbyte=get_p2pk_vbyte()) - script = bitcoin.address_to_script(addr) - outs = [script + ":1000", addr+":2000",{"script":script, "value":3000}] - tx = bitcoin.mktx(ins, outs) - print(tx) - #rewrite with invalid output - outs.append({"foo": "bar"}) - with pytest.raises(Exception) as e_info: - tx = bitcoin.mktx(ins, outs) - -def test_bintxhash(setup_tx_creation): - tx = "abcdef1234" - x = bitcoin.bin_txhash(tx) - assert binascii.hexlify(x).decode('ascii') == "121480fc2cccd5103434a9c88b037e08ef6c4f9f95dfb85b56f7043a344613fe" - def test_all_same_priv(setup_tx_creation): #recipient - priv = "aa"*32 + "01" - addr = bitcoin.privkey_to_address(priv, magicbyte=get_p2pk_vbyte()) + priv = b"\xaa"*32 + b"\x01" + pub = bitcoin.privkey_to_pubkey(priv) + addr = str(bitcoin.CCoinAddress.from_scriptPubKey( + bitcoin.CScript([bitcoin.OP_0, bitcoin.Hash160(pub)]))) wallet_service = make_wallets(1, [[1,0,0,0,0]], 1)[0]['wallet'] #make another utxo on the same address addrinwallet = wallet_service.get_addr(0,0,0) @@ -89,52 +42,41 @@ def test_all_same_priv(setup_tx_creation): outs = [{"address": addr, "value": 1000000}] ins = list(insfull.keys()) tx = bitcoin.mktx(ins, outs) - tx = bitcoin.signall(tx, wallet_service.get_key_from_addr(addrinwallet)) - -@pytest.mark.parametrize( - "signall", - [ - (True,), - (False,), - ]) -def test_verify_tx_input(setup_tx_creation, signall): - priv = "aa"*32 + "01" - addr = bitcoin.privkey_to_address(priv, magicbyte=get_p2pk_vbyte()) + scripts = {} + for i, j in enumerate(ins): + scripts[i] = (insfull[j]["script"], insfull[j]["value"]) + success, msg = wallet_service.sign_tx(tx, scripts) + assert success, msg + +def test_verify_tx_input(setup_tx_creation): + priv = b"\xaa"*32 + b"\x01" + pub = bitcoin.privkey_to_pubkey(priv) + script = bitcoin.pubkey_to_p2sh_p2wpkh_script(pub) + addr = str(bitcoin.CCoinAddress.from_scriptPubKey(script)) wallet_service = make_wallets(1, [[2,0,0,0,0]], 1)[0]['wallet'] wallet_service.sync_wallet(fast=True) insfull = wallet_service.select_utxos(0, 110000000) - print(insfull) outs = [{"address": addr, "value": 1000000}] ins = list(insfull.keys()) tx = bitcoin.mktx(ins, outs) - desertx = bitcoin.deserialize(tx) - print(desertx) - if signall: - privdict = {} - for index, ins in enumerate(desertx['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) - ad = insfull[utxo]['address'] - priv = wallet_service.get_key_from_addr(ad) - privdict[utxo] = priv - tx = bitcoin.signall(tx, privdict) - else: - for index, ins in enumerate(desertx['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) - ad = insfull[utxo]['address'] - priv = wallet_service.get_key_from_addr(ad) - tx = bitcoin.sign(tx, index, priv) - desertx2 = bitcoin.deserialize(tx) - print(desertx2) - sig, pub = bitcoin.deserialize_script(desertx2['ins'][0]['script']) - print(sig, pub) - pubscript = bitcoin.address_to_script(bitcoin.pubkey_to_address( - pub, magicbyte=get_p2pk_vbyte())) - sig = binascii.unhexlify(sig) - pub = binascii.unhexlify(pub) - sig_good = bitcoin.verify_tx_input(tx, 0, pubscript, - sig, pub) - assert sig_good - + scripts = {0: (insfull[ins[0]]["script"], bitcoin.coins_to_satoshi(1))} + success, msg = wallet_service.sign_tx(tx, scripts) + assert success, msg + # testing Joinmarket's ability to verify transaction inputs + # of others: pretend we don't have a wallet owning the transaction, + # and instead verify an input using the (sig, pub, scriptCode) data + # that is sent by counterparties: + cScrWit = tx.wit.vtxinwit[0].scriptWitness + sig = cScrWit.stack[0] + pub = cScrWit.stack[1] + scriptSig = tx.vin[0].scriptSig + tx2 = bitcoin.mktx(ins, outs) + res = bitcoin.verify_tx_input(tx2, 0, scriptSig, + bitcoin.pubkey_to_p2sh_p2wpkh_script(pub), + amount = bitcoin.coins_to_satoshi(1), + witness = bitcoin.CScript([sig, pub])) + assert res + def test_absurd_fees(setup_tx_creation): """Test triggering of ValueError exception if the transaction fees calculated from the blockchain @@ -157,7 +99,6 @@ def test_create_sighash_txs(setup_tx_creation): wallet_service.sync_wallet(fast=True) amount = 350000000 ins_full = wallet_service.select_utxos(0, amount) - print("using hashcode: " + str(sighash)) txid = make_sign_and_push(ins_full, wallet_service, amount, hashcode=sighash) assert txid @@ -165,118 +106,39 @@ def test_create_sighash_txs(setup_tx_creation): with pytest.raises(Exception) as e_info: fake_utxos = wallet_service.select_utxos(4, 1000000000) - -def test_spend_p2sh_utxos(setup_tx_creation): - #make a multisig address from 3 privs - privs = [struct.pack(b'B', x) * 32 + b'\x01' for x in range(1, 4)] - pubs = [bitcoin.privkey_to_pubkey(binascii.hexlify(priv).decode('ascii')) for priv in privs] - script = bitcoin.mk_multisig_script(pubs, 2) - msig_addr = bitcoin.p2sh_scriptaddr(script, magicbyte=196) - #pay into it - wallet_service = make_wallets(1, [[2, 0, 0, 0, 1]], 3)[0]['wallet'] - wallet_service.sync_wallet(fast=True) - amount = 350000000 - ins_full = wallet_service.select_utxos(0, amount) - txid = make_sign_and_push(ins_full, wallet_service, amount, output_addr=msig_addr) - assert txid - #wait for mining - time.sleep(1) - #spend out; the input can be constructed from the txid of previous - msig_in = txid + ":0" - ins = [msig_in] - #random output address and change addr - output_addr = wallet_service.get_internal_addr(1) - amount2 = amount - 50000 - outs = [{'value': amount2, 'address': output_addr}] - tx = bitcoin.mktx(ins, outs) - sigs = [] - for priv in privs[:2]: - sigs.append(bitcoin.get_p2sh_signature(tx, 0, script, binascii.hexlify(priv).decode('ascii'))) - tx = bitcoin.apply_multisignatures(tx, 0, script, sigs) - txid = jm_single().bc_interface.pushtx(tx) - assert txid - def test_spend_p2wpkh(setup_tx_creation): #make 3 p2wpkh outputs from 3 privs privs = [struct.pack(b'B', x) * 32 + b'\x01' for x in range(1, 4)] - pubs = [bitcoin.privkey_to_pubkey( - binascii.hexlify(priv).decode('ascii')) for priv in privs] + pubs = [bitcoin.privkey_to_pubkey(priv) for priv in privs] scriptPubKeys = [bitcoin.pubkey_to_p2wpkh_script(pub) for pub in pubs] - addresses = [bitcoin.pubkey_to_p2wpkh_address(pub) for pub in pubs] + addresses = [str(bitcoin.CCoinAddress.from_scriptPubKey( + spk)) for spk in scriptPubKeys] #pay into it wallet_service = make_wallets(1, [[3, 0, 0, 0, 0]], 3)[0]['wallet'] wallet_service.sync_wallet(fast=True) amount = 35000000 p2wpkh_ins = [] - for addr in addresses: + for i, addr in enumerate(addresses): ins_full = wallet_service.select_utxos(0, amount) txid = make_sign_and_push(ins_full, wallet_service, amount, output_addr=addr) assert txid - p2wpkh_ins.append(txid + ":0") + p2wpkh_ins.append((txid, 0)) + txhex = jm_single().bc_interface.get_transaction(txid) #wait for mining - time.sleep(1) + jm_single().bc_interface.tick_forward_chain(1) #random output address output_addr = wallet_service.get_internal_addr(1) amount2 = amount*3 - 50000 outs = [{'value': amount2, 'address': output_addr}] tx = bitcoin.mktx(p2wpkh_ins, outs) - sigs = [] - for i, priv in enumerate(privs): - # sign each of 3 inputs - tx = bitcoin.p2wpkh_sign(tx, i, binascii.hexlify(priv), - amount, native=True) - # check that verify_tx_input correctly validates; - # to do this, we need to extract the signature and get the scriptCode - # of this pubkey - scriptCode = bitcoin.pubkey_to_p2pkh_script(pubs[i]) - witness = bitcoin.deserialize(tx)['ins'][i]['txinwitness'] - assert len(witness) == 2 - assert witness[1] == pubs[i] - sig = witness[0] - assert bitcoin.verify_tx_input(tx, i, scriptPubKeys[i], sig, - pubs[i], scriptCode=scriptCode, amount=amount) - txid = jm_single().bc_interface.pushtx(tx) - assert txid - -def test_spend_p2wsh(setup_tx_creation): - #make 2 x 2 of 2multisig outputs; will need 4 privs - privs = [struct.pack(b'B', x) * 32 + b'\x01' for x in range(1, 5)] - privs = [binascii.hexlify(priv).decode('ascii') for priv in privs] - pubs = [bitcoin.privkey_to_pubkey(priv) for priv in privs] - redeemScripts = [bitcoin.mk_multisig_script(pubs[i:i+2], 2) for i in [0, 2]] - scriptPubKeys = [bitcoin.pubkeys_to_p2wsh_multisig_script(pubs[i:i+2]) for i in [0, 2]] - addresses = [bitcoin.pubkeys_to_p2wsh_multisig_address(pubs[i:i+2]) for i in [0, 2]] - #pay into it - wallet_service = make_wallets(1, [[3, 0, 0, 0, 0]], 3)[0]['wallet'] - wallet_service.sync_wallet(fast=True) - amount = 35000000 - p2wsh_ins = [] - for addr in addresses: - ins_full = wallet_service.select_utxos(0, amount) - txid = make_sign_and_push(ins_full, wallet_service, amount, output_addr=addr) - assert txid - p2wsh_ins.append(txid + ":0") - #wait for mining - time.sleep(1) - #random output address and change addr - output_addr = wallet_service.get_internal_addr(1) - amount2 = amount*2 - 50000 - outs = [{'value': amount2, 'address': output_addr}] - tx = bitcoin.mktx(p2wsh_ins, outs) - sigs = [] - for i in range(2): - sigs = [] - for priv in privs[i*2:i*2+2]: - # sign input j with each of 2 keys - sig = bitcoin.get_p2sh_signature(tx, i, redeemScripts[i], priv, amount=amount) - sigs.append(sig) - # check that verify_tx_input correctly validates; - assert bitcoin.verify_tx_input(tx, i, scriptPubKeys[i], sig, - bitcoin.privkey_to_pubkey(priv), - scriptCode=redeemScripts[i], amount=amount) - tx = bitcoin.apply_p2wsh_multisignatures(tx, i, redeemScripts[i], sigs) - txid = jm_single().bc_interface.pushtx(tx) + for i, priv in enumerate(privs): + # sign each of 3 inputs; note that bitcoin.sign + # automatically validates each signature it creates. + sig, msg = bitcoin.sign(tx, i, priv, amount=amount, native=True) + if not sig: + assert False, msg + txid = jm_single().bc_interface.pushtx(tx.serialize()) assert txid def test_spend_freeze_script(setup_tx_creation): @@ -320,7 +182,6 @@ def test_spend_freeze_script(setup_tx_creation): push_success = jm_single().bc_interface.pushtx(tx) assert push_success == required_success - @pytest.fixture(scope="module") def setup_tx_creation(): load_test_config() diff --git a/jmclient/test/test_valid_addresses.py b/jmclient/test/test_valid_addresses.py index a3d8238a3..fcacc6403 100644 --- a/jmclient/test/test_valid_addresses.py +++ b/jmclient/test/test_valid_addresses.py @@ -1,10 +1,18 @@ from jmclient.configure import validate_address, load_test_config from jmclient import jm_single +import jmbitcoin as btc import json import pytest import os testdir = os.path.dirname(os.path.realpath(__file__)) +def address_valid_somewhere(addr): + for x in ["bitcoin", "bitcoin/testnet", "bitcoin/regtest"]: + btc.select_chain_params(x) + if validate_address(addr)[0]: + return True + return False + def test_non_addresses(setup_addresses): #could flesh this out with other examples res, msg = validate_address(2) @@ -17,9 +25,8 @@ def test_b58_invalid_addresses(setup_addresses): invalid_key_list = json.loads(json_data) for k in invalid_key_list: bad_key = k[0] - res, message = validate_address(bad_key) - assert res == False, "Incorrectly validated address: " + bad_key + " with message: " + message - + res = address_valid_somewhere(bad_key) + assert res == False, "Incorrectly validated address: " + bad_key def test_b58_valid_addresses(): with open(os.path.join(testdir,"base58_keys_valid.json"), "r") as f: @@ -30,30 +37,36 @@ def test_b58_valid_addresses(): if not prop_dict["isPrivkey"]: if prop_dict["isTestnet"]: jm_single().config.set("BLOCKCHAIN", "network", "testnet") + btc.select_chain_params("bitcoin/testnet") else: jm_single().config.set("BLOCKCHAIN", "network", "mainnet") + btc.select_chain_params("bitcoin") #if using pytest -s ; sanity check to see what's actually being tested - print('testing this address: ', addr) res, message = validate_address(addr) assert res == True, "Incorrectly failed to validate address: " + addr + " with message: " + message jm_single().config.set("BLOCKCHAIN", "network", "testnet") + btc.select_chain_params("bitcoin/regtest") def test_valid_bech32_addresses(): valids = ["BC1QW508D6QEJXTDG4Y5R3ZARVARY0C5XW7KV8F3T4", "tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q0sl5k7", - "bc1pw508d6qejxtdg4y5r3zarvary0c5xw7kw508d6qejxtdg4y5r3zarvary0c5xw7k7grplx", - "BC1SW50QA3JX3S", - "bc1zw508d6qejxtdg4y5r3zarvaryvg6kdaj", + # TODO these are valid bech32 addresses but rejected by bitcointx + # because they are not witness version 0; add others. + #"bc1pw508d6qejxtdg4y5r3zarvary0c5xw7kw508d6qejxtdg4y5r3zarvary0c5xw7k7grplx", + #"BC1SW50QA3JX3S", + #"bc1zw508d6qejxtdg4y5r3zarvaryvg6kdaj", "tb1qqqqqp399et2xygdj5xreqhjjvcmzhxw4aywxecjdzew6hylgvsesrxh6hy"] for va in valids: - print("Testing this address: ", va) if va.lower()[:2] == "bc": jm_single().config.set("BLOCKCHAIN", "network", "mainnet") + btc.select_chain_params("bitcoin") else: jm_single().config.set("BLOCKCHAIN", "network", "testnet") + btc.select_chain_params("bitcoin/testnet") res, message = validate_address(va) assert res == True, "Incorrect failed to validate address: " + va + " with message: " + message jm_single().config.set("BLOCKCHAIN", "network", "testnet") + btc.select_chain_params("bitcoin/regtest") def test_invalid_bech32_addresses(): invalids = [ @@ -68,8 +81,7 @@ def test_invalid_bech32_addresses(): "tb1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3pjxtptv", "bc1gmk9yu"] for iva in invalids: - print("Testing this address: ", iva) - res, message = validate_address(iva) + res = address_valid_somewhere(iva) assert res == False, "Incorrectly validated address: " + iva @pytest.fixture(scope="module") diff --git a/jmclient/test/test_wallet.py b/jmclient/test/test_wallet.py index 092694782..e0e0e4a14 100644 --- a/jmclient/test/test_wallet.py +++ b/jmclient/test/test_wallet.py @@ -6,12 +6,14 @@ import pytest import jmbitcoin as btc -from commontest import binarize_tx, ensure_bip65_activated -from jmbase import get_log +from commontest import ensure_bip65_activated +from jmbase import (get_log, utxostr_to_utxo, utxo_to_utxostr, + hextobin, bintohex) from jmclient import load_test_config, jm_single, \ SegwitLegacyWallet,BIP32Wallet, BIP49Wallet, LegacyWallet,\ VolatileStorage, get_network, cryptoengine, WalletError,\ SegwitWallet, WalletService, SegwitLegacyWalletFidelityBonds,\ + BTC_P2PKH, BTC_P2SH_P2WPKH,\ FidelityBondMixin, FidelityBondWatchonlyWallet, wallet_gettimelockaddress from test_blockchaininterface import sync_test_wallet @@ -20,10 +22,7 @@ def signed_tx_is_segwit(tx): - for inp in tx['ins']: - if 'txinwitness' not in inp: - return False - return True + return tx.has_witness() def assert_segwit(tx): @@ -47,10 +46,11 @@ def get_populated_wallet(amount=10**8, num=3): def fund_wallet_addr(wallet, addr, value_btc=1): - txin_id = jm_single().bc_interface.grab_coins(addr, value_btc) + # special case, grab_coins returns hex from rpc: + txin_id = hextobin(jm_single().bc_interface.grab_coins(addr, value_btc)) txinfo = jm_single().bc_interface.get_transaction(txin_id) - txin = btc.deserialize(unhexlify(txinfo['hex'])) - utxo_in = wallet.add_new_utxos_(txin, unhexlify(txin_id), 1) + txin = btc.CMutableTransaction.deserialize(btc.x(txinfo["hex"])) + utxo_in = wallet.add_new_utxos(txin, 1) assert len(utxo_in) == 1 return list(utxo_in.keys())[0] @@ -367,13 +367,15 @@ def test_signing_imported(setup_wallet, wif, keytype, type_check): path = wallet.import_private_key(MIXDEPTH, wif, keytype) utxo = fund_wallet_addr(wallet, wallet.get_address_from_path(path)) # The dummy output is constructed as an unspendable p2sh: - tx = btc.deserialize(btc.mktx(['{}:{}'.format( - hexlify(utxo[0]).decode('ascii'), utxo[1])], - [btc.p2sh_scriptaddr(b"\x00",magicbyte=196) + ':' + str(10**8 - 9000)])) + tx = btc.mktx([utxo], + [{"address": str(btc.CCoinAddress.from_scriptPubKey( + btc.CScript(b"\x00").to_p2sh_scriptPubKey())), + "value": 10**8 - 9000}]) script = wallet.get_script_from_path(path) - tx = wallet.sign_tx(tx, {0: (script, 10**8)}) + success, msg = wallet.sign_tx(tx, {0: (script, 10**8)}) + assert success, msg type_check(tx) - txout = jm_single().bc_interface.pushtx(btc.serialize(tx)) + txout = jm_single().bc_interface.pushtx(tx.serialize()) assert txout @@ -385,17 +387,19 @@ def test_signing_imported(setup_wallet, wif, keytype, type_check): def test_signing_simple(setup_wallet, wallet_cls, type_check): jm_single().config.set('BLOCKCHAIN', 'network', 'testnet') storage = VolatileStorage() - wallet_cls.initialize(storage, get_network()) + wallet_cls.initialize(storage, get_network(), entropy=b"\xaa"*16) wallet = wallet_cls(storage) utxo = fund_wallet_addr(wallet, wallet.get_internal_addr(0)) # The dummy output is constructed as an unspendable p2sh: - tx = btc.deserialize(btc.mktx(['{}:{}'.format( - hexlify(utxo[0]).decode('ascii'), utxo[1])], - [btc.p2sh_scriptaddr(b"\x00",magicbyte=196) + ':' + str(10**8 - 9000)])) + tx = btc.mktx([utxo], + [{"address": str(btc.CCoinAddress.from_scriptPubKey( + btc.CScript(b"\x00").to_p2sh_scriptPubKey())), + "value": 10**8 - 9000}]) script = wallet.get_script(0, 1, 0) - tx = wallet.sign_tx(tx, {0: (script, 10**8)}) + success, msg = wallet.sign_tx(tx, {0: (script, 10**8)}) + assert success, msg type_check(tx) - txout = jm_single().bc_interface.pushtx(btc.serialize(tx)) + txout = jm_single().bc_interface.pushtx(tx.serialize()) assert txout def test_timelocked_output_signing(setup_wallet): @@ -450,18 +454,18 @@ def test_add_utxos(setup_wallet): for md in range(1, wallet.max_mixdepth + 1): assert balances[md] == 0 - utxos = wallet.get_utxos_by_mixdepth_() + utxos = wallet.get_utxos_by_mixdepth() assert len(utxos[0]) == num_tx for md in range(1, wallet.max_mixdepth + 1): assert not utxos[md] with pytest.raises(Exception): # no funds in mixdepth - wallet.select_utxos_(1, amount) + wallet.select_utxos(1, amount) with pytest.raises(Exception): # not enough funds - wallet.select_utxos_(0, amount * (num_tx + 1)) + wallet.select_utxos(0, amount * (num_tx + 1)) wallet.reset_utxos() assert wallet.get_balance_by_mixdepth()[0] == 0 @@ -472,12 +476,12 @@ def test_select_utxos(setup_wallet): amount = 10**8 wallet = get_populated_wallet(amount) - utxos = wallet.select_utxos_(0, amount // 2) + utxos = wallet.select_utxos(0, amount // 2) assert len(utxos) == 1 utxos = list(utxos.keys()) - more_utxos = wallet.select_utxos_(0, int(amount * 1.5), utxo_filter=utxos) + more_utxos = wallet.select_utxos(0, int(amount * 1.5), utxo_filter=utxos) assert len(more_utxos) == 2 assert utxos[0] not in more_utxos @@ -488,14 +492,11 @@ def test_add_new_utxos(setup_wallet): scripts = [wallet.get_new_script(x, True) for x in range(3)] tx_scripts = list(scripts) - tx_scripts.append(b'\x22'*17) - - tx = btc.deserialize(btc.mktx( - ['0'*64 + ':2'], [{'script': hexlify(s).decode('ascii'), 'value': 10**8} - for s in tx_scripts])) - binarize_tx(tx) - txid = b'\x01' * 32 - added = wallet.add_new_utxos_(tx, txid, 1) + tx = btc.mktx( + [(b"\x00"*32, 2)], + [{"address": wallet.script_to_addr(s), + "value": 10**8} for s in tx_scripts]) + added = wallet.add_new_utxos(tx, 1) assert len(added) == len(scripts) added_scripts = {x['script'] for x in added.values()} @@ -517,21 +518,20 @@ def test_remove_old_utxos(setup_wallet): for i in range(3): txin = jm_single().bc_interface.grab_coins( wallet.get_internal_addr(1), 1) - wallet.add_utxo(unhexlify(txin), 0, wallet.get_script(1, 1, i), 10**8, 1) + wallet.add_utxo(btc.x(txin), 0, wallet.get_script(1, 1, i), 10**8, 1) - inputs = wallet.select_utxos_(0, 10**8) - inputs.update(wallet.select_utxos_(1, 2 * 10**8)) + inputs = wallet.select_utxos(0, 10**8) + inputs.update(wallet.select_utxos(1, 2 * 10**8)) assert len(inputs) == 3 tx_inputs = list(inputs.keys()) tx_inputs.append((b'\x12'*32, 6)) - tx = btc.deserialize(btc.mktx( - ['{}:{}'.format(hexlify(txid).decode('ascii'), i) for txid, i in tx_inputs], - ['0' * 36 + ':' + str(3 * 10**8 - 1000)])) - binarize_tx(tx) + tx = btc.mktx(tx_inputs, + [{"address": "2N9gfkUsFW7Kkb1Eurue7NzUxUt7aNJiS1U", + "value": 3 * 10**8 - 1000}]) - removed = wallet.remove_old_utxos_(tx) + removed = wallet.remove_old_utxos(tx) assert len(removed) == len(inputs) for txid in removed: @@ -760,6 +760,14 @@ def test_wallet_mixdepth_decrease(setup_wallet): max_mixdepth = wallet.max_mixdepth assert max_mixdepth >= 1, "bad default value for mixdepth for this test" utxo = fund_wallet_addr(wallet, wallet.get_internal_addr(max_mixdepth), 1) + bci = jm_single().bc_interface + unspent_list = bci.rpc('listunspent', [0]) + # filter on label, but note (a) in certain circumstances (in- + # wallet transfer) it is possible for the utxo to be labeled + # with the external label, and (b) the wallet will know if it + # belongs or not anyway (is_known_addr): + our_unspent_list = [x for x in unspent_list if ( + bci.is_address_labeled(x, wallet.get_wallet_name()))] assert wallet.get_balance_by_mixdepth()[max_mixdepth] == 10**8 wallet.close() storage_data = wallet._storage.file_data @@ -777,7 +785,7 @@ def test_wallet_mixdepth_decrease(setup_wallet): # wallet.select_utxos will still return utxos from higher mixdepths # because we explicitly ask for a specific mixdepth - assert utxo in new_wallet.select_utxos_(max_mixdepth, 10**7) + assert utxo in new_wallet.select_utxos(max_mixdepth, 10**7) def test_watchonly_wallet(setup_wallet): jm_single().config.set('BLOCKCHAIN', 'network', 'testnet') @@ -819,6 +827,7 @@ def test_watchonly_wallet(setup_wallet): @pytest.fixture(scope='module') def setup_wallet(): load_test_config() + btc.select_chain_params("bitcoin/regtest") #see note in cryptoengine.py: cryptoengine.BTC_P2WPKH.VBYTE = 100 jm_single().bc_interface.tick_forward_chain_interval = 2 diff --git a/jmclient/test/test_wallets.py b/jmclient/test/test_wallets.py index 66d60589e..7344ead3e 100644 --- a/jmclient/test/test_wallets.py +++ b/jmclient/test/test_wallets.py @@ -8,7 +8,8 @@ import json import pytest -from jmbase import get_log +from jmbase import get_log, hextobin +import jmbitcoin as btc from jmclient import ( load_test_config, jm_single, estimate_tx_fee, BitcoinCoreInterface, Mnemonic) @@ -43,9 +44,10 @@ def test_query_utxo_set(setup_wallets): txid = do_tx(wallet_service, 90000000) txid2 = do_tx(wallet_service, 20000000) print("Got txs: ", txid, txid2) - res1 = jm_single().bc_interface.query_utxo_set(txid + ":0", includeunconf=True) + res1 = jm_single().bc_interface.query_utxo_set( + (txid, 0), includeunconf=True) res2 = jm_single().bc_interface.query_utxo_set( - [txid + ":0", txid2 + ":1"], + [(txid, 0), (txid2, 1)], includeconf=True, includeunconf=True) assert len(res1) == 1 assert len(res2) == 2 @@ -53,7 +55,7 @@ def test_query_utxo_set(setup_wallets): assert not 'confirms' in res1[0] assert 'confirms' in res2[0] assert 'confirms' in res2[1] - res3 = jm_single().bc_interface.query_utxo_set("ee" * 32 + ":25") + res3 = jm_single().bc_interface.query_utxo_set((b"\xee" * 32, 25)) assert res3 == [None] @@ -69,11 +71,11 @@ def test_wrong_network_bci(setup_wallets): def test_pushtx_errors(setup_wallets): """Ensure pushtx fails return False """ - badtxhex = "aaaa" - assert not jm_single().bc_interface.pushtx(badtxhex) + badtx = b"\xaa\xaa" + assert not jm_single().bc_interface.pushtx(badtx) #Break the authenticated jsonrpc and try again jm_single().bc_interface.jsonRpc.port = 18333 - assert not jm_single().bc_interface.pushtx(t_raw_signed_tx) + assert not jm_single().bc_interface.pushtx(hextobin(t_raw_signed_tx)) #rebuild a valid jsonrpc inside the bci load_test_config() diff --git a/jmclient/test/test_walletservice.py b/jmclient/test/test_walletservice.py index a652a404b..f46238baa 100644 --- a/jmclient/test/test_walletservice.py +++ b/jmclient/test/test_walletservice.py @@ -39,7 +39,6 @@ def test_address_reuse_freezing(setup_walletservice): """ context = {'cb_called': 0} def reuse_callback(utxostr): - print("Address reuse freezing callback on utxo: ", utxostr) context['cb_called'] += 1 # we must fund after initial sync (for imports), hence # "populated" with no coins diff --git a/jmclient/test/test_yieldgenerator.py b/jmclient/test/test_yieldgenerator.py index 2fd253943..332c50b3e 100644 --- a/jmclient/test/test_yieldgenerator.py +++ b/jmclient/test/test_yieldgenerator.py @@ -1,6 +1,6 @@ import unittest - +from jmbitcoin import CMutableTxOut, CMutableTransaction from jmclient import load_test_config, jm_single,\ SegwitLegacyWallet, VolatileStorage, YieldGeneratorBasic, \ get_network, WalletService @@ -28,14 +28,12 @@ def __init__(self, balances): self.add_utxo_at_mixdepth(m, b) def add_utxo_at_mixdepth(self, mixdepth, balance): - tx = {'outs': [{'script': self.get_internal_script(mixdepth), - 'value': balance}]} - # We need to generate a fake "txid" that has to be unique for all the - # UTXOs that are added to the wallet. For that, we simply use the - # script, and make it fit the required length (32 bytes). - txid = tx['outs'][0]['script'] + b'x' * 32 - txid = txid[:32] - self.add_new_utxos_(tx, txid, 1) + txout = CMutableTxOut(balance, self.get_internal_script(mixdepth)) + tx = CMutableTransaction() + tx.vout = [txout] + # (note: earlier requirement that txid be generated uniquely is now + # automatic; tx.GetTxid() functions correctly within the wallet). + self.add_new_utxos(tx, 1) def assert_utxos_from_mixdepth(self, utxos, expected): """Asserts that the list of UTXOs (as returned from UTXO selection diff --git a/jmdaemon/test/test_message_channel.py b/jmdaemon/test/test_message_channel.py index 949e36e03..9ca38915d 100644 --- a/jmdaemon/test/test_message_channel.py +++ b/jmdaemon/test/test_message_channel.py @@ -15,6 +15,7 @@ import struct import traceback import threading +import binascii import jmbitcoin as bitcoin from dummy_mc import DummyMessageChannel @@ -22,10 +23,11 @@ jlog = get_log() def make_valid_nick(i=0): - nick_priv = hashlib.sha256(struct.pack(b'B', i)*16).hexdigest() + '01' - nick_pubkey = bitcoin.privtopub(nick_priv) - nick_pkh_raw = hashlib.sha256(nick_pubkey.encode('ascii')).digest()[:NICK_HASH_LENGTH] - nick_pkh = bitcoin.b58encode(nick_pkh_raw) + nick_priv = hashlib.sha256(struct.pack(b'B', i)*16).digest() + b"\x01" + nick_pubkey = bitcoin.privkey_to_pubkey(nick_priv) + nick_pkh_raw = hashlib.sha256(binascii.hexlify( + nick_pubkey)).digest()[:NICK_HASH_LENGTH] + nick_pkh = bitcoin.base58.encode(nick_pkh_raw) #right pad to maximum possible; b58 is not fixed length. #Use 'O' as one of the 4 not included chars in base58. nick_pkh += 'O' * (NICK_MAX_ENCODED - len(nick_pkh)) diff --git a/scripts/add-utxo.py b/scripts/add-utxo.py index e564df6b2..ea6de4448 100755 --- a/scripts/add-utxo.py +++ b/scripts/add-utxo.py @@ -17,7 +17,7 @@ from jmclient import load_program_config, jm_single, get_p2pk_vbyte,\ open_wallet, WalletService, add_external_commitments, update_commitments,\ PoDLE, get_podle_commitments, get_utxo_info, validate_utxo_data, quit,\ - get_wallet_path, add_base_options + get_wallet_path, add_base_options, BTCEngine, BTC_P2SH_P2WPKH from jmbase.support import EXIT_SUCCESS, EXIT_FAILURE, EXIT_ARGERROR, jmprint @@ -32,9 +32,10 @@ def generate_single_podle_sig(u, priv, i): This calls the underlying 'raw' code based on the class PoDLE, not the library 'generate_podle' which intelligently searches and updates commitments. """ - #Convert priv to hex - hexpriv = btc.from_wif_privkey(priv, vbyte=get_p2pk_vbyte()) - podle = PoDLE(u, hexpriv) + #Convert priv from wif; require P2SH-P2WPKH keys + rawpriv, keytype = BTCEngine.wif_to_privkey(priv) + assert keytype == BTC_P2SH_P2WPKH + podle = PoDLE(u, rawpriv) r = podle.generate_podle(i) return (r['P'], r['P2'], r['sig'], r['e'], r['commit']) diff --git a/scripts/convert_old_wallet.py b/scripts/convert_old_wallet.py index 4a23cc946..1896547ee 100755 --- a/scripts/convert_old_wallet.py +++ b/scripts/convert_old_wallet.py @@ -7,10 +7,9 @@ from collections import defaultdict from pyaes import AESModeOfOperationCBC, Decrypter from jmbase import JM_APP_NAME -from jmclient import Storage, load_program_config +from jmclient import Storage, load_program_config, BTCEngine from jmclient.wallet_utils import get_password, get_wallet_cls,\ cli_get_wallet_passphrase_check, get_wallet_path -from jmbitcoin import wif_compressed_privkey class ConvertException(Exception): @@ -99,7 +98,7 @@ def new_wallet_from_data(data, file_name): for md in data['imported']: for privkey in data['imported'][md]: privkey += b'\x01' - wif = wif_compressed_privkey(hexlify(privkey).decode('ascii')) + wif = BTCEngine.privkey_to_wif(privkey) wallet.import_private_key(md, wif) wallet.save() diff --git a/scripts/joinmarket-qt.py b/scripts/joinmarket-qt.py index d10bdc8ae..12eee3a92 100755 --- a/scripts/joinmarket-qt.py +++ b/scripts/joinmarket-qt.py @@ -75,7 +75,7 @@ wallet_generate_recover_bip39, wallet_display, get_utxos_enabled_disabled,\ NO_ROUNDING, get_max_cj_fee_values, get_default_max_absolute_fee, \ get_default_max_relative_fee, RetryableStorageError, add_base_options, \ - FidelityBondMixin + BTCEngine, BTC_P2SH_P2WPKH, FidelityBondMixin from qtsupport import ScheduleWizard, TumbleRestartWizard, config_tips,\ config_types, QtHandler, XStream, Buttons, OkButton, CancelButton,\ PasswordDialog, MyTreeWidget, JMQtMessageBox, BLUE_FG,\ @@ -1530,14 +1530,18 @@ def exportPrivkeysJson(self): done = False def privkeys_thread(): + # To explain this (given setting was already done in + # load_program_config), see: + # https://github.com/Simplexum/python-bitcointx/blob/9f1fa67a5445f8c187ef31015a4008bc5a048eea/bitcointx/__init__.py#L242-L243 + # note, we ignore the return value as we only want to apply + # the chainparams setting logic: + get_blockchain_interface_instance(jm_single().config) for addr in addresses: time.sleep(0.1) if done: break priv = self.wallet_service.get_key_from_addr(addr) - private_keys[addr] = btc.wif_compressed_privkey( - priv, - vbyte=get_p2pk_vbyte()) + private_keys[addr] = BTCEngine.privkey_to_wif(priv) self.computing_privkeys_signal.emit() self.show_privkeys_signal.emit() @@ -1569,10 +1573,13 @@ def show_privkeys(): privkeys_fn + '.json'), "wb") as f: for addr, pk in private_keys.items(): #sanity check - if not addr == btc.pubkey_to_p2sh_p2wpkh_address( - btc.privkey_to_pubkey( - btc.from_wif_privkey(pk, vbyte=get_p2pk_vbyte()) - ), get_p2sh_vbyte()): + rawpriv, keytype = BTCEngine.wif_to_privkey(pk) + if not keytype == BTC_P2SH_P2WPKH: + JMQtMessageBox(None, "Failed to create privkey export, " + "should be keytype p2sh-p2wpkh but is not.", + mbtype='crit') + return + if not addr == self.wallet_service._ENGINE.privkey_to_address(rawpriv): JMQtMessageBox(None, "Failed to create privkey export -" +\ " critical error in key parsing.", mbtype='crit') @@ -2009,7 +2016,7 @@ def onTabChange(i): jm_single().bc_interface.simulating = True jm_single().maker_timeout_sec = 15 #trigger start with a fake tx - jm_single().bc_interface.pushtx("00"*20) + jm_single().bc_interface.pushtx(b"\x00"*20) #prepare for logging for dname in ['logs', 'wallets', 'cmtdata']: diff --git a/scripts/sendtomany.py b/scripts/sendtomany.py index 23941db0d..33118a08e 100755 --- a/scripts/sendtomany.py +++ b/scripts/sendtomany.py @@ -8,10 +8,10 @@ from pprint import pformat from optparse import OptionParser import jmbitcoin as btc -from jmbase import get_log, jmprint +from jmbase import get_log, jmprint, bintohex, utxostr_to_utxo from jmclient import load_program_config, estimate_tx_fee, jm_single,\ get_p2pk_vbyte, validate_address, get_utxo_info, add_base_options,\ - validate_utxo_data, quit + validate_utxo_data, quit, BTCEngine, BTC_P2SH_P2WPKH, BTC_P2PKH log = get_log() @@ -39,10 +39,12 @@ def sign(utxo, priv, destaddrs, segwit=True): log.info("Using fee: " + str(fee)) for i, addr in enumerate(destaddrs): outs.append({'address': addr, 'value': share}) - unsigned_tx = btc.mktx(ins, outs) + tx = btc.mktx(ins, outs) amtforsign = amt if segwit else None - return btc.sign(unsigned_tx, 0, btc.from_wif_privkey( - priv, vbyte=get_p2pk_vbyte()), amount=amtforsign) + rawpriv, _ = BTCEngine.wif_to_privkey(priv) + success, msg = btc.sign(tx, 0, rawpriv, amount=amtforsign) + assert success, msg + return tx def main(): parser = OptionParser( @@ -106,17 +108,20 @@ def main(): for d in destaddrs: if not validate_address(d): quit(parser, "Address was not valid; wrong network?: " + d) - txsigned = sign(u, priv, destaddrs, segwit = not options.nonsegwit) + success, utxo = utxostr_to_utxo(u) + if not success: + quit(parser, "Failed to load utxo from string: " + utxo) + txsigned = sign(utxo, priv, destaddrs, segwit = not options.nonsegwit) if not txsigned: log.info("Transaction signing operation failed, see debug messages for details.") return - log.debug("Got signed transaction:\n" + txsigned) + log.info("Got signed transaction:\n" + bintohex(txsigned.serialize())) log.debug("Deserialized:") - log.debug(pformat(btc.deserialize(txsigned))) + log.debug(pformat(str(txsigned))) if input('Would you like to push to the network? (y/n):')[0] != 'y': log.info("You chose not to broadcast the transaction, quitting.") return - jm_single().bc_interface.pushtx(txsigned) + jm_single().bc_interface.pushtx(txsigned.serialize()) if __name__ == "__main__": main() diff --git a/test/test_segwit.py b/test/test_segwit.py index a222e5f90..b5cc7a4c7 100644 --- a/test/test_segwit.py +++ b/test/test_segwit.py @@ -7,7 +7,7 @@ from pprint import pformat import jmbitcoin as btc import pytest -from jmbase import get_log +from jmbase import get_log, hextobin, bintohex from jmclient import load_test_config, jm_single, LegacyWallet log = get_log() @@ -19,9 +19,9 @@ def test_segwit_valid_txs(setup_segwit): for j in valid_txs: if len(j) < 2: continue - deserialized_tx = btc.deserialize(str(j[1])) + deserialized_tx = btc.CMutableTransaction.deserialize(hextobin(j[1])) print(pformat(deserialized_tx)) - assert btc.serialize(deserialized_tx) == str(j[1]) + assert deserialized_tx.serialize() == hextobin(j[1]) #TODO use bcinterface to decoderawtransaction #and compare the json values @@ -63,8 +63,8 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount, sw_wallet_service = make_wallets(1, [[len(segwit_ins), 0, 0, 0, 0]], segwit_amt)[0]['wallet'] sw_wallet_service.sync_wallet(fast=True) - nsw_utxos = nsw_wallet_service.get_utxos_by_mixdepth(hexfmt=False)[MIXDEPTH] - sw_utxos = sw_wallet_service.get_utxos_by_mixdepth(hexfmt=False)[MIXDEPTH] + nsw_utxos = nsw_wallet_service.get_utxos_by_mixdepth()[MIXDEPTH] + sw_utxos = sw_wallet_service.get_utxos_by_mixdepth()[MIXDEPTH] assert len(o_ins) <= len(nsw_utxos), "sync failed" assert len(segwit_ins) <= len(sw_utxos), "sync failed" @@ -92,8 +92,8 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount, # FIXME: encoding mess, mktx should accept binary input formats tx_ins = [] - for i, (txid, data) in sorted(all_ins.items(), key=lambda x: x[0]): - tx_ins.append('{}:{}'.format(binascii.hexlify(txid[0]).decode('ascii'), txid[1])) + for i, (txin, data) in sorted(all_ins.items(), key=lambda x: x[0]): + tx_ins.append(txin) # create outputs FEE = 50000 @@ -104,11 +104,11 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount, change_amt = total_amt_in_sat - amount - FEE tx_outs = [ - {'script': binascii.hexlify(cj_script).decode('ascii'), + {'address': nsw_wallet_service.script_to_addr(cj_script), 'value': amount}, - {'script': binascii.hexlify(change_script).decode('ascii'), + {'address': nsw_wallet_service.script_to_addr(change_script), 'value': change_amt}] - tx = btc.deserialize(btc.mktx(tx_ins, tx_outs)) + tx = btc.mktx(tx_ins, tx_outs) # import new addresses to bitcoind jm_single().bc_interface.import_addresses( @@ -120,18 +120,20 @@ def test_spend_p2sh_p2wpkh_multi(setup_segwit, wallet_structure, in_amt, amount, for nsw_in_index in o_ins: inp = nsw_ins[nsw_in_index][1] scripts[nsw_in_index] = (inp['script'], inp['value']) - tx = nsw_wallet_service.sign_tx(tx, scripts) + success, msg = nsw_wallet_service.sign_tx(tx, scripts) + assert success, msg scripts = {} for sw_in_index in segwit_ins: inp = sw_ins[sw_in_index][1] scripts[sw_in_index] = (inp['script'], inp['value']) - tx = sw_wallet_service.sign_tx(tx, scripts) + success, msg = sw_wallet_service.sign_tx(tx, scripts) + assert success, msg print(tx) # push and verify - txid = jm_single().bc_interface.pushtx(btc.serialize(tx)) + txid = jm_single().bc_interface.pushtx(tx.serialize()) assert txid balances = jm_single().bc_interface.get_received_by_addr( From 22ed0e04157d897aa1704a50cdc12d24383488ac Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Tue, 21 Apr 2020 01:06:46 +0100 Subject: [PATCH 02/15] Adds psbt creation and signing support in JM wallet. Subclassed btcointx.CKeyBase to create a privkey type that uses our signing code. These are then used to allow signing of inputs in transactions owned by our wallet. Created a PSBTWalletMixin to perform the above function, and added it to our three wallet types. Wrote a detailed creation, sign and broadcast test for a psbt using utxos from within and outside the wallet. Detailed tests across all 3 wallet types for psbt Tests cover also mixed inputs of different types and owned/unowned. Direct send now exported to be used in tests rather than only script usage, also supports returning a tx object rather than only a txid. --- jmbitcoin/jmbitcoin/__init__.py | 2 + jmbitcoin/jmbitcoin/secp256k1_main.py | 47 +++++++ jmclient/jmclient/__init__.py | 2 +- jmclient/jmclient/taker_utils.py | 13 +- jmclient/jmclient/wallet.py | 118 ++++++++++++++++- jmclient/test/test_psbt_wallet.py | 176 ++++++++++++++++++++++++++ 6 files changed, 349 insertions(+), 9 deletions(-) create mode 100644 jmclient/test/test_psbt_wallet.py diff --git a/jmbitcoin/jmbitcoin/__init__.py b/jmbitcoin/jmbitcoin/__init__.py index 00fc5194f..2a7f48161 100644 --- a/jmbitcoin/jmbitcoin/__init__.py +++ b/jmbitcoin/jmbitcoin/__init__.py @@ -8,8 +8,10 @@ from bitcointx.core import (x, b2x, b2lx, lx, COutPoint, CTxOut, CTxIn, CTxInWitness, CTxWitness, CMutableTransaction, Hash160, coins_to_satoshi, satoshi_to_coins) +from bitcointx.core.key import KeyStore from bitcointx.core.script import (CScript, OP_0, SignatureHash, SIGHASH_ALL, SIGVERSION_WITNESS_V0, CScriptWitness) from bitcointx.wallet import (CBitcoinSecret, P2WPKHBitcoinAddress, CCoinAddress, P2SHCoinAddress) +from bitcointx.core.psbt import PartiallySignedTransaction diff --git a/jmbitcoin/jmbitcoin/secp256k1_main.py b/jmbitcoin/jmbitcoin/secp256k1_main.py index db0e7cbe3..45a5bd6da 100644 --- a/jmbitcoin/jmbitcoin/secp256k1_main.py +++ b/jmbitcoin/jmbitcoin/secp256k1_main.py @@ -9,6 +9,7 @@ from bitcointx import base58 from bitcointx.core import Hash +from bitcointx.core.key import CKeyBase, CPubKey from bitcointx.signmessage import BitcoinMessage #Required only for PoDLE calculation: @@ -215,3 +216,49 @@ def ecdsa_raw_verify(msg, pub, sig, rawmsg=False): except Exception as e: return False return retval + +class JMCKey(bytes, CKeyBase): + """An encapsulated private key. + This subclasses specifically for JM's own signing code. + + Attributes: + + pub - The corresponding CPubKey for this private key + secret_bytes - Secret data, 32 bytes (needed because subclasses may have trailing data) + + is_compressed() - True if compressed + + """ + + def __init__(self, b): + CKeyBase.__init__(self, b, compressed=True) + + def is_compressed(self): + return True + + @property + def secret_bytes(self): + assert isinstance(self, bytes) + return self[:32] + + def sign(self, hash): + assert isinstance(hash, (bytes, bytearray)) + if len(hash) != 32: + raise ValueError('Hash must be exactly 32 bytes long') + # TODO: non default sighash flag. + return ecdsa_raw_sign(hash, self.secret_bytes + b"\x01", rawmsg=True) + + + def verify(self, hash, sig): + return self.pub.verify(hash, sig) + + def verify_nonstrict(self, hash, sig): + return self.pub.verify_nonstrict(hash, sig) + + @classmethod + def from_secret_bytes(cls, secret, compressed=True): + return cls(secret, compressed=compressed) + + @classmethod + def from_bytes(cls, data): + raise NotImplementedError('subclasses must override from_bytes()') diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index 0b73d682e..180e5a11c 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -42,7 +42,7 @@ from .commitment_utils import get_utxo_info, validate_utxo_data, quit from .taker_utils import (tumbler_taker_finished_update, restart_waiter, restart_wait, get_tumble_log, direct_send, - tumbler_filter_orders_callback) + tumbler_filter_orders_callback, direct_send) from .cli_options import (add_base_options, add_common_options, get_tumbler_parser, get_max_cj_fee_values, check_regtest, get_sendpayment_parser, diff --git a/jmclient/jmclient/taker_utils.py b/jmclient/jmclient/taker_utils.py index 4f325442b..dc4a9b894 100644 --- a/jmclient/jmclient/taker_utils.py +++ b/jmclient/jmclient/taker_utils.py @@ -21,7 +21,8 @@ """ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, - accept_callback=None, info_callback=None): + accept_callback=None, info_callback=None, + return_transaction=False): """Send coins directly from one mixdepth to one destination address; does not need IRC. Sweep as for normal sendpayment (set amount=0). If answeryes is True, callback/command line query is not performed. @@ -38,7 +39,8 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, pushed), and returns nothing. This function returns: - The txid if transaction is pushed, False otherwise + The txid if transaction is pushed, False otherwise, + or the full CMutableTransaction if return_transaction is True. """ #Sanity checks assert validate_address(destination)[0] or is_burn_destination(destination) @@ -134,8 +136,8 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, log.info("Using a fee of : " + amount_to_str(fee_est) + ".") if amount != 0: log.info("Using a change value of: " + amount_to_str(changeval) + ".") - tx = make_shuffled_tx(list(utxos.keys()), outs, 2, compute_tx_locktime()) - list(utxos.keys()), outs, 2, tx_locktime), utxos) + tx = make_shuffled_tx(list(utxos.keys()), outs, 2, tx_locktime) + inscripts = {} for i, txinp in enumerate(tx.vin): u = (txinp.prevout.hash[::-1], txinp.prevout.n) @@ -165,7 +167,8 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, successmsg = "Transaction sent: " + txid cb = log.info if not info_callback else info_callback cb(successmsg) - return txid + txinfo = txid if not return_transaction else tx + return txinfo def sign_tx(wallet_service, tx, utxos): diff --git a/jmclient/jmclient/wallet.py b/jmclient/jmclient/wallet.py index 2a62a28c3..a1b80ebbf 100644 --- a/jmclient/jmclient/wallet.py +++ b/jmclient/jmclient/wallet.py @@ -985,6 +985,118 @@ def close(self): def __del__(self): self.close() +class DummyKeyStore(btc.KeyStore): + @classmethod + def from_iterable(cls, iterable, **kwargs): + kstore = cls(**kwargs) + for k in iterable: + kstore.add_key(k) + return kstore + + def add_key(self, k): + if isinstance(k, btc.CKeyBase): + if k.pub.key_id in self._privkeys: + assert self._privkeys[k.pub.key_id] == k + else: + self._privkeys[k.pub.key_id] = k + else: + raise ValueError('object supplied to add_key is of unrecognized type') + +class PSBTWalletMixin(object): + """ + Mixin for BaseWallet to provide BIP174 + functions. + """ + def __init__(self, storage, **kwargs): + super(PSBTWalletMixin, self).__init__(storage, **kwargs) + + @staticmethod + def witness_utxos_to_psbt_utxos(utxos): + """ Given a dict of utxos as returned from select_utxos, + convert them to the format required to populate PSBT inputs, + namely CTxOut. Note that the non-segwit case is different, there + you should provide an entire CMutableTransaction object instead. + """ + return [btc.CMutableTxOut(v["value"], + v["script"]) for _, v in utxos.items()] + + def create_psbt_from_tx(self, tx, spent_outs=None): + """ Given a CMutableTransaction object, which should not currently + contain signatures, we create and return a new PSBT object of type + btc.PartiallySignedTransaction. + Optionally the information about the spent outputs that is stored + in PSBT_IN_NONWITNESS_UTXO, PSBT_IN_WITNESS_UTXO and PSBT_IN_REDEEM_SCRIPT + can also be provided, one item per input, in the tuple (spent_outs). + These objects should be either CMutableTransaction, CTxOut or None, + Note that redeem script information cannot be provided for inputs which + we don't own. + """ + # TODO: verify tx contains no signatures as a sanity check? + new_psbt = btc.PartiallySignedTransaction(unsigned_tx=tx) + if spent_outs is None: + # user has not provided input script information; psbt + # will not yet be usable for signing. + return new_psbt + for i, txinput in enumerate(new_psbt.inputs): + if spent_outs[i] is None: + # as above, will not be signable in this case + continue + if isinstance(spent_outs[i], (btc.CMutableTransaction, btc.CMutableTxOut)): + # note that we trust the caller to choose Tx vs TxOut as according + # to non-witness/witness: + txinput.utxo = spent_outs[i] + else: + assert False, "invalid spent output type passed into PSBT creator" + # we now insert redeemscripts where that is possible and necessary: + for i, txinput in enumerate(new_psbt.inputs): + if isinstance(txinput.utxo, btc.CMutableTxOut): + # witness + if txinput.utxo.scriptPubKey.is_witness_scriptpubkey(): + # nothing needs inserting; the scriptSig is empty. + continue + elif txinput.utxo.scriptPubKey.is_p2sh(): + try: + path = self.script_to_path(txinput.utxo.scriptPubKey) + except AssertionError: + # this happens when an input is provided but it's not in + # this wallet; in this case, we cannot set the redeem script. + continue + privkey, _ = self._get_priv_from_path(path) + txinput.redeem_script = btc.pubkey_to_p2wpkh_script( + btc.privkey_to_pubkey(privkey)) + return new_psbt + + def sign_psbt(self, in_psbt, with_sign_result=False): + """ Given a serialized PSBT in raw binary format, + iterate over the inputs and sign all that we can sign with this wallet. + NB IT IS UP TO CALLERS TO ENSURE THAT THEY ACTUALLY WANT TO SIGN + THIS TRANSACTION! + The above is important especially in coinjoin scenarios. + Return: (psbt, msg) + msg: error message or None + if not `with_sign_result`: + psbt: signed psbt in binary serialization, or None if error. + if `with_sign_result` True: + psbt: (PSBT_SignResult object, psbt (deserialized) object) + """ + try: + new_psbt = btc.PartiallySignedTransaction.from_binary(in_psbt) + except: + return None, "Unable to deserialize the PSBT object, invalid format." + privkeys = [] + for k, v in self._utxos._utxo.items(): + for k2, v2 in v.items(): + privkeys.append(self._get_priv_from_path(v2[0])) + jmckeys = list(btc.JMCKey(x[0][:-1]) for x in privkeys) + new_keystore = DummyKeyStore.from_iterable(jmckeys) + try: + signresult = new_psbt.sign(new_keystore) + except Exception as e: + return None, repr(e) + if not with_sign_result: + return new_psbt.serialize(), None + else: + return (signresult, new_psbt), None class ImportWalletMixin(object): """ @@ -1543,7 +1655,7 @@ def get_details(self, path): return self._get_mixdepth_from_path(path), path[-2], path[-1] -class LegacyWallet(ImportWalletMixin, BIP32Wallet): +class LegacyWallet(ImportWalletMixin, PSBTWalletMixin, BIP32Wallet): TYPE = TYPE_P2PKH _ENGINE = ENGINES[TYPE_P2PKH] @@ -1804,10 +1916,10 @@ class BIP84Wallet(BIP32PurposedWallet): _PURPOSE = 2**31 + 84 _ENGINE = ENGINES[TYPE_P2WPKH] -class SegwitLegacyWallet(ImportWalletMixin, BIP39WalletMixin, BIP49Wallet): +class SegwitLegacyWallet(ImportWalletMixin, BIP39WalletMixin, PSBTWalletMixin, BIP49Wallet): TYPE = TYPE_P2SH_P2WPKH -class SegwitWallet(ImportWalletMixin, BIP39WalletMixin, BIP84Wallet): +class SegwitWallet(ImportWalletMixin, BIP39WalletMixin, PSBTWalletMixin, BIP84Wallet): TYPE = TYPE_P2WPKH class SegwitLegacyWalletFidelityBonds(FidelityBondMixin, SegwitLegacyWallet): diff --git a/jmclient/test/test_psbt_wallet.py b/jmclient/test/test_psbt_wallet.py new file mode 100644 index 000000000..a2c4cd68f --- /dev/null +++ b/jmclient/test/test_psbt_wallet.py @@ -0,0 +1,176 @@ +#! /usr/bin/env python +'''Test of psbt creation, update, signing and finalizing + using the functionality of the PSBT Wallet Mixin. + Note that Joinmarket's PSBT code is a wrapper around + bitcointx.core.psbt, and the basic test vectors for + BIP174 are tested there, not here. + ''' + +import time +import binascii +import struct +import copy +from commontest import make_wallets + +import jmbitcoin as bitcoin +import pytest +from jmbase import get_log, bintohex, hextobin +from jmclient import (load_test_config, jm_single, direct_send, + SegwitLegacyWallet, SegwitWallet, LegacyWallet) + +log = get_log() + +def dummy_accept_callback(tx, destaddr, actual_amount, fee_est): + return True +def dummy_info_callback(msg): + pass + +def test_create_and_sign_psbt_with_legacy(setup_psbt_wallet): + """ The purpose of this test is to check that we can create and + then partially sign a PSBT where we own one input and the other input + is of legacy p2pkh type. + """ + wallet_service = make_wallets(1, [[1,0,0,0,0]], 1)[0]['wallet'] + wallet_service.sync_wallet(fast=True) + utxos = wallet_service.select_utxos(0, bitcoin.coins_to_satoshi(0.5)) + assert len(utxos) == 1 + # create a legacy address and make a payment into it + legacy_addr = bitcoin.CCoinAddress.from_scriptPubKey( + bitcoin.pubkey_to_p2pkh_script( + bitcoin.privkey_to_pubkey(b"\x01"*33))) + tx = direct_send(wallet_service, bitcoin.coins_to_satoshi(0.3), 0, + str(legacy_addr), accept_callback=dummy_accept_callback, + info_callback=dummy_info_callback, + return_transaction=True) + assert tx + # this time we will have one utxo worth <~ 0.7 + my_utxos = wallet_service.select_utxos(0, bitcoin.coins_to_satoshi(0.5)) + assert len(my_utxos) == 1 + # find the outpoint for the legacy address we're spending + n = -1 + for i, t in enumerate(tx.vout): + if bitcoin.CCoinAddress.from_scriptPubKey(t.scriptPubKey) == legacy_addr: + n = i + assert n > -1 + utxos = copy.deepcopy(my_utxos) + utxos[(tx.GetTxid()[::-1], n)] ={"script": legacy_addr.to_scriptPubKey(), + "value": bitcoin.coins_to_satoshi(0.3)} + outs = [{"value": bitcoin.coins_to_satoshi(0.998), + "address": wallet_service.get_addr(0,0,0)}] + tx2 = bitcoin.mktx(list(utxos.keys()), outs) + spent_outs = wallet_service.witness_utxos_to_psbt_utxos(my_utxos) + spent_outs.append(tx) + new_psbt = wallet_service.create_psbt_from_tx(tx2, spent_outs) + signed_psbt_and_signresult, err = wallet_service.sign_psbt( + new_psbt.serialize(), with_sign_result=True) + assert err is None + signresult, signed_psbt = signed_psbt_and_signresult + assert signresult.num_inputs_signed == 1 + assert signresult.num_inputs_final == 1 + assert not signresult.is_final + +@pytest.mark.parametrize('unowned_utxo, wallet_cls', [ + (True, SegwitLegacyWallet), + (False, SegwitLegacyWallet), + (True, SegwitWallet), + (False, SegwitWallet), + (True, LegacyWallet), + (False, LegacyWallet), +]) +def test_create_psbt_and_sign(setup_psbt_wallet, unowned_utxo, wallet_cls): + """ Plan of test: + 1. Create a wallet and source 3 destination addresses. + 2. Make, and confirm, transactions that fund the 3 addrs. + 3. Create a new tx spending 2 of those 3 utxos and spending + another utxo we don't own (extra is optional per `unowned_utxo`). + 4. Create a psbt using the above transaction and corresponding + `spent_outs` field to fill in the redeem script. + 5. Compare resulting PSBT with expected structure. + 6. Use the wallet's sign_psbt method to sign the whole psbt, which + means signing each input we own. + 7. Check that each input is finalized as per expected. Check that the whole + PSBT is or is not finalized as per whether there is an unowned utxo. + 8. In case where whole psbt is finalized, attempt to broadcast the tx. + """ + # steps 1 and 2: + wallet_service = make_wallets(1, [[3,0,0,0,0]], 1, + wallet_cls=wallet_cls)[0]['wallet'] + wallet_service.sync_wallet(fast=True) + utxos = wallet_service.select_utxos(0, bitcoin.coins_to_satoshi(1.5)) + # for legacy wallets, psbt creation requires querying for the spending + # transaction: + if wallet_cls == LegacyWallet: + fulltxs = [] + for utxo, v in utxos.items(): + fulltxs.append(jm_single().bc_interface.get_deser_from_gettransaction( + jm_single().bc_interface.get_transaction(utxo[0]))) + + assert len(utxos) == 2 + u_utxos = {} + if unowned_utxo: + # note: tx creation uses the key only; psbt creation uses the value, + # which can be fake here; we do not intend to attempt to fully + # finalize a psbt with an unowned input. See + # https://github.com/Simplexum/python-bitcointx/issues/30 + # the redeem script creation (which is artificial) will be + # avoided in future. + priv = b"\xaa"*32 + b"\x01" + pub = bitcoin.privkey_to_pubkey(priv) + script = bitcoin.pubkey_to_p2sh_p2wpkh_script(pub) + redeem_script = bitcoin.pubkey_to_p2wpkh_script(pub) + u_utxos[(b"\xaa"*32, 12)] = {"value": 1000, "script": script} + utxos.update(u_utxos) + # outputs aren't interesting for this test (we selected 1.5 but will get 2): + outs = [{"value": bitcoin.coins_to_satoshi(1.999), + "address": wallet_service.get_addr(0,0,0)}] + tx = bitcoin.mktx(list(utxos.keys()), outs) + + if wallet_cls != LegacyWallet: + spent_outs = wallet_service.witness_utxos_to_psbt_utxos(utxos) + else: + spent_outs = fulltxs + # the extra input is segwit: + if unowned_utxo: + spent_outs.extend( + wallet_service.witness_utxos_to_psbt_utxos(u_utxos)) + newpsbt = wallet_service.create_psbt_from_tx(tx, spent_outs) + # see note above + if unowned_utxo: + newpsbt.inputs[-1].redeem_script = redeem_script + print(bintohex(newpsbt.serialize())) + # we cannot compare with a fixed expected result due to wallet randomization, but we can + # check psbt structure: + expected_inputs_length = 3 if unowned_utxo else 2 + assert len(newpsbt.inputs) == expected_inputs_length + assert len(newpsbt.outputs) == 1 + # note: redeem_script field is a CScript which is a bytes instance, + # so checking length is best way to check for existence (comparison + # with None does not work): + if wallet_cls == SegwitLegacyWallet: + assert len(newpsbt.inputs[0].redeem_script) != 0 + assert len(newpsbt.inputs[1].redeem_script) != 0 + if unowned_utxo: + assert newpsbt.inputs[2].redeem_script == redeem_script + + signed_psbt_and_signresult, err = wallet_service.sign_psbt( + newpsbt.serialize(), with_sign_result=True) + assert err is None + signresult, signed_psbt = signed_psbt_and_signresult + expected_signed_inputs = len(utxos) if not unowned_utxo else len(utxos)-1 + assert signresult.num_inputs_signed == expected_signed_inputs + assert signresult.num_inputs_final == expected_signed_inputs + + if not unowned_utxo: + assert signresult.is_final + # only in case all signed do we try to broadcast: + extracted_tx = signed_psbt.extract_transaction().serialize() + assert jm_single().bc_interface.pushtx(extracted_tx) + else: + # transaction extraction must fail for not-fully-signed psbts: + with pytest.raises(ValueError) as e: + extracted_tx = signed_psbt.extract_transaction() + + +@pytest.fixture(scope="module") +def setup_psbt_wallet(): + load_test_config() From f0607812434c139e39276f80a09e7a7dfabff436 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Thu, 23 Apr 2020 16:44:43 +0100 Subject: [PATCH 03/15] Add SNICKER support to wallets. This commit uses the now created PSBTWalletMixin and additionally creates a SNICKERWalletMixin, and adds a SNICKERReceiver object to jmclient. A test of the end to end workflow of create and then co-sign a SNICKER coinjoin as per the draft BIP is in test_snicker. Additional changes: updated python-bitcointx dependency to >=1.0.5 Minor refactoring of callbacks in tests and additional redeem script checks to PSBTWalletMixin.sign_psbt. Note that this work replaces #403 . --- jmbitcoin/jmbitcoin/__init__.py | 1 + jmbitcoin/jmbitcoin/secp256k1_ecies.py | 91 ++++++++ jmbitcoin/jmbitcoin/secp256k1_main.py | 14 +- jmbitcoin/jmbitcoin/snicker.py | 58 +++++ jmbitcoin/setup.py | 4 +- jmbitcoin/test/test_ecdh.py | 60 ++++++ jmbitcoin/test/test_ecies.py | 43 ++++ jmclient/jmclient/__init__.py | 1 + jmclient/jmclient/cryptoengine.py | 2 +- jmclient/jmclient/snicker_receiver.py | 224 ++++++++++++++++++++ jmclient/jmclient/wallet.py | 280 ++++++++++++++++++++++++- jmclient/test/commontest.py | 6 + jmclient/test/test_psbt_wallet.py | 7 +- jmclient/test/test_snicker.py | 118 +++++++++++ 14 files changed, 897 insertions(+), 12 deletions(-) create mode 100644 jmbitcoin/jmbitcoin/secp256k1_ecies.py create mode 100644 jmbitcoin/jmbitcoin/snicker.py create mode 100644 jmbitcoin/test/test_ecdh.py create mode 100644 jmbitcoin/test/test_ecies.py create mode 100644 jmclient/jmclient/snicker_receiver.py create mode 100644 jmclient/test/test_snicker.py diff --git a/jmbitcoin/jmbitcoin/__init__.py b/jmbitcoin/jmbitcoin/__init__.py index 2a7f48161..bfb3f455a 100644 --- a/jmbitcoin/jmbitcoin/__init__.py +++ b/jmbitcoin/jmbitcoin/__init__.py @@ -2,6 +2,7 @@ from jmbitcoin.secp256k1_main import * from jmbitcoin.secp256k1_transaction import * from jmbitcoin.secp256k1_deterministic import * +from jmbitcoin.snicker import * from jmbitcoin.amount import * from jmbitcoin.bip21 import * from bitcointx import select_chain_params diff --git a/jmbitcoin/jmbitcoin/secp256k1_ecies.py b/jmbitcoin/jmbitcoin/secp256k1_ecies.py new file mode 100644 index 000000000..9d928baa7 --- /dev/null +++ b/jmbitcoin/jmbitcoin/secp256k1_ecies.py @@ -0,0 +1,91 @@ +#!/usr/bin/python +from __future__ import (absolute_import, division, + print_function, unicode_literals) +from builtins import * # noqa: F401 +from future.utils import native +import coincurve as secp256k1 +import base64 +import hmac +import hashlib +import pyaes +import os +import jmbitcoin as btc + +ECIES_MAGIC_BYTES = b'BIE1' + +class ECIESDecryptionError(Exception): + pass + +# AES primitives. See BIP-SNICKER for specification. +def aes_encrypt(key, data, iv): + encrypter = pyaes.Encrypter( + pyaes.AESModeOfOperationCBC(key, iv=native(iv))) + enc_data = encrypter.feed(data) + enc_data += encrypter.feed() + + return enc_data + +def aes_decrypt(key, data, iv): + decrypter = pyaes.Decrypter( + pyaes.AESModeOfOperationCBC(key, iv=native(iv))) + try: + dec_data = decrypter.feed(data) + dec_data += decrypter.feed() + except ValueError: + # note decryption errors can come from PKCS7 padding errors + raise ECIESDecryptionError() + return dec_data + +def ecies_encrypt(message, pubkey): + """ Take a privkey in raw byte serialization, + and a pubkey serialized in compressed, binary format (33 bytes), + and output the shared secret as a 32 byte hash digest output. + The exact calculation is: + shared_secret = SHA256(privkey * pubkey) + .. where * is elliptic curve scalar multiplication. + See https://github.com/bitcoin/bitcoin/blob/master/src/secp256k1/src/modules/ecdh/main_impl.h + for implementation details. + """ + # create an ephemeral pubkey for this encryption: + while True: + r = os.urandom(32) + # use compressed serialization of the pubkey R: + try: + R = btc.privkey_to_pubkey(r + b"\x01") + break + except: + # accounts for improbable overflow: + continue + # note that this is *not* ECDH as in the secp256k1_ecdh module, + # since it uses sha512: + ecdh_key = btc.multiply(r, pubkey) + key = hashlib.sha512(ecdh_key).digest() + iv, key_e, key_m = key[0:16], key[16:32], key[32:] + ciphertext = aes_encrypt(key_e, message, iv=iv) + encrypted = ECIES_MAGIC_BYTES + R + ciphertext + mac = hmac.new(key_m, encrypted, hashlib.sha256).digest() + return base64.b64encode(encrypted + mac) + +def ecies_decrypt(privkey, encrypted): + if len(privkey) == 33 and privkey[-1] == 1: + privkey = privkey[:32] + encrypted = base64.b64decode(encrypted) + if len(encrypted) < 85: + raise Exception('invalid ciphertext: length') + magic = encrypted[:4] + if magic != ECIES_MAGIC_BYTES: + raise ECIESDecryptionError() + ephemeral_pubkey = encrypted[4:37] + try: + testR = secp256k1.PublicKey(ephemeral_pubkey) + except: + raise ECIESDecryptionError() + ciphertext = encrypted[37:-32] + mac = encrypted[-32:] + ecdh_key = btc.multiply(privkey, ephemeral_pubkey) + key = hashlib.sha512(ecdh_key).digest() + iv, key_e, key_m = key[0:16], key[16:32], key[32:] + if mac != hmac.new(key_m, encrypted[:-32], hashlib.sha256).digest(): + raise ECIESDecryptionError() + return aes_decrypt(key_e, ciphertext, iv=iv) + diff --git a/jmbitcoin/jmbitcoin/secp256k1_main.py b/jmbitcoin/jmbitcoin/secp256k1_main.py index 45a5bd6da..197809fe6 100644 --- a/jmbitcoin/jmbitcoin/secp256k1_main.py +++ b/jmbitcoin/jmbitcoin/secp256k1_main.py @@ -8,7 +8,7 @@ import coincurve as secp256k1 from bitcointx import base58 -from bitcointx.core import Hash +from bitcointx.core import Hash, CBitcoinTransaction from bitcointx.core.key import CKeyBase, CPubKey from bitcointx.signmessage import BitcoinMessage @@ -166,6 +166,18 @@ def add_privkeys(priv1, priv2): res += b'\x01' return res +def ecdh(privkey, pubkey): + """ Take a privkey in raw byte serialization, + and a pubkey serialized in compressed, binary format (33 bytes), + and output the shared secret as a 32 byte hash digest output. + The exact calculation is: + shared_secret = SHA256(privkey * pubkey) + .. where * is elliptic curve scalar multiplication. + See https://github.com/bitcoin/bitcoin/blob/master/src/secp256k1/src/modules/ecdh/main_impl.h + for implementation details. + """ + secp_privkey = secp256k1.PrivateKey(privkey) + return secp_privkey.ecdh(pubkey) def ecdsa_raw_sign(msg, priv, diff --git a/jmbitcoin/jmbitcoin/snicker.py b/jmbitcoin/jmbitcoin/snicker.py new file mode 100644 index 000000000..5c6be9493 --- /dev/null +++ b/jmbitcoin/jmbitcoin/snicker.py @@ -0,0 +1,58 @@ +from __future__ import (absolute_import, division, + print_function, unicode_literals) +from builtins import * # noqa: F401 + +# Implementation of proposal as per +# https://gist.github.com/AdamISZ/2c13fb5819bd469ca318156e2cf25d79 +# (BIP SNICKER) +# TODO: BIP69 is removed in this implementation, will update BIP draft. + +from jmbitcoin.secp256k1_ecies import * +from jmbitcoin.secp256k1_main import * +from jmbitcoin.secp256k1_transaction import * + +SNICKER_MAGIC_BYTES = b'SNICKER' + +# Flags may be added in future versions +SNICKER_FLAG_NONE = b"\x00" + +def snicker_pubkey_tweak(pub, tweak): + """ use secp256k1 library to perform tweak. + Both `pub` and `tweak` are expected as byte strings + (33 and 32 bytes respectively). + Return value is also a 33 byte string serialization + of the resulting pubkey (compressed). + """ + base_pub = secp256k1.PublicKey(pub) + return base_pub.add(tweak).format() + +def snicker_privkey_tweak(priv, tweak): + """ use secp256k1 library to perform tweak. + Both `priv` and `tweak` are expected as byte strings + (32 or 33 and 32 bytes respectively). + Return value isa 33 byte string serialization + of the resulting private key/secret (with compression flag). + """ + if len(priv) == 33 and priv[-1] == 1: + priv = priv[:-1] + base_priv = secp256k1.PrivateKey(priv) + return base_priv.add(tweak).secret + b'\x01' + +def verify_snicker_output(tx, pub, tweak, spk_type='p2sh-p2wpkh'): + """ A convenience function to check that one output address in a transaction + is a SNICKER-type tweak of an existing key. Returns the index of the output + for which this is True (and there must be only 1), and the derived spk, + or -1 and None if it is not found exactly once. + TODO Add support for other scriptPubKey types. + """ + assert isinstance(tx, btc.CBitcoinTransaction) + expected_destination_pub = snicker_pubkey_tweak(pub, tweak) + expected_destination_spk = pubkey_to_p2sh_p2wpkh_script(expected_destination_pub) + found = 0 + for i, o in enumerate(tx.vout): + if o.scriptPubKey == expected_destination_spk: + found += 1 + found_index = i + if found != 1: + return -1, None + return found_index, expected_destination_spk diff --git a/jmbitcoin/setup.py b/jmbitcoin/setup.py index 018025f3a..75ab2c8c7 100644 --- a/jmbitcoin/setup.py +++ b/jmbitcoin/setup.py @@ -9,6 +9,6 @@ author_email='', license='GPL', packages=['jmbitcoin'], - install_requires=['future', 'coincurve', 'python-bitcointx', 'urldecode'], - python_requires='>=3.5', + install_requires=['future', 'coincurve', 'urldecode', + 'python-bitcointx>=1.0.5', 'pyaes'], zip_safe=False) diff --git a/jmbitcoin/test/test_ecdh.py b/jmbitcoin/test/test_ecdh.py new file mode 100644 index 000000000..c417c462b --- /dev/null +++ b/jmbitcoin/test/test_ecdh.py @@ -0,0 +1,60 @@ +#! /usr/bin/env python +from __future__ import (absolute_import, division, + print_function, unicode_literals) +from builtins import * # noqa: F401 +'''Tests coincurve binding to libsecp256k1 ecdh module code''' + +import hashlib +import jmbitcoin as btc +from jmbase import hextobin +import pytest +import os +import json +testdir = os.path.dirname(os.path.realpath(__file__)) + +def test_ecdh(): + """Using private key test vectors from Bitcoin Core. + 1. Import a set of private keys from the json file. + 2. Calculate the corresponding public keys. + 3. Do ECDH on the cartesian product (x, Y), with x private + and Y public keys, for all combinations. + 4. Compare the result from CoinCurve with the manual + multiplication xY following by hash (sha256). Note that + sha256(xY) is the default hashing function used for ECDH + in libsecp256k1. + + Since there are about 20 private keys in the json file, this + creates around 400 test cases (note xX is still valid). + """ + with open(os.path.join(testdir,"base58_keys_valid.json"), "r") as f: + json_data = f.read() + valid_keys_list = json.loads(json_data) + extracted_privkeys = [] + for a in valid_keys_list: + key, hex_key, prop_dict = a + if prop_dict["isPrivkey"]: + c, k = btc.read_privkey(hextobin(hex_key)) + extracted_privkeys.append(k) + extracted_pubkeys = [btc.privkey_to_pubkey(x) for x in extracted_privkeys] + for p in extracted_privkeys: + for P in extracted_pubkeys: + c, k = btc.read_privkey(p) + shared_secret = btc.ecdh(k, P) + assert len(shared_secret) == 32 + # try recreating the shared secret manually: + pre_secret = btc.multiply(p, P) + derived_secret = hashlib.sha256(pre_secret).digest() + assert derived_secret == shared_secret + + # test some important failure cases; null key, overflow case + privkeys_invalid = [b'\x00'*32, hextobin( + 'fffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141')] + for p in privkeys_invalid: + with pytest.raises(Exception) as e_info: + shared_secret = btc.ecdh(p, extracted_pubkeys[0]) + pubkeys_invalid = [b'0xff' + extracted_pubkeys[0][1:], b'0x00'*12] + for p in extracted_privkeys: + with pytest.raises(Exception) as e_info: + shared_secret = btc.ecdh(p, pubkeys_invalid[0]) + with pytest.raises(Exception) as e_info: + shared_secret = btc.ecdh(p, pubkeys_invalid[1]) diff --git a/jmbitcoin/test/test_ecies.py b/jmbitcoin/test/test_ecies.py new file mode 100644 index 000000000..4a529c5c3 --- /dev/null +++ b/jmbitcoin/test/test_ecies.py @@ -0,0 +1,43 @@ +#! /usr/bin/env python +from __future__ import (absolute_import, division, + print_function, unicode_literals) +from builtins import * # noqa: F401 +'''Tests ECIES implementation as defined in BIP-SNICKER +(and will be updated if that is).''' + +from jmbase import hextobin +import jmbitcoin as btc +import base64 +import os +import json +testdir = os.path.dirname(os.path.realpath(__file__)) + +def test_ecies(): + """Using private key test vectors from Bitcoin Core. + 1. Import a set of private keys from the json file. + 2. Calculate the corresponding public keys. + 3. Do ECDH on the cartesian product (x, Y), with x private + and Y public keys, for all combinations. + 4. Compare the result from CoinCurve with the manual + multiplication xY following by hash (sha256). Note that + sha256(xY) is the default hashing function used for ECDH + in libsecp256k1. + + Since there are about 20 private keys in the json file, this + creates around 400 test cases (note xX is still valid). + """ + with open(os.path.join(testdir,"base58_keys_valid.json"), "r") as f: + json_data = f.read() + valid_keys_list = json.loads(json_data) + print("got valid keys list") + extracted_privkeys = [] + for a in valid_keys_list: + key, hex_key, prop_dict = a + if prop_dict["isPrivkey"]: + c, k = btc.read_privkey(hextobin(hex_key)) + + extracted_privkeys.append(k) + extracted_pubkeys = [btc.privkey_to_pubkey(x) for x in extracted_privkeys] + for (priv, pub) in zip(extracted_privkeys, extracted_pubkeys): + test_message = base64.b64encode(os.urandom(15)*20) + assert btc.ecies_decrypt(priv, btc.ecies_encrypt(test_message, pub)) == test_message diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index 180e5a11c..f088b9455 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -55,6 +55,7 @@ from .wallet_service import WalletService from .maker import Maker, P2EPMaker from .yieldgenerator import YieldGenerator, YieldGeneratorBasic, ygmain +from .snicker_receiver import SNICKERError, SNICKERReceiver # Set default logging handler to avoid "No handler found" warnings. diff --git a/jmclient/jmclient/cryptoengine.py b/jmclient/jmclient/cryptoengine.py index c24d1085c..d713191f8 100644 --- a/jmclient/jmclient/cryptoengine.py +++ b/jmclient/jmclient/cryptoengine.py @@ -191,7 +191,7 @@ def privkey_to_address(cls, privkey): @classmethod def pubkey_to_address(cls, pubkey): script = cls.pubkey_to_script(pubkey) - return str(btc.script_to_address(script, cls.VBYTE)) + return str(btc.CCoinAddress.from_scriptPubKey(script)) @classmethod def pubkey_has_address(cls, pubkey, addr): diff --git a/jmclient/jmclient/snicker_receiver.py b/jmclient/jmclient/snicker_receiver.py new file mode 100644 index 000000000..8d9106708 --- /dev/null +++ b/jmclient/jmclient/snicker_receiver.py @@ -0,0 +1,224 @@ +#! /usr/bin/env python +from __future__ import (absolute_import, division, + print_function, unicode_literals) +from builtins import * # noqa: F401 + +import sys +import binascii + +import jmbitcoin as btc +from jmclient.configure import get_p2pk_vbyte, jm_single +from jmbase import (get_log, EXIT_FAILURE, utxo_to_utxostr, + bintohex, hextobin) + +jlog = get_log() + +class SNICKERError(Exception): + pass + +class SNICKERReceiver(object): + supported_flags = [] + import_branch = 0 + # TODO implement http api or similar + # for polling, here just a file: + proposals_source = "proposals.txt" + + def __init__(self, wallet_service, income_threshold=0, + acceptance_callback=None): + """ + Class to manage processing of SNICKER proposals and + co-signs and broadcasts in case the application level + configuration permits. + + `acceptance_callback`, if specified, must have arguments + and return type as for the default_acceptance_callback + in this class. + """ + + # This is a Joinmarket WalletService object. + self.wallet_service = wallet_service + + # The simplest filter on accepting SNICKER joins: + # that they pay a minimum of this value in satoshis, + # which can be negative (to account for fees). + # TODO this will be a config variable. + self.income_threshold = income_threshold + + # The acceptance callback which defines if we accept + # a valid proposal and sign it, or not. + if acceptance_callback is None: + self.acceptance_callback = self.default_acceptance_callback + else: + self.acceptance_callback = acceptance_callback + + # A list of currently viable key candidates; these must + # all be (pub)keys for which the privkey is accessible, + # i.e. they must be in-wallet keys. + # This list will be continuously updated by polling the + # wallet. + self.candidate_keys = [] + + # A list of already processed proposals + self.processed_proposals = [] + + # maintain a list of all successfully broadcast + # SNICKER transactions in the current run. + self.successful_txs = [] + + def poll_for_proposals(self): + """ Intended to be invoked in a LoopingCall or other + event loop. + Retrieves any entries in the proposals_source, then + compares with existing, + and invokes parse_proposal on all new entries. + # TODO considerable thought should go into how to store + proposals cross-runs, and also handling of keys, which + must be optional. + """ + new_proposals = [] + with open(self.proposals_source, "rb") as f: + current_entries = f.readlines() + for entry in current_entries: + if entry in self.processed_proposals: + continue + new_proposals.append(entry) + if not self.process_proposals(new_proposals): + jlog.error("Critical logic error, shutting down.") + sys.exit(EXIT_FAILURE) + self.processed_proposals.extend(new_proposals) + + def default_acceptance_callback(self, our_ins, their_ins, + our_outs, their_outs): + """ Accepts lists of inputs as CTXIns, + a single output (belonging to us) as a CTxOut, + and a list of other outputs (belonging not to us) in same + format, and must return only True or False representing acceptance. + + Note that this code is relying on the calling function to give + accurate information about the outputs. + """ + # we must find the utxo in our wallet to get its amount. + # this serves as a sanity check that the input is indeed + # ours. + # we use get_all* because for these purposes mixdepth + # is irrelevant. + utxos = self.wallet_service.get_all_utxos() + print("gau returned these utxos: ", utxos) + our_in_amts = [] + our_out_amts = [] + for i in our_ins: + utxo_for_i = (i.prevout.hash[::-1], i.prevout.n) + if utxo_for_i not in utxos.keys(): + success, utxostr =utxo_to_utxostr(utxo_for_i) + if not success: + jlog.error("Code error: input utxo in wrong format.") + jlog.debug("The input utxo was not found: " + utxostr) + return False + our_in_amts.append(utxos[utxo_for_i]["value"]) + for o in our_outs: + our_out_amts.append(o.nValue) + if sum(our_out_amts) - sum(our_in_amts) < self.income_threshold: + return False + return True + + def process_proposals(self, proposals): + """ Each entry in `proposals` is of form: + encrypted_proposal - base64 string + key - hex encoded compressed pubkey, or '' + if the key is not null, we attempt to decrypt and + process according to that key, else cycles over all keys. + + If all SNICKER validations succeed, the decision to spend is + entirely dependent on self.acceptance_callback. + If the callback returns True, we co-sign and broadcast the + transaction and also update the wallet with the new + imported key (TODO: future versions will enable searching + for these keys using history + HD tree; note the jmbitcoin + snicker.py module DOES insist on ECDH being correctly used, + so this will always be possible for transactions created here. + + Returned is a list of txids of any transactions which + were broadcast, unless a critical error occurs, in which case + False is returned (to minimize this function's trust in other + parts of the code being executed, if something appears to be + inconsistent, we trigger immediate halt with this return). + """ + + for kp in proposals: + try: + p, k = kp.split(b',') + except: + jlog.error("Invalid proposal string, ignoring: " + kp) + if k is not None: + # note that this operation will succeed as long as + # the key is in the wallet._script_map, which will + # be true if the key is at an HD index lower than + # the current wallet.index_cache + k = hextobin(k.decode('utf-8')) + addr = self.wallet_service.pubkey_to_addr(k) + if not self.wallet_service.is_known_addr(addr): + jlog.debug("Key not recognized as part of our " + "wallet, ignoring.") + continue + # TODO: interface/API of SNICKERWalletMixin would better take + # address as argument here, not privkey: + priv = self.wallet_service.get_key_from_addr(addr) + result = self.wallet_service.parse_proposal_to_signed_tx( + priv, p, self.acceptance_callback) + if result[0] is not None: + tx, tweak, out_spk = result + + # We will: rederive the key as a sanity check, + # and see if it matches the claimed spk. + # Then, we import the key into the wallet + # (even though it's re-derivable from history, this + # is the easiest for a first implementation). + # Finally, we co-sign, then push. + # (Again, simplest function: checks already passed, + # so do it automatically). + # TODO: the more sophisticated actions. + tweaked_key = btc.snicker_pubkey_tweak(k, tweak) + tweaked_spk = btc.pubkey_to_p2sh_p2wpkh_script(tweaked_key) + if not tweaked_spk == out_spk: + jlog.error("The spk derived from the pubkey does " + "not match the scriptPubkey returned from " + "the snicker module - code error.") + return False + # before import, we should derive the tweaked *private* key + # from the tweak, also: + tweaked_privkey = btc.snicker_privkey_tweak(priv, tweak) + if not btc.privkey_to_pubkey(tweaked_privkey) == tweaked_key: + jlog.error("Was not able to recover tweaked pubkey " + "from tweaked privkey - code error.") + jlog.error("Expected: " + bintohex(tweaked_key)) + jlog.error("Got: " + bintohex(btc.privkey_to_pubkey( + tweaked_privkey))) + return False + # the recreated private key matches, so we import to the wallet, + # note that type = None here is because we use the same + # scriptPubKey type as the wallet, this has been implicitly + # checked above by deriving the scriptPubKey. + self.wallet_service.import_private_key(self.import_branch, + self.wallet_service._ENGINE.privkey_to_wif(tweaked_privkey), + key_type=self.wallet_service.TYPE) + + + # TODO condition on automatic brdcst or not + if not jm_single().bc_interface.pushtx(tx.serialize()): + jlog.error("Failed to broadcast SNICKER CJ.") + return False + self.successful_txs.append(tx) + return True + else: + jlog.debug('Failed to parse proposal: ' + result[1]) + continue + else: + # Some extra work to implement checking all possible + # keys. + raise NotImplementedError() + + # Completed processing all proposals without any logic + # errors (whether the proposals were valid or accepted + # or not). + return True + diff --git a/jmclient/jmclient/wallet.py b/jmclient/jmclient/wallet.py index a1b80ebbf..2a9dc1e73 100644 --- a/jmclient/jmclient/wallet.py +++ b/jmclient/jmclient/wallet.py @@ -5,6 +5,7 @@ import collections import numbers import random +import base64 from binascii import hexlify, unhexlify from datetime import datetime from calendar import timegm @@ -751,6 +752,18 @@ def get_utxos_by_mixdepth(self, include_disabled=False, includeheight=False): script_utxos[md][utxo]['height'] = height return script_utxos + + def get_all_utxos(self, include_disabled=False): + """ Get all utxos in the wallet, format of return + is as for get_utxos_by_mixdepth for each mixdepth. + """ + mix_utxos = self.get_utxos_by_mixdepth( + include_disabled=include_disabled) + all_utxos = {} + for d in mix_utxos.values(): + all_utxos.update(d) + return all_utxos + @classmethod def _get_merge_algorithm(cls, algorithm_name=None): if not algorithm_name: @@ -1089,6 +1102,31 @@ def sign_psbt(self, in_psbt, with_sign_result=False): privkeys.append(self._get_priv_from_path(v2[0])) jmckeys = list(btc.JMCKey(x[0][:-1]) for x in privkeys) new_keystore = DummyKeyStore.from_iterable(jmckeys) + + # for p2sh inputs that we want to sign, the redeem_script + # field must be populated by us, as the counterparty did not + # know it. If this was set in an earlier create-psbt role, + # then overwriting it is harmless (preimage resistance). + if isinstance(self, SegwitLegacyWallet): + for i, txinput in enumerate(new_psbt.inputs): + tu = txinput.utxo + if isinstance(tu, btc.CTxOut): + # witness + if tu.scriptPubKey.is_witness_scriptpubkey(): + # native segwit; no insertion needed. + continue + elif tu.scriptPubKey.is_p2sh(): + try: + path = self.script_to_path(tu.scriptPubKey) + except AssertionError: + # this happens when an input is provided but it's not in + # this wallet; in this case, we cannot set the redeem script. + continue + privkey, _ = self._get_priv_from_path(path) + txinput.redeem_script = btc.pubkey_to_p2wpkh_script( + btc.privkey_to_pubkey(privkey)) + # no else branch; any other form of scriptPubKey will just be + # ignored. try: signresult = new_psbt.sign(new_keystore) except Exception as e: @@ -1098,6 +1136,244 @@ def sign_psbt(self, in_psbt, with_sign_result=False): else: return (signresult, new_psbt), None +class SNICKERWalletMixin(object): + + SUPPORTED_SNICKER_VERSIONS = bytes([0, 1]) + + def __init__(self, storage, **kwargs): + super(SNICKERWalletMixin, self).__init__(storage, **kwargs) + + def create_snicker_proposal(self, our_input, their_input, our_input_utxo, + their_input_utxo, net_transfer, network_fee, + our_priv, their_pub, our_spk, change_spk, + encrypted=True, version_byte=1): + """ Creates a SNICKER proposal from the given transaction data. + This only applies to existing specification, i.e. SNICKER v 00 or 01. + This is only to be used for Joinmarket and only segwit wallets. + `our_input`, `their_input` - utxo format used in JM wallets, + keyed by (tixd, n), as dicts (currently of single entry). + `our_input_utxo`, `their..` - type CTxOut (contains value, scriptPubKey) + net_transfer - amount, after bitcoin transaction fee, transferred from + Proposer (our) to Receiver (their). May be negative. + network_fee - total bitcoin network transaction fee to be paid (so estimates + must occur before this function). + `our_priv`, `their_pub` - these are the keys to be used in ECDH to derive + the tweak as per the BIP. Note `their_pub` may or may not be associated with + the input of the receiver, so is specified here separately. Note also that + according to the BIP the privkey we use *must* be the one corresponding to + the input we provided, else (properly coded) Receivers will reject our + proposal. + `our_spk` - a scriptPubKey for the Proposer coinjoin output + `change_spk` - a change scriptPubkey for the proposer as per BIP + `encrypted` - whether or not to return the ECIES encrypted version of the + proposal. + `version_byte` - which of currently specified Snicker versions is being + used, (0 for reused address, 1 for inferred key). + returns: + if encrypted is True: + base 64 encoded encrypted transaction proposal as a string + else: + binary serialized plaintext SNICKER message. + """ + assert isinstance(self, PSBTWalletMixin) + # before constructing the bitcoin transaction we must calculate the output + # amounts + # TODO investigate arithmetic for negative transfer + if our_input_utxo.nValue - their_input_utxo.nValue - network_fee <= 0: + raise Exception( + "Cannot create SNICKER proposal, Proposer input too small") + total_input_amount = our_input_utxo.nValue + their_input_utxo.nValue + total_output_amount = total_input_amount - network_fee + receiver_output_amount = their_input_utxo.nValue + net_transfer + proposer_output_amount = total_output_amount - receiver_output_amount + + # we must also use ecdh to calculate the output scriptpubkey for the + # receiver + # First, check that `our_priv` corresponds to scriptPubKey in + # `our_input_utxo` to prevent callers from making useless proposals. + expected_pub = btc.privkey_to_pubkey(our_priv) + expected_spk = self.pubkey_to_script(expected_pub) + assert our_input_utxo.scriptPubKey == expected_spk + # now we create the ecdh based tweak: + tweak_bytes = btc.ecdh(our_priv[:-1], their_pub) + tweaked_pub = btc.snicker_pubkey_tweak(their_pub, tweak_bytes) + # TODO: remove restriction to one scriptpubkey type + tweaked_spk = btc.pubkey_to_p2sh_p2wpkh_script(tweaked_pub) + tweaked_addr, our_addr, change_addr = [str( + btc.CCoinAddress.from_scriptPubKey(x)) for x in ( + tweaked_spk, expected_spk, change_spk)] + # now we must construct the three outputs with correct output amounts. + outputs = [{"address": tweaked_addr, "value": receiver_output_amount}] + outputs.append({"address": our_addr, "value": receiver_output_amount}) + outputs.append({"address": change_addr, + "value": total_output_amount - 2 * receiver_output_amount}) + assert all([x["value"] > 0 for x in outputs]) + + # version and locktime as currently specified in the BIP + # for 0/1 version SNICKER. + tx = btc.make_shuffled_tx([our_input, their_input], outputs, + version=2, locktime=0) + # we need to know which randomized input is ours: + our_index = -1 + for i, inp in enumerate(tx.vin): + if our_input == (inp.prevout.hash[::-1], inp.prevout.n): + our_index = i + assert our_index in [0, 1], "code error: our input not in tx" + spent_outs = [our_input_utxo, their_input_utxo] + if our_index == 1: + spent_outs = spent_outs[::-1] + # create the psbt and then sign our input. + snicker_psbt = self.create_psbt_from_tx(tx, spent_outs=spent_outs) + + # having created the PSBT, sign our input + # TODO this requires bitcointx updated minor version else fails + signed_psbt_and_signresult, err = self.sign_psbt( + snicker_psbt.serialize(), with_sign_result=True) + assert err is None + signresult, partially_signed_psbt = signed_psbt_and_signresult + assert signresult.num_inputs_signed == 1 + assert signresult.num_inputs_final == 1 + assert not signresult.is_final + snicker_serialized_message = btc.SNICKER_MAGIC_BYTES + bytes( + [version_byte]) + btc.SNICKER_FLAG_NONE + tweak_bytes + \ + partially_signed_psbt.serialize() + + if not encrypted: + return snicker_serialized_message + + # encryption has been requested; + # we apply ECIES in the form given by the BIP. + return btc.ecies_encrypt(snicker_serialized_message, their_pub) + + def parse_proposal_to_signed_tx(self, privkey, proposal, + acceptance_callback): + """ Given a candidate privkey (binary and compressed format), + and a candidate encrypted SNICKER proposal, attempt to decrypt + and validate it in all aspects. If validation fails the first + return value is None and the second is the reason as a string. + + If all validation checks pass, the next step is checking + acceptance according to financial rules: the acceptance + callback must be a function that accepts four arguments: + (our_ins, their_ins, our_outs, their_outs), where *ins values + are lists of CTxIns and *outs are lists of CTxOuts, + and must return only True/False where True means that the + transaction should be signed. + + If True is returned from the callback, the following are returned + from this function: + (raw transaction for broadcasting (serialized), + tweak value as bytes, derived output spk belonging to receiver) + + Note: flags is currently always None as version is only 0 or 1. + """ + assert isinstance(self, PSBTWalletMixin) + + our_pub = btc.privkey_to_pubkey(privkey) + + if len(proposal) < 5: + return None, "Invalid proposal, too short." + + if base64.b64decode(proposal)[:4] == btc.ECIES_MAGIC_BYTES: + # attempt decryption and reject if fails: + try: + snicker_message = btc.ecies_decrypt(privkey, proposal) + except Exception as e: + return None, "Failed to decrypt." + repr(e) + else: + snicker_message = proposal + + # magic + version,flag + tweak + psbt: + # TODO replace '20' with the minimum feasible PSBT. + if len(snicker_message) < 7 + 2 + 32 + 20: + return None, "Invalid proposal, too short." + + if snicker_message[:7] != btc.SNICKER_MAGIC_BYTES: + return None, "Invalid SNICKER magic bytes." + + version_byte = bytes([snicker_message[7]]) + flag_byte = bytes([snicker_message[8]]) + if version_byte not in self.SUPPORTED_SNICKER_VERSIONS: + return None, "Unrecognized SNICKER version: " + version_byte + if flag_byte != btc.SNICKER_FLAG_NONE: + return None, "Invalid flag byte for version 0,1: " + flag_byte + + tweak_bytes = snicker_message[9:41] + candidate_psbt_serialized = snicker_message[41:] + # attempt to validate the PSBT's format: + try: + cpsbt = btc.PartiallySignedTransaction.from_base64_or_binary( + candidate_psbt_serialized) + except: + return None, "Invalid PSBT format." + + utx = cpsbt.unsigned_tx + # validate that it contains one signature, and two inputs. + # else the proposal is invalid. To achieve this, we call + # PartiallySignedTransaction.sign() with an empty KeyStore, + # which populates the 'is_signed' info fields for us. Note that + # we do not use the PSBTWalletMixin.sign_psbt() which automatically + # signs with our keys. + if not len(utx.vin) == 2: + return None, "PSBT proposal does not contain 2 inputs." + testsignresult = cpsbt.sign(btc.KeyStore(), finalize=False) + print("got sign result: ", testsignresult) + # Note: "num_inputs_signed" refers to how many *we* signed, + # which is obviously none here as we provided no keys. + if not (testsignresult.num_inputs_signed == 0 and \ + testsignresult.num_inputs_final == 1 and \ + not testsignresult.is_final): + return None, "PSBT proposal does not contain 1 signature." + + # Validate that we own one SNICKER style output: + spk = btc.verify_snicker_output(utx, our_pub, tweak_bytes) + + if spk[0] == -1: + return None, "Tweaked destination not found exactly once." + our_output_index = spk[0] + our_output_amount = utx.vout[our_output_index].nValue + + # At least one other output must have an amount equal to that at + # `our_output_index`, according to the spec. + found = 0 + for i, o in enumerate(utx.vout): + if i == our_output_index: + continue + if o.nValue == our_output_amount: + found += 1 + if found != 1: + return None, "Invalid SNICKER, there are not two equal outputs." + + # To allow the acceptance callback to assess validity, we must identify + # which input is ours and which is(are) not. + # TODO This check may (will) change if we allow non-p2sh-pwpkh inputs: + unsigned_index = -1 + for i, psbtinputsigninfo in enumerate(testsignresult.inputs_info): + if psbtinputsigninfo is None: + unsigned_index = i + break + assert unsigned_index != -1 + # All validation checks passed. We now check whether the + #transaction is acceptable according to the caller: + if not acceptance_callback([utx.vin[unsigned_index]], + [x for i, x in enumerate(utx.vin) if i != unsigned_index], + [utx.vout[our_output_index]], + [x for i, x in enumerate(utx.vout) if i != our_output_index]): + return None, "Caller rejected transaction for signing." + + # Acceptance passed, prepare the deserialized tx for signing by us: + signresult_and_signedpsbt, err = self.sign_psbt(cpsbt.serialize(), + with_sign_result=True) + if err: + return None, "Unable to sign proposed PSBT, reason: " + err + signresult, signed_psbt = signresult_and_signedpsbt + assert signresult.num_inputs_signed == 1 + assert signresult.num_inputs_final == 2 + assert signresult.is_final + # we now know the transaction is valid and fully signed; return to caller, + # along with supporting data for this tx: + return (signed_psbt.extract_transaction(), tweak_bytes, spk[1]) + class ImportWalletMixin(object): """ Mixin for BaseWallet to support importing keys. @@ -1916,10 +2192,10 @@ class BIP84Wallet(BIP32PurposedWallet): _PURPOSE = 2**31 + 84 _ENGINE = ENGINES[TYPE_P2WPKH] -class SegwitLegacyWallet(ImportWalletMixin, BIP39WalletMixin, PSBTWalletMixin, BIP49Wallet): +class SegwitLegacyWallet(ImportWalletMixin, BIP39WalletMixin, PSBTWalletMixin, SNICKERWalletMixin, BIP49Wallet): TYPE = TYPE_P2SH_P2WPKH -class SegwitWallet(ImportWalletMixin, BIP39WalletMixin, PSBTWalletMixin, BIP84Wallet): +class SegwitWallet(ImportWalletMixin, BIP39WalletMixin, PSBTWalletMixin, SNICKERWalletMixin, BIP84Wallet): TYPE = TYPE_P2WPKH class SegwitLegacyWalletFidelityBonds(FidelityBondMixin, SegwitLegacyWallet): diff --git a/jmclient/test/commontest.py b/jmclient/test/commontest.py index 36ed7c444..6295b1713 100644 --- a/jmclient/test/commontest.py +++ b/jmclient/test/commontest.py @@ -27,6 +27,12 @@ default_max_cj_fee = (1, float('inf')) +# callbacks for making transfers in-script with direct_send: +def dummy_accept_callback(tx, destaddr, actual_amount, fee_est): + return True +def dummy_info_callback(msg): + pass + class DummyBlockchainInterface(BlockchainInterface): def __init__(self): self.fake_query_results = None diff --git a/jmclient/test/test_psbt_wallet.py b/jmclient/test/test_psbt_wallet.py index a2c4cd68f..93ac3353e 100644 --- a/jmclient/test/test_psbt_wallet.py +++ b/jmclient/test/test_psbt_wallet.py @@ -10,7 +10,7 @@ import binascii import struct import copy -from commontest import make_wallets +from commontest import make_wallets, dummy_accept_callback, dummy_info_callback import jmbitcoin as bitcoin import pytest @@ -20,11 +20,6 @@ log = get_log() -def dummy_accept_callback(tx, destaddr, actual_amount, fee_est): - return True -def dummy_info_callback(msg): - pass - def test_create_and_sign_psbt_with_legacy(setup_psbt_wallet): """ The purpose of this test is to check that we can create and then partially sign a PSBT where we own one input and the other input diff --git a/jmclient/test/test_snicker.py b/jmclient/test/test_snicker.py new file mode 100644 index 000000000..8417a78b7 --- /dev/null +++ b/jmclient/test/test_snicker.py @@ -0,0 +1,118 @@ +#! /usr/bin/env python +from __future__ import (absolute_import, division, + print_function, unicode_literals) +from builtins import * # noqa: F401 +'''Test of unusual transaction types creation and push to +network to check validity.''' + +import binascii +from commontest import make_wallets, dummy_accept_callback, dummy_info_callback + +import jmbitcoin as btc +import pytest +from jmbase import get_log, bintohex, hextobin +from jmclient import (load_test_config, jm_single, + estimate_tx_fee, SNICKERReceiver, direct_send) + +log = get_log() + +@pytest.mark.parametrize( + "nw, wallet_structures, mean_amt, sdev_amt, amt, net_transfer", [ + (2, [[1, 0, 0, 0, 0]] * 2, 4, 0, 20000000, 1000), + ]) +def test_snicker_e2e(setup_snicker, nw, wallet_structures, + mean_amt, sdev_amt, amt, net_transfer): + """ Test strategy: + 1. create two wallets. + 2. with wallet 1 (Receiver), create a single transaction + tx1, from mixdepth 0 to 1. + 3. with wallet 2 (Proposer), take pubkey of all inputs from tx1, and use + them to create snicker proposals to the non-change out of tx1, + in base64 and place in proposals.txt. + 4. Receiver polls for proposals in the file manually (instead of twisted + LoopingCall) and processes them. + 5. Check for valid final transaction with broadcast. + """ + wallets = make_wallets(nw, wallet_structures, mean_amt, sdev_amt) + for w in wallets.values(): + w['wallet'].sync_wallet(fast=True) + print(wallets) + wallet_r = wallets[0]['wallet'] + wallet_p = wallets[1]['wallet'] + # next, create a tx from the receiver wallet + our_destn_script = wallet_r.get_new_script(1, 1) + tx = direct_send(wallet_r, btc.coins_to_satoshi(0.3), 0, + wallet_r.script_to_addr(our_destn_script), + accept_callback=dummy_accept_callback, + info_callback=dummy_info_callback, + return_transaction=True) + + assert tx, "Failed to spend from receiver wallet" + print("Parent transaction OK. It was: ") + print(tx) + wallet_r.process_new_tx(tx) + # we must identify the receiver's output we're going to use; + # it can be destination or change, that's up to the proposer + # to guess successfully; here we'll just choose index 0. + txid1 = tx.GetTxid()[::-1] + txid1_index = 0 + + receiver_start_bal = sum([x['value'] for x in wallet_r.get_all_utxos( + ).values()]) + + # Now create a proposal for every input index in tx1 + # (version 1 proposals mean we source keys from the/an + # ancestor transaction) + propose_keys = [] + for i in range(len(tx.vin)): + # todo check access to pubkey + sig, pub = [a for a in iter(tx.wit.vtxinwit[i].scriptWitness)] + propose_keys.append(pub) + # the proposer wallet needs to choose a single + # utxo that is bigger than the output amount of tx1 + prop_m_utxos = wallet_p.get_utxos_by_mixdepth()[0] + prop_utxo = prop_m_utxos[list(prop_m_utxos)[0]] + # get the private key for that utxo + priv = wallet_p.get_key_from_addr( + wallet_p.script_to_addr(prop_utxo['script'])) + prop_input_amt = prop_utxo['value'] + # construct the arguments for the snicker proposal: + our_input = list(prop_m_utxos)[0] # should be (txid, index) + their_input = (txid1, txid1_index) + our_input_utxo = btc.CMutableTxOut(prop_utxo['value'], + prop_utxo['script']) + fee_est = estimate_tx_fee(len(tx.vin), 2) + change_spk = wallet_p.get_new_script(0, 1) + + encrypted_proposals = [] + + for p in propose_keys: + # TODO: this can be a loop over all outputs, + # not just one guessed output, if desired. + encrypted_proposals.append( + wallet_p.create_snicker_proposal( + our_input, their_input, + our_input_utxo, + tx.vout[txid1_index], + net_transfer, + fee_est, + priv, + p, + prop_utxo['script'], + change_spk, + version_byte=1) + b"," + bintohex(p).encode('utf-8')) + with open("test_proposals.txt", "wb") as f: + f.write(b"\n".join(encrypted_proposals)) + sR = SNICKERReceiver(wallet_r) + sR.proposals_source = "test_proposals.txt" # avoid clashing with mainnet + sR.poll_for_proposals() + assert len(sR.successful_txs) == 1 + wallet_r.process_new_tx(sR.successful_txs[0]) + end_utxos = wallet_r.get_all_utxos() + print("At end the receiver has these utxos: ", end_utxos) + receiver_end_bal = sum([x['value'] for x in end_utxos.values()]) + assert receiver_end_bal == receiver_start_bal + net_transfer + +@pytest.fixture(scope="module") +def setup_snicker(): + load_test_config() From de3ad532260aae34a9158c173d5bc392bc6e31db Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Thu, 23 Apr 2020 19:50:45 +0100 Subject: [PATCH 04/15] Support output of PSBT instead of broadcast in direct_send --- jmclient/jmclient/cli_options.py | 8 +++ jmclient/jmclient/taker_utils.py | 86 ++++++++++++++++++++------------ scripts/sendpayment.py | 7 ++- 3 files changed, 68 insertions(+), 33 deletions(-) diff --git a/jmclient/jmclient/cli_options.py b/jmclient/jmclient/cli_options.py index ae822e0ce..bbd0ccc76 100644 --- a/jmclient/jmclient/cli_options.py +++ b/jmclient/jmclient/cli_options.py @@ -519,6 +519,14 @@ def get_sendpayment_parser(): help='specify recipient IRC nick for a ' 'p2ep style payment, for example:\n' 'J5Ehn3EieVZFtm4q ') + parser.add_option('--psbt', + action='store_true', + dest='with_psbt', + default=False, + help='output as psbt instead of ' + 'broadcasting the transaction. ' + 'Currently only works with direct ' + 'send (-N 0).') add_common_options(parser) return parser diff --git a/jmclient/jmclient/taker_utils.py b/jmclient/jmclient/taker_utils.py index dc4a9b894..6f5c72798 100644 --- a/jmclient/jmclient/taker_utils.py +++ b/jmclient/jmclient/taker_utils.py @@ -11,7 +11,8 @@ schedule_to_text from .wallet import BaseWallet, estimate_tx_fee, compute_tx_locktime, \ FidelityBondMixin -from jmbitcoin import make_shuffled_tx, amount_to_str, mk_burn_script +from jmbitcoin import make_shuffled_tx, amount_to_str, mk_burn_script,\ + PartiallySignedTransaction, CMutableTxOut from jmbase.support import EXIT_SUCCESS log = get_log() @@ -22,7 +23,7 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, accept_callback=None, info_callback=None, - return_transaction=False): + return_transaction=False, with_final_psbt=False): """Send coins directly from one mixdepth to one destination address; does not need IRC. Sweep as for normal sendpayment (set amount=0). If answeryes is True, callback/command line query is not performed. @@ -39,8 +40,13 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, pushed), and returns nothing. This function returns: - The txid if transaction is pushed, False otherwise, - or the full CMutableTransaction if return_transaction is True. + 1. False if there is any failure. + 2. The txid if transaction is pushed, and return_transaction is False, + and with_final_psbt is False. + 3. The full CMutableTransaction is return_transaction is True and + with_final_psbt is False. + 4. The PSBT object if with_final_psbt is True, and in + this case the transaction is *NOT* broadcast. """ #Sanity checks assert validate_address(destination)[0] or is_burn_destination(destination) @@ -139,37 +145,53 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, tx = make_shuffled_tx(list(utxos.keys()), outs, 2, tx_locktime) inscripts = {} + spent_outs = [] for i, txinp in enumerate(tx.vin): u = (txinp.prevout.hash[::-1], txinp.prevout.n) inscripts[i] = (utxos[u]["script"], utxos[u]["value"]) - success, msg = wallet_service.sign_tx(tx, inscripts) - if not success: - log.error("Failed to sign transaction, quitting. Error msg: " + msg) - return - log.info("Got signed transaction:\n") - log.info(pformat(str(tx))) - log.info("In serialized form (for copy-paste):") - log.info(bintohex(tx.serialize())) - actual_amount = amount if amount != 0 else total_inputs_val - fee_est - log.info("Sends: " + amount_to_str(actual_amount) + " to destination: " + destination) - if not answeryes: - if not accept_callback: - if input('Would you like to push to the network? (y/n):')[0] != 'y': - log.info("You chose not to broadcast the transaction, quitting.") - return False - else: - accepted = accept_callback(pformat(str(tx)), destination, actual_amount, - fee_est) - if not accepted: - return False - jm_single().bc_interface.pushtx(tx.serialize()) - txid = bintohex(tx.GetTxid()[::-1]) - successmsg = "Transaction sent: " + txid - cb = log.info if not info_callback else info_callback - cb(successmsg) - txinfo = txid if not return_transaction else tx - return txinfo - + spent_outs.append(CMutableTxOut(utxos[u]["value"], + utxos[u]["script"])) + if with_final_psbt: + # here we have the PSBTWalletMixin do the signing stage + # for us: + new_psbt = wallet_service.create_psbt_from_tx(tx, spent_outs=spent_outs) + serialized_psbt, err = wallet_service.sign_psbt(new_psbt.serialize()) + if err: + log.error("Failed to sign PSBT, quitting. Error message: " + err) + return False + new_psbt_signed = PartiallySignedTransaction.deserialize(serialized_psbt) + print("Completed PSBT created: ") + print(pformat(new_psbt_signed)) + # TODO add more readable info here as for case below. + return new_psbt_signed + else: + success, msg = wallet_service.sign_tx(tx, inscripts) + if not success: + log.error("Failed to sign transaction, quitting. Error msg: " + msg) + return + log.info("Got signed transaction:\n") + log.info(pformat(str(tx))) + log.info("In serialized form (for copy-paste):") + log.info(bintohex(tx.serialize())) + actual_amount = amount if amount != 0 else total_inputs_val - fee_est + log.info("Sends: " + amount_to_str(actual_amount) + " to destination: " + destination) + if not answeryes: + if not accept_callback: + if input('Would you like to push to the network? (y/n):')[0] != 'y': + log.info("You chose not to broadcast the transaction, quitting.") + return False + else: + accepted = accept_callback(pformat(str(tx)), destination, actual_amount, + fee_est) + if not accepted: + return False + jm_single().bc_interface.pushtx(tx.serialize()) + txid = bintohex(tx.GetTxid()[::-1]) + successmsg = "Transaction sent: " + txid + cb = log.info if not info_callback else info_callback + cb(successmsg) + txinfo = txid if not return_transaction else tx + return txinfo def sign_tx(wallet_service, tx, utxos): stx = deserialize(tx) diff --git a/scripts/sendpayment.py b/scripts/sendpayment.py index 8ee03d925..f71bef972 100755 --- a/scripts/sendpayment.py +++ b/scripts/sendpayment.py @@ -190,7 +190,12 @@ def main(): .format(exp_tx_fees_ratio)) if options.makercount == 0 and not options.p2ep: - direct_send(wallet_service, amount, mixdepth, destaddr, options.answeryes) + tx = direct_send(wallet_service, amount, mixdepth, destaddr, + options.answeryes, with_final_psbt=options.with_psbt) + if options.with_psbt: + log.info("This PSBT is fully signed and can be sent externally for " + "broadcasting:") + log.info(tx.to_base64()) return if wallet.get_txtype() == 'p2pkh': From 037a2c1023b02b8ff34afcace751ba32712e4219 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Sat, 25 Apr 2020 11:59:12 +0100 Subject: [PATCH 05/15] Adds full payjoin workflow test --- jmbitcoin/jmbitcoin/__init__.py | 5 +- jmclient/jmclient/wallet.py | 2 +- jmclient/test/test_psbt_wallet.py | 163 +++++++++++++++++++++++++++++- 3 files changed, 165 insertions(+), 5 deletions(-) diff --git a/jmbitcoin/jmbitcoin/__init__.py b/jmbitcoin/jmbitcoin/__init__.py index bfb3f455a..b27436a5a 100644 --- a/jmbitcoin/jmbitcoin/__init__.py +++ b/jmbitcoin/jmbitcoin/__init__.py @@ -7,8 +7,9 @@ from jmbitcoin.bip21 import * from bitcointx import select_chain_params from bitcointx.core import (x, b2x, b2lx, lx, COutPoint, CTxOut, CTxIn, - CTxInWitness, CTxWitness, CMutableTransaction, - Hash160, coins_to_satoshi, satoshi_to_coins) + CTxInWitness, CTxWitness, CTransaction, + CMutableTransaction, Hash160, + coins_to_satoshi, satoshi_to_coins) from bitcointx.core.key import KeyStore from bitcointx.core.script import (CScript, OP_0, SignatureHash, SIGHASH_ALL, SIGVERSION_WITNESS_V0, CScriptWitness) diff --git a/jmclient/jmclient/wallet.py b/jmclient/jmclient/wallet.py index 2a9dc1e73..59613a772 100644 --- a/jmclient/jmclient/wallet.py +++ b/jmclient/jmclient/wallet.py @@ -1054,7 +1054,7 @@ def create_psbt_from_tx(self, tx, spent_outs=None): if spent_outs[i] is None: # as above, will not be signable in this case continue - if isinstance(spent_outs[i], (btc.CMutableTransaction, btc.CMutableTxOut)): + if isinstance(spent_outs[i], (btc.CTransaction, btc.CTxOut)): # note that we trust the caller to choose Tx vs TxOut as according # to non-witness/witness: txinput.utxo = spent_outs[i] diff --git a/jmclient/test/test_psbt_wallet.py b/jmclient/test/test_psbt_wallet.py index 93ac3353e..154bd8d81 100644 --- a/jmclient/test/test_psbt_wallet.py +++ b/jmclient/test/test_psbt_wallet.py @@ -164,8 +164,167 @@ def test_create_psbt_and_sign(setup_psbt_wallet, unowned_utxo, wallet_cls): # transaction extraction must fail for not-fully-signed psbts: with pytest.raises(ValueError) as e: extracted_tx = signed_psbt.extract_transaction() - - + +@pytest.mark.parametrize('payment_amt, wallet_cls_sender, wallet_cls_receiver', [ + (0.05, SegwitLegacyWallet, SegwitLegacyWallet), + (0.95, SegwitLegacyWallet, SegwitWallet), + (0.05, SegwitWallet, SegwitLegacyWallet), + (0.95, SegwitWallet, SegwitWallet), +]) +def test_payjoin_workflow(setup_psbt_wallet, payment_amt, wallet_cls_sender, + wallet_cls_receiver): + """ Workflow step 1: + Create a payment from a wallet, and create a finalized PSBT. + This step is fairly trivial as the functionality is built-in to + PSBTWalletMixin. + Note that only Segwit* wallets are supported for PayJoin. + + Workflow step 2: + Receiver creates a new partially signed PSBT with the same amount + and at least one more utxo. + + Workflow step 3: + Given a partially signed PSBT created by a receiver, here the sender + completes (co-signs) the PSBT they are given. Note this code is a PSBT + functionality check, and does NOT include the detailed checks that + the sender should perform before agreeing to sign (see: + https://github.com/btcpayserver/btcpayserver-doc/blob/eaac676866a4d871eda5fd7752b91b88fdf849ff/Payjoin-spec.md#receiver-side + ). + """ + + wallet_r = make_wallets(1, [[3,0,0,0,0]], 1, + wallet_cls=wallet_cls_receiver)[0]["wallet"] + wallet_s = make_wallets(1, [[3,0,0,0,0]], 1, + wallet_cls=wallet_cls_sender)[0]["wallet"] + for w in [wallet_r, wallet_s]: + w.sync_wallet(fast=True) + + # destination address for payment: + destaddr = str(bitcoin.CCoinAddress.from_scriptPubKey( + bitcoin.pubkey_to_p2wpkh_script(bitcoin.privkey_to_pubkey(b"\x01"*33)))) + + payment_amt = bitcoin.coins_to_satoshi(payment_amt) + + # *** STEP 1 *** + # ************** + + # create a normal tx from the sender wallet: + payment_psbt = direct_send(wallet_s, payment_amt, 0, destaddr, + accept_callback=dummy_accept_callback, + info_callback=dummy_info_callback, + with_final_psbt=True) + + # ensure that the payemnt amount is what was intended: + out_amts = [x.nValue for x in payment_psbt.unsigned_tx.vout] + # NOTE this would have to change for more than 2 outputs: + assert any([out_amts[i] == payment_amt for i in [0, 1]]) + + # ensure that we can actually broadcast the created tx: + # (note that 'extract_transaction' represents an implicit + # PSBT finality check). + extracted_tx = payment_psbt.extract_transaction().serialize() + # don't want to push the tx right now, because of test structure + # (in production code this isn't really needed, we will not + # produce invalid payment transactions). + res = jm_single().bc_interface.rpc('testmempoolaccept', + [[bintohex(extracted_tx)]]) + assert res[0]["allowed"], "Payment transaction was rejected from mempool." + + # *** STEP 2 *** + # ************** + + # This step will not be in Joinmarket code for the first cut, + # it will be done by the merchant, but included here for the data flow. + # receiver grabs a random utxo here (as per previous sentence, this is + # the merchant's responsibility, not ours, but see earlier code in + # jmclient.maker.P2EPMaker for possibe heuristics). + # for more generality we test with two receiver-utxos, not one. + all_receiver_utxos = wallet_r.get_all_utxos() + # TODO is there a less verbose way to get any 2 utxos from the dict? + receiver_utxos_keys = list(all_receiver_utxos.keys())[:2] + receiver_utxos = {k: v for k, v in all_receiver_utxos.items( + ) if k in receiver_utxos_keys} + + # receiver will do other checks as discussed above, including payment + # amount; as discussed above, this is out of the scope of this PSBT test. + + # construct unsigned tx for payjoin-psbt: + payjoin_tx_inputs = [(x.prevout.hash[::-1], + x.prevout.n) for x in payment_psbt.unsigned_tx.vin] + payjoin_tx_inputs.extend(receiver_utxos.keys()) + # find payment output and change output + pay_out = None + change_out = None + for o in payment_psbt.unsigned_tx.vout: + jm_out_fmt = {"value": o.nValue, + "address": str(bitcoin.CCoinAddress.from_scriptPubKey( + o.scriptPubKey))} + if o.nValue == payment_amt: + assert pay_out is None + pay_out = jm_out_fmt + else: + assert change_out is None + change_out = jm_out_fmt + + # we now know there were two outputs and know which is payment. + # bump payment output with our input: + outs = [pay_out, change_out] + our_inputs_val = sum([v["value"] for _, v in receiver_utxos.items()]) + pay_out["value"] += our_inputs_val + print("we bumped the payment output value by: ", our_inputs_val) + print("It is now: ", pay_out["value"]) + unsigned_payjoin_tx = bitcoin.make_shuffled_tx(payjoin_tx_inputs, outs, + version=payment_psbt.unsigned_tx.nVersion, + locktime=payment_psbt.unsigned_tx.nLockTime) + print("we created this unsigned tx: ") + print(unsigned_payjoin_tx) + # to create the PSBT we need the spent_outs for each input, + # in the right order: + spent_outs = [] + for i, inp in enumerate(unsigned_payjoin_tx.vin): + input_found = False + for j, inp2 in enumerate(payment_psbt.unsigned_tx.vin): + if inp.prevout == inp2.prevout: + spent_outs.append(payment_psbt.inputs[j].utxo) + input_found = True + break + if input_found: + continue + # if we got here this input is ours, we must find + # it from our original utxo choice list: + for ru in receiver_utxos.keys(): + if (inp.prevout.hash[::-1], inp.prevout.n) == ru: + spent_outs.append( + wallet_r.witness_utxos_to_psbt_utxos( + {ru: receiver_utxos[ru]})[0]) + input_found = True + break + # there should be no other inputs: + assert input_found + + r_payjoin_psbt = wallet_r.create_psbt_from_tx(unsigned_payjoin_tx, + spent_outs=spent_outs) + signresultandpsbt, err = wallet_r.sign_psbt(r_payjoin_psbt.serialize(), + with_sign_result=True) + assert not err, err + signresult, receiver_signed_psbt = signresultandpsbt + assert signresult.num_inputs_final == len(receiver_utxos) + assert not signresult.is_final + + # *** STEP 3 *** + # ************** + + # take the half-signed PSBT, validate and co-sign: + + signresultandpsbt, err = wallet_s.sign_psbt( + receiver_signed_psbt.serialize(), with_sign_result=True) + assert not err, err + signresult, sender_signed_psbt = signresultandpsbt + assert signresult.is_final + # broadcast the tx + extracted_tx = sender_signed_psbt.extract_transaction().serialize() + assert jm_single().bc_interface.pushtx(extracted_tx) + @pytest.fixture(scope="module") def setup_psbt_wallet(): load_test_config() From 03a13598e89b6db263018ad884dd741583713ba7 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Tue, 28 Apr 2020 19:47:08 +0100 Subject: [PATCH 06/15] Adds libsecp256k1 installation and addresses reviews Update no-history-sync code: This updates the new functionality in jmclient.wallet_utils in the no-history-sync PR #444 to be compatible with the python-bitcointx refactoring. Remove all future/py2 compatibility code remaining: This is in line with #525 and corrects erroneous addition of more compatibility code. Addresses all flake8 complaints (ununsed imports etc) Addresses review of @dgpv Addresses review of @kristapsk --- install.sh | 38 ++++++++++++++++++++ jmbase/jmbase/support.py | 16 ++++----- jmbase/setup.py | 4 +-- jmbitcoin/jmbitcoin/__init__.py | 2 -- jmbitcoin/jmbitcoin/secp256k1_ecies.py | 9 ++--- jmbitcoin/jmbitcoin/secp256k1_main.py | 35 +++--------------- jmbitcoin/jmbitcoin/secp256k1_transaction.py | 28 +++++---------- jmbitcoin/jmbitcoin/snicker.py | 6 +--- jmbitcoin/setup.py | 4 +-- jmbitcoin/test/test_ecdh.py | 3 -- jmbitcoin/test/test_ecies.py | 3 -- jmbitcoin/test/test_tx_signing.py | 5 ++- jmclient/jmclient/client_protocol.py | 5 ++- jmclient/jmclient/commitment_utils.py | 5 +-- jmclient/jmclient/configure.py | 4 +-- jmclient/jmclient/cryptoengine.py | 10 +++--- jmclient/jmclient/electruminterface.py | 3 +- jmclient/jmclient/maker.py | 8 ++--- jmclient/jmclient/podle.py | 1 - jmclient/jmclient/snicker_receiver.py | 6 +--- jmclient/jmclient/storage.py | 6 ++-- jmclient/jmclient/taker.py | 23 ++++++------ jmclient/jmclient/taker_utils.py | 16 ++------- jmclient/jmclient/wallet.py | 28 ++++----------- jmclient/jmclient/wallet_service.py | 5 ++- jmclient/jmclient/wallet_utils.py | 34 +++++++++--------- jmclient/jmclient/yieldgenerator.py | 7 ++-- jmclient/setup.py | 5 ++- jmclient/test/commontest.py | 5 ++- jmclient/test/test_client_protocol.py | 2 +- jmclient/test/test_coinjoin.py | 2 +- jmclient/test/test_maker.py | 3 +- jmclient/test/test_payjoin.py | 2 +- jmclient/test/test_podle.py | 1 - jmclient/test/test_privkeys.py | 8 ++--- jmclient/test/test_psbt_wallet.py | 5 +-- jmclient/test/test_snicker.py | 14 +++----- jmclient/test/test_taker.py | 3 +- jmclient/test/test_tx_creation.py | 5 +-- jmclient/test/test_wallet.py | 3 +- jmclient/test/test_wallets.py | 1 - jmdaemon/jmdaemon/daemon_protocol.py | 3 +- jmdaemon/jmdaemon/irc.py | 1 - jmdaemon/jmdaemon/message_channel.py | 7 ++-- jmdaemon/setup.py | 4 +-- jmdaemon/test/test_daemon_protocol.py | 3 +- scripts/add-utxo.py | 5 ++- scripts/joinmarket-qt.py | 2 +- scripts/sendtomany.py | 5 ++- test/test_segwit.py | 11 +----- 50 files changed, 158 insertions(+), 256 deletions(-) diff --git a/install.sh b/install.sh index ef456a9dd..31b7ac418 100755 --- a/install.sh +++ b/install.sh @@ -234,6 +234,40 @@ libffi_install () popd } +libsecp256k1_build() +{ + make clean + ./autogen.sh + ./configure \ + --enable-module-recovery \ + --disable-jni \ + --prefix "${jm_root}" \ + --enable-experimental \ + --enable-module-ecdh \ + --enable-benchmark=no + make + if ! make check; then + return 1 + fi +} + +libsecp256k1_install() +{ + secp256k1_lib_tar='0d9540b13ffcd7cd44cc361b8744b93d88aa76ba' + secp256k1_lib_sha="0803d2dddbf6dd702c379118f066f638bcef6b07eea959f12d31ad2f4721fbe1" + secp256k1_lib_url='https://github.com/bitcoin-core/secp256k1/archive' + if ! dep_get "${secp256k1_lib_tar}.tar.gz" "${secp256k1_lib_sha}" "${secp256k1_lib_url}"; then + return 1 + fi + pushd "secp256k1-${secp256k1_lib_tar}" + if libsecp256k1_build; then + make install + else + return 1 + fi + popd +} + libsodium_build () { make uninstall @@ -419,6 +453,10 @@ main () # echo "Openssl was not built. Exiting." # return 1 # fi + if ! libsecp256k1_install; then + echo "libsecp256k1 was not built. Exiting." + return 1 + fi if ! libffi_install; then echo "Libffi was not built. Exiting." return 1 diff --git a/jmbase/jmbase/support.py b/jmbase/jmbase/support.py index 6afad0af6..406cf22cf 100644 --- a/jmbase/jmbase/support.py +++ b/jmbase/jmbase/support.py @@ -220,9 +220,9 @@ def print_jm_version(option, opt_str, value, parser): # helper functions for conversions of format between over-the-wire JM # and internal. See details in hexbin() docstring. -def cv(x): - success, utxo = utxostr_to_utxo(x) - if success: +def _convert(x): + good, utxo = utxostr_to_utxo(x) + if good: return utxo else: try: @@ -239,18 +239,18 @@ def listchanger(l): elif isinstance(x, dict): rlist.append(dictchanger(x)) else: - rlist.append(cv(x)) + rlist.append(_convert(x)) return rlist def dictchanger(d): rdict = {} for k, v in d.items(): if isinstance(v, dict): - rdict[cv(k)] = dictchanger(v) + rdict[_convert(k)] = dictchanger(v) elif isinstance(v, list): - rdict[cv(k)] = listchanger(v) + rdict[_convert(k)] = listchanger(v) else: - rdict[cv(k)] = cv(v) + rdict[_convert(k)] = _convert(v) return rdict def hexbin(func): @@ -276,7 +276,7 @@ def func_wrapper(inst, *args, **kwargs): elif isinstance(arg, dict): newargs.append(dictchanger(arg)) else: - newargs.append(cv(arg)) + newargs.append(_convert(arg)) return func(inst, *newargs, **kwargs) return func_wrapper \ No newline at end of file diff --git a/jmbase/setup.py b/jmbase/setup.py index 9d990025f..1f4933806 100644 --- a/jmbase/setup.py +++ b/jmbase/setup.py @@ -9,7 +9,7 @@ author_email='', license='GPL', packages=['jmbase'], - install_requires=['future', 'twisted==19.7.0', 'service-identity', + install_requires=['twisted==19.7.0', 'service-identity', 'chromalog==1.0.5'], - python_requires='>=3.3', + python_requires='>=3.6', zip_safe=False) diff --git a/jmbitcoin/jmbitcoin/__init__.py b/jmbitcoin/jmbitcoin/__init__.py index b27436a5a..b8efa5a6c 100644 --- a/jmbitcoin/jmbitcoin/__init__.py +++ b/jmbitcoin/jmbitcoin/__init__.py @@ -13,7 +13,5 @@ from bitcointx.core.key import KeyStore from bitcointx.core.script import (CScript, OP_0, SignatureHash, SIGHASH_ALL, SIGVERSION_WITNESS_V0, CScriptWitness) -from bitcointx.wallet import (CBitcoinSecret, P2WPKHBitcoinAddress, CCoinAddress, - P2SHCoinAddress) from bitcointx.core.psbt import PartiallySignedTransaction diff --git a/jmbitcoin/jmbitcoin/secp256k1_ecies.py b/jmbitcoin/jmbitcoin/secp256k1_ecies.py index 9d928baa7..8e821088b 100644 --- a/jmbitcoin/jmbitcoin/secp256k1_ecies.py +++ b/jmbitcoin/jmbitcoin/secp256k1_ecies.py @@ -1,8 +1,5 @@ #!/usr/bin/python -from __future__ import (absolute_import, division, - print_function, unicode_literals) -from builtins import * # noqa: F401 -from future.utils import native + import coincurve as secp256k1 import base64 import hmac @@ -19,7 +16,7 @@ class ECIESDecryptionError(Exception): # AES primitives. See BIP-SNICKER for specification. def aes_encrypt(key, data, iv): encrypter = pyaes.Encrypter( - pyaes.AESModeOfOperationCBC(key, iv=native(iv))) + pyaes.AESModeOfOperationCBC(key, iv=iv)) enc_data = encrypter.feed(data) enc_data += encrypter.feed() @@ -27,7 +24,7 @@ def aes_encrypt(key, data, iv): def aes_decrypt(key, data, iv): decrypter = pyaes.Decrypter( - pyaes.AESModeOfOperationCBC(key, iv=native(iv))) + pyaes.AESModeOfOperationCBC(key, iv=iv)) try: dec_data = decrypter.feed(data) dec_data += decrypter.feed() diff --git a/jmbitcoin/jmbitcoin/secp256k1_main.py b/jmbitcoin/jmbitcoin/secp256k1_main.py index 197809fe6..87cba6c14 100644 --- a/jmbitcoin/jmbitcoin/secp256k1_main.py +++ b/jmbitcoin/jmbitcoin/secp256k1_main.py @@ -1,15 +1,11 @@ #!/usr/bin/python -from future.utils import native_bytes, bytes_to_native_str -import binascii -import hashlib -import sys import base64 import struct import coincurve as secp256k1 from bitcointx import base58 -from bitcointx.core import Hash, CBitcoinTransaction -from bitcointx.core.key import CKeyBase, CPubKey +from bitcointx.core import Hash +from bitcointx.core.key import CKeyBase from bitcointx.signmessage import BitcoinMessage #Required only for PoDLE calculation: @@ -60,7 +56,7 @@ def privkey_to_pubkey(priv): and return compressed/uncompressed public key as appropriate.''' compressed, priv = read_privkey(priv) #secp256k1 checks for validity of key value. - newpriv = secp256k1.PrivateKey(secret=native_bytes(priv)) + newpriv = secp256k1.PrivateKey(secret=priv) return newpriv.public_key.format(compressed) # b58check wrapper functions around bitcointx.base58 functions: @@ -137,7 +133,7 @@ def multiply(s, pub, return_serialized=True): ''' newpub = secp256k1.PublicKey(pub) #see note to "tweak_mul" function in podle.py - res = newpub.multiply(native_bytes(s)) + res = newpub.multiply(s) if not return_serialized: return res return res.format() @@ -245,32 +241,9 @@ class JMCKey(bytes, CKeyBase): def __init__(self, b): CKeyBase.__init__(self, b, compressed=True) - def is_compressed(self): - return True - - @property - def secret_bytes(self): - assert isinstance(self, bytes) - return self[:32] - def sign(self, hash): assert isinstance(hash, (bytes, bytearray)) if len(hash) != 32: raise ValueError('Hash must be exactly 32 bytes long') # TODO: non default sighash flag. return ecdsa_raw_sign(hash, self.secret_bytes + b"\x01", rawmsg=True) - - - def verify(self, hash, sig): - return self.pub.verify(hash, sig) - - def verify_nonstrict(self, hash, sig): - return self.pub.verify_nonstrict(hash, sig) - - @classmethod - def from_secret_bytes(cls, secret, compressed=True): - return cls(secret, compressed=compressed) - - @classmethod - def from_bytes(cls, data): - raise NotImplementedError('subclasses must override from_bytes()') diff --git a/jmbitcoin/jmbitcoin/secp256k1_transaction.py b/jmbitcoin/jmbitcoin/secp256k1_transaction.py index aca9903c1..0966f12e9 100644 --- a/jmbitcoin/jmbitcoin/secp256k1_transaction.py +++ b/jmbitcoin/jmbitcoin/secp256k1_transaction.py @@ -1,20 +1,14 @@ #!/usr/bin/python -from past.builtins import basestring -from io import BytesIO -import binascii -import copy -import re -import os -import struct + # note, only used for non-cryptographic randomness: import random from jmbitcoin.secp256k1_main import * from bitcointx.core import (CMutableTransaction, Hash160, CTxInWitness, - CTxWitness, CMutableOutPoint, CMutableTxIn, - CMutableTxOut, ValidationError, lx, x) + CMutableOutPoint, CMutableTxIn, + CMutableTxOut, ValidationError) from bitcointx.core.script import * -from bitcointx.wallet import P2WPKHBitcoinAddress, CCoinAddress +from bitcointx.wallet import P2WPKHCoinAddress, CCoinAddress, P2PKHCoinAddress from bitcointx.core.scripteval import (VerifyScript, SCRIPT_VERIFY_WITNESS, SCRIPT_VERIFY_P2SH, SIGVERSION_WITNESS_V0) @@ -74,10 +68,7 @@ def pubkey_to_p2pkh_script(pub, require_compressed=False): representing the corresponding pay-to-pubkey-hash scriptPubKey. """ - if not is_valid_pubkey(pub, require_compressed=require_compressed): - raise Exception("Invalid pubkey") - return CScript([OP_DUP, OP_HASH160, Hash160(pub), - OP_EQUALVERIFY, OP_CHECKSIG]) + return P2PKHCoinAddress.from_pubkey(pub).to_scriptPubKey() def pubkey_to_p2wpkh_script(pub): """ @@ -85,9 +76,7 @@ def pubkey_to_p2wpkh_script(pub): representing the corresponding pay-to-witness-pubkey-hash scriptPubKey. """ - if not is_valid_pubkey(pub, True): - raise Exception("Invalid pubkey") - return CScript([OP_0, Hash160(pub)]) + return P2WPKHCoinAddress.from_pubkey(pub).to_scriptPubKey() def pubkey_to_p2sh_p2wpkh_script(pub): """ @@ -146,8 +135,7 @@ def return_err(e): return None, "Error in signing: " + repr(e) assert isinstance(tx, CMutableTransaction) - # using direct local access to libsecp256k1 binding, because - # python-bitcoinlib uses OpenSSL key management: + pub = privkey_to_pubkey(priv) if not amount: @@ -175,7 +163,7 @@ def return_err(e): input_scriptPubKey = pubkey_to_p2wpkh_script(pub) # only created for convenience access to scriptCode: - input_address = P2WPKHBitcoinAddress.from_scriptPubKey(input_scriptPubKey) + input_address = P2WPKHCoinAddress.from_scriptPubKey(input_scriptPubKey) # function name is misleading here; redeemScript only applies to p2sh. scriptCode = input_address.to_redeemScript() diff --git a/jmbitcoin/jmbitcoin/snicker.py b/jmbitcoin/jmbitcoin/snicker.py index 5c6be9493..05d5b4bb6 100644 --- a/jmbitcoin/jmbitcoin/snicker.py +++ b/jmbitcoin/jmbitcoin/snicker.py @@ -1,7 +1,3 @@ -from __future__ import (absolute_import, division, - print_function, unicode_literals) -from builtins import * # noqa: F401 - # Implementation of proposal as per # https://gist.github.com/AdamISZ/2c13fb5819bd469ca318156e2cf25d79 # (BIP SNICKER) @@ -45,7 +41,7 @@ def verify_snicker_output(tx, pub, tweak, spk_type='p2sh-p2wpkh'): or -1 and None if it is not found exactly once. TODO Add support for other scriptPubKey types. """ - assert isinstance(tx, btc.CBitcoinTransaction) + assert isinstance(tx, btc.CTransaction) expected_destination_pub = snicker_pubkey_tweak(pub, tweak) expected_destination_spk = pubkey_to_p2sh_p2wpkh_script(expected_destination_pub) found = 0 diff --git a/jmbitcoin/setup.py b/jmbitcoin/setup.py index 75ab2c8c7..9330fc002 100644 --- a/jmbitcoin/setup.py +++ b/jmbitcoin/setup.py @@ -9,6 +9,6 @@ author_email='', license='GPL', packages=['jmbitcoin'], - install_requires=['future', 'coincurve', 'urldecode', - 'python-bitcointx>=1.0.5', 'pyaes'], + python_requires='>=3.6', + install_requires=['coincurve', 'python-bitcointx>=1.0.5', 'pyaes', 'urldecode'], zip_safe=False) diff --git a/jmbitcoin/test/test_ecdh.py b/jmbitcoin/test/test_ecdh.py index c417c462b..5988385b1 100644 --- a/jmbitcoin/test/test_ecdh.py +++ b/jmbitcoin/test/test_ecdh.py @@ -1,7 +1,4 @@ #! /usr/bin/env python -from __future__ import (absolute_import, division, - print_function, unicode_literals) -from builtins import * # noqa: F401 '''Tests coincurve binding to libsecp256k1 ecdh module code''' import hashlib diff --git a/jmbitcoin/test/test_ecies.py b/jmbitcoin/test/test_ecies.py index 4a529c5c3..a34e58371 100644 --- a/jmbitcoin/test/test_ecies.py +++ b/jmbitcoin/test/test_ecies.py @@ -1,7 +1,4 @@ #! /usr/bin/env python -from __future__ import (absolute_import, division, - print_function, unicode_literals) -from builtins import * # noqa: F401 '''Tests ECIES implementation as defined in BIP-SNICKER (and will be updated if that is).''' diff --git a/jmbitcoin/test/test_tx_signing.py b/jmbitcoin/test/test_tx_signing.py index 4db7f3c6e..05cbbf521 100644 --- a/jmbitcoin/test/test_tx_signing.py +++ b/jmbitcoin/test/test_tx_signing.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -import sys import pytest import binascii @@ -26,7 +25,7 @@ def test_sign_standard_txs(addrtype): # (note that the input utxo is fake so we are really only creating # a destination here). scriptPubKey = btc.CScript([btc.OP_0, btc.Hash160(pub)]) - address = btc.P2WPKHBitcoinAddress.from_scriptPubKey(scriptPubKey) + address = btc.P2WPKHCoinAddress.from_scriptPubKey(scriptPubKey) # Create a dummy outpoint; use same 32 bytes for convenience txid = priv[:32] @@ -66,7 +65,7 @@ def test_mk_shuffled_tx(): # prepare two addresses for the outputs pub = btc.privkey_to_pubkey(btc.Hash(b"priv") + b"\x01") scriptPubKey = btc.CScript([btc.OP_0, btc.Hash160(pub)]) - addr1 = btc.P2WPKHBitcoinAddress.from_scriptPubKey(scriptPubKey) + addr1 = btc.P2WPKHCoinAddress.from_scriptPubKey(scriptPubKey) scriptPubKey_p2sh = scriptPubKey.to_p2sh_scriptPubKey() addr2 = btc.CCoinAddress.from_scriptPubKey(scriptPubKey_p2sh) diff --git a/jmclient/jmclient/client_protocol.py b/jmclient/jmclient/client_protocol.py index 9711d9b0d..b580c8847 100644 --- a/jmclient/jmclient/client_protocol.py +++ b/jmclient/jmclient/client_protocol.py @@ -1,5 +1,4 @@ #! /usr/bin/env python -from future.utils import iteritems from twisted.internet import protocol, reactor, task from twisted.internet.error import (ConnectionLost, ConnectionAborted, ConnectionClosed, ConnectionDone) @@ -15,7 +14,7 @@ import os import sys from jmbase import (get_log, EXIT_FAILURE, hextobin, bintohex, - utxo_to_utxostr, dictchanger) + utxo_to_utxostr) from jmclient import (jm_single, get_irc_mchannels, RegtestBitcoinCoreInterface) import jmbitcoin as btc @@ -303,7 +302,7 @@ def on_JM_TX_RECEIVED(self, nick, txhex, offer): return {"accepted": True} def tx_match(self, txd): - for k,v in iteritems(self.finalized_offers): + for k,v in self.finalized_offers.items(): # Tx considered defined by its output set if v["txd"].vout == txd.vout: offerinfo = v diff --git a/jmclient/jmclient/commitment_utils.py b/jmclient/jmclient/commitment_utils.py index ab598adba..49233eeb7 100644 --- a/jmclient/jmclient/commitment_utils.py +++ b/jmclient/jmclient/commitment_utils.py @@ -1,10 +1,7 @@ import sys -import jmbitcoin as btc from jmbase import jmprint -from jmclient import (jm_single, get_p2pk_vbyte, get_p2sh_vbyte, - BTCEngine, TYPE_P2PKH, TYPE_P2SH_P2WPKH, - BTC_P2PKH, BTC_P2SH_P2WPKH) +from jmclient import jm_single, BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH from jmbase.support import EXIT_FAILURE, utxostr_to_utxo diff --git a/jmclient/jmclient/configure.py b/jmclient/jmclient/configure.py index 45f5df239..17c885c3a 100644 --- a/jmclient/jmclient/configure.py +++ b/jmclient/jmclient/configure.py @@ -379,13 +379,13 @@ def validate_address(addr): try: # automatically respects the network # as set in btc.select_chain_params(...) - x = btc.CCoinAddress(addr) + dummyaddr = btc.CCoinAddress(addr) except Exception as e: return False, repr(e) # additional check necessary because python-bitcointx # does not check hash length on p2sh construction. try: - x.to_scriptPubKey() + dummyaddr.to_scriptPubKey() except Exception as e: return False, repr(e) return True, "address validated" diff --git a/jmclient/jmclient/cryptoengine.py b/jmclient/jmclient/cryptoengine.py index d713191f8..17f114870 100644 --- a/jmclient/jmclient/cryptoengine.py +++ b/jmclient/jmclient/cryptoengine.py @@ -1,11 +1,11 @@ - -from binascii import hexlify, unhexlify from collections import OrderedDict import struct import jmbitcoin as btc -from .configure import get_network, jm_single +from jmbase import bintohex +from .configure import get_network + #NOTE: before fidelity bonds and watchonly wallet, each of these types corresponded # to one wallet type and one engine, not anymore @@ -31,7 +31,7 @@ def detect_script_type(script_str): script = btc.CScript(script_str) if not script.is_valid(): raise EngineError("Unknown script type for script '{}'" - .format(hexlify(script_str))) + .format(bintohex(script_str))) if script.is_p2pkh(): return TYPE_P2PKH elif script.is_p2sh(): @@ -42,7 +42,7 @@ def detect_script_type(script_str): elif script.is_witness_v0_keyhash(): return TYPE_P2WPKH raise EngineError("Unknown script type for script '{}'" - .format(hexlify(script_str))) + .format(bintohex(script_str))) class classproperty(object): """ diff --git a/jmclient/jmclient/electruminterface.py b/jmclient/jmclient/electruminterface.py index 6327a9884..773d83042 100644 --- a/jmclient/jmclient/electruminterface.py +++ b/jmclient/jmclient/electruminterface.py @@ -1,4 +1,3 @@ -from future.utils import iteritems import jmbitcoin as btc import json import queue as Queue @@ -332,7 +331,7 @@ def sync_unspent(self, wallet): for m in range(wallet.max_mixdepth): for fc in [0, 1]: branch_list = [] - for k, v in iteritems(self.temp_addr_history[m][fc]): + for k, v in self.temp_addr_history[m][fc].items(): if k == "finished": continue if v["used"]: diff --git a/jmclient/jmclient/maker.py b/jmclient/jmclient/maker.py index 8ae48342c..b4b9ab6c3 100644 --- a/jmclient/jmclient/maker.py +++ b/jmclient/jmclient/maker.py @@ -1,5 +1,4 @@ #! /usr/bin/env python -from future.utils import iteritems import base64 import pprint import random @@ -7,8 +6,7 @@ import abc import jmbitcoin as btc -from jmbase import (bintohex, hextobin, hexbin, - get_log, EXIT_SUCCESS, EXIT_FAILURE) +from jmbase import bintohex, hexbin, get_log, EXIT_SUCCESS, EXIT_FAILURE from jmclient.wallet import estimate_tx_fee, compute_tx_locktime from jmclient.wallet_service import WalletService from jmclient.configure import jm_single @@ -470,7 +468,7 @@ def on_tx_received(self, nick, txser): [x[1] for x in utxo.values()]) total_sender_input = 0 - for i, u in iteritems(utxo): + for i, u in utxo.items(): if utxo_data[i] is None: return (False, "Proposed transaction contains invalid utxos") total_sender_input += utxo_data[i]["value"] @@ -505,7 +503,7 @@ def on_tx_received(self, nick, txser): # Manual verification of the transaction signatures. # TODO handle native segwit properly - for i, u in iteritems(utxo): + for i, u in utxo.items(): if not btc.verify_tx_input(tx, i, tx.vin[i].scriptSig, btc.CScript(utxo_data[i]["script"]), diff --git a/jmclient/jmclient/podle.py b/jmclient/jmclient/podle.py index d7dd26f76..dc47738bc 100644 --- a/jmclient/jmclient/podle.py +++ b/jmclient/jmclient/podle.py @@ -5,7 +5,6 @@ import sys import hashlib import json -import binascii import struct from pprint import pformat from jmbase import jmprint diff --git a/jmclient/jmclient/snicker_receiver.py b/jmclient/jmclient/snicker_receiver.py index 8d9106708..969a14a45 100644 --- a/jmclient/jmclient/snicker_receiver.py +++ b/jmclient/jmclient/snicker_receiver.py @@ -1,13 +1,9 @@ #! /usr/bin/env python -from __future__ import (absolute_import, division, - print_function, unicode_literals) -from builtins import * # noqa: F401 import sys -import binascii import jmbitcoin as btc -from jmclient.configure import get_p2pk_vbyte, jm_single +from jmclient.configure import jm_single from jmbase import (get_log, EXIT_FAILURE, utxo_to_utxostr, bintohex, hextobin) diff --git a/jmclient/jmclient/storage.py b/jmclient/jmclient/storage.py index 34d969d97..1989979e0 100644 --- a/jmclient/jmclient/storage.py +++ b/jmclient/jmclient/storage.py @@ -1,5 +1,3 @@ -from future.utils import native - import os import shutil import atexit @@ -253,7 +251,7 @@ def _decrypt_file(self, password, data): def _encrypt(self, data, iv): encrypter = pyaes.Encrypter( - pyaes.AESModeOfOperationCBC(self._hash.hash, iv=native(iv))) + pyaes.AESModeOfOperationCBC(self._hash.hash, iv=iv)) enc_data = encrypter.feed(self.MAGIC_DETECT_ENC + data) enc_data += encrypter.feed() @@ -261,7 +259,7 @@ def _encrypt(self, data, iv): def _decrypt(self, data, iv): decrypter = pyaes.Decrypter( - pyaes.AESModeOfOperationCBC(self._hash.hash, iv=native(iv))) + pyaes.AESModeOfOperationCBC(self._hash.hash, iv=iv)) try: dec_data = decrypter.feed(data) dec_data += decrypter.feed() diff --git a/jmclient/jmclient/taker.py b/jmclient/jmclient/taker.py index 9ee185ac4..a4f13cba7 100644 --- a/jmclient/jmclient/taker.py +++ b/jmclient/jmclient/taker.py @@ -1,5 +1,4 @@ #! /usr/bin/env python -from future.utils import iteritems import base64 import pprint @@ -8,11 +7,11 @@ import jmbitcoin as btc from jmclient.configure import jm_single, validate_address -from jmbase import get_log, hextobin, bintohex, hexbin +from jmbase import get_log, bintohex, hexbin from jmclient.support import (calc_cj_fee, weighted_order_choose, choose_orders, choose_sweep_orders) from jmclient.wallet import estimate_tx_fee, compute_tx_locktime -from jmclient.podle import generate_podle, get_podle_commitments, PoDLE +from jmclient.podle import generate_podle, get_podle_commitments from jmclient.wallet_service import WalletService from .output import generate_podle_error_string from .cryptoengine import EngineError @@ -354,7 +353,7 @@ def receive_utxos(self, ioauth_data): rejected_counterparties = [] #Need to authorize against the btc pubkey first. - for nick, nickdata in iteritems(ioauth_data): + for nick, nickdata in ioauth_data.items(): utxo_list, auth_pub, cj_addr, change_addr, btc_sig, maker_pk = nickdata if not self.auth_counterparty(btc_sig, auth_pub, maker_pk): jlog.debug( @@ -374,7 +373,7 @@ def receive_utxos(self, ioauth_data): self.maker_utxo_data = {} - for nick, nickdata in iteritems(ioauth_data): + for nick, nickdata in ioauth_data.items(): utxo_list, auth_pub, cj_addr, change_addr, _, _ = nickdata utxo_data = jm_single().bc_interface.query_utxo_set(utxo_list) self.utxos[nick] = utxo_list @@ -452,8 +451,7 @@ def receive_utxos(self, ioauth_data): #used to track return of signatures for phase 2 self.nonrespondants = list(self.maker_utxo_data.keys()) - my_total_in = sum([va['value'] for u, va in iteritems(self.input_utxos) - ]) + my_total_in = sum([va['value'] for u, va in self.input_utxos.items()]) if self.my_change_addr: #Estimate fee per choice of next/3/6 blocks targetting. estimated_fee = estimate_tx_fee( @@ -559,7 +557,7 @@ def on_sig(self, nick, sigb64): utxo_data = jm_single().bc_interface.query_utxo_set([x[ 1] for x in utxo.values()]) # insert signatures - for i, u in iteritems(utxo): + for i, u in utxo.items(): if utxo_data[i] is None: continue # Check if the sender included the scriptCode in the sig message; @@ -698,7 +696,7 @@ def priv_utxo_pairs_from_utxos(utxos, age, amt): new_utxos, too_old, too_small = filter_by_coin_age_amt(list(utxos.keys()), age, amt) new_utxos_dict = {k: v for k, v in utxos.items() if k in new_utxos} - for k, v in iteritems(new_utxos_dict): + for k, v in new_utxos_dict.items(): addr = self.wallet_service.script_to_addr(v["script"]) priv = self.wallet_service.get_key_from_addr(addr) if priv: #can be null from create-unsigned @@ -831,7 +829,8 @@ def push(self): self.on_finished_callback(False, fromtx=True) else: if nick_to_use: - return (nick_to_use, tx) + # TODO option not currently functional + return (nick_to_use, self.latest_tx.serialize()) #if push was not successful, return None def self_sign_and_push(self): @@ -985,7 +984,7 @@ def receive_utxos(self, ioauth_data): # use output destination self.my_cj_addr and use amount self.amount self.outputs.append({'address': self.my_cj_addr, 'value': self.cjamount}) - my_total_in = sum([va['value'] for u, va in iteritems(self.input_utxos)]) + my_total_in = sum([va['value'] for u, va in self.input_utxos.items()]) # estimate the fee for the version of the transaction which is # not coinjoined: est_fee = estimate_tx_fee(len(self.input_utxos), 2, @@ -1137,7 +1136,7 @@ def on_tx_received(self, nick, txser): # Next we'll verify each of the counterparty's inputs, # while at the same time gathering the total they spent. total_receiver_input = 0 - for i, u in iteritems(retrieve_utxos): + for i, u in retrieve_utxos.items(): if utxo_data[i] is None: return (False, "Proposed transaction contains invalid utxos") total_receiver_input += utxo_data[i]["value"] diff --git a/jmclient/jmclient/taker_utils.py b/jmclient/jmclient/taker_utils.py index 6f5c72798..b6c488620 100644 --- a/jmclient/jmclient/taker_utils.py +++ b/jmclient/jmclient/taker_utils.py @@ -1,4 +1,3 @@ -from future.utils import iteritems import logging import pprint import os @@ -81,8 +80,7 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, log.error( "There are no available utxos in mixdepth: " + str(mixdepth) + ", quitting.") return - - total_inputs_val = sum([va['value'] for u, va in iteritems(utxos)]) + total_inputs_val = sum([va['value'] for u, va in utxos.items()]) if is_burn_destination(destination): if len(utxos) > 1: @@ -116,7 +114,7 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, fee_est = estimate_tx_fee(len(utxos), 2, txtype=txtype) else: fee_est = initial_fee_est - total_inputs_val = sum([va['value'] for u, va in iteritems(utxos)]) + total_inputs_val = sum([va['value'] for u, va in utxos.items()]) changeval = total_inputs_val - fee_est - amount outs = [{"value": amount, "address": destination}] change_addr = wallet_service.get_internal_addr(mixdepth) @@ -193,16 +191,6 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, txinfo = txid if not return_transaction else tx return txinfo -def sign_tx(wallet_service, tx, utxos): - stx = deserialize(tx) - our_inputs = {} - for index, ins in enumerate(stx['ins']): - utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) - script = wallet_service.addr_to_script(utxos[utxo]['address']) - amount = utxos[utxo]['value'] - our_inputs[index] = (script, amount) - return wallet_service.sign_tx(stx, our_inputs) - def get_tumble_log(logsdir): tumble_log = logging.getLogger('tumbler') tumble_log.setLevel(logging.DEBUG) diff --git a/jmclient/jmclient/wallet.py b/jmclient/jmclient/wallet.py index 59613a772..cd15f5585 100644 --- a/jmclient/jmclient/wallet.py +++ b/jmclient/jmclient/wallet.py @@ -28,7 +28,7 @@ from .support import get_random_bytes from . import mn_encode, mn_decode import jmbitcoin as btc -from jmbase import JM_WALLET_NAME_PREFIX, bintohex +from jmbase import JM_WALLET_NAME_PREFIX """ @@ -998,23 +998,6 @@ def close(self): def __del__(self): self.close() -class DummyKeyStore(btc.KeyStore): - @classmethod - def from_iterable(cls, iterable, **kwargs): - kstore = cls(**kwargs) - for k in iterable: - kstore.add_key(k) - return kstore - - def add_key(self, k): - if isinstance(k, btc.CKeyBase): - if k.pub.key_id in self._privkeys: - assert self._privkeys[k.pub.key_id] == k - else: - self._privkeys[k.pub.key_id] = k - else: - raise ValueError('object supplied to add_key is of unrecognized type') - class PSBTWalletMixin(object): """ Mixin for BaseWallet to provide BIP174 @@ -1062,7 +1045,7 @@ def create_psbt_from_tx(self, tx, spent_outs=None): assert False, "invalid spent output type passed into PSBT creator" # we now insert redeemscripts where that is possible and necessary: for i, txinput in enumerate(new_psbt.inputs): - if isinstance(txinput.utxo, btc.CMutableTxOut): + if isinstance(txinput.utxo, btc.CTxOut): # witness if txinput.utxo.scriptPubKey.is_witness_scriptpubkey(): # nothing needs inserting; the scriptSig is empty. @@ -1094,14 +1077,15 @@ def sign_psbt(self, in_psbt, with_sign_result=False): """ try: new_psbt = btc.PartiallySignedTransaction.from_binary(in_psbt) - except: - return None, "Unable to deserialize the PSBT object, invalid format." + except Exception as e: + return None, "Unable to deserialize binary PSBT, error: " + repr(e) privkeys = [] for k, v in self._utxos._utxo.items(): for k2, v2 in v.items(): privkeys.append(self._get_priv_from_path(v2[0])) jmckeys = list(btc.JMCKey(x[0][:-1]) for x in privkeys) - new_keystore = DummyKeyStore.from_iterable(jmckeys) + new_keystore = btc.KeyStore.from_iterable(jmckeys, + require_path_templates=False) # for p2sh inputs that we want to sign, the redeem_script # field must be populated by us, as the counterparty did not diff --git a/jmclient/jmclient/wallet_service.py b/jmclient/jmclient/wallet_service.py index 35521058a..6eb892bfc 100644 --- a/jmclient/jmclient/wallet_service.py +++ b/jmclient/jmclient/wallet_service.py @@ -3,7 +3,6 @@ import collections import time import ast -import binascii import sys from decimal import Decimal from copy import deepcopy @@ -16,8 +15,8 @@ from jmclient.blockchaininterface import (INF_HEIGHT, BitcoinCoreInterface, BitcoinCoreNoHistoryInterface) from jmclient.wallet import FidelityBondMixin -from jmbase.support import jmprint, EXIT_SUCCESS, utxo_to_utxostr, bintohex, hextobin -from jmbitcoin import lx +from jmbase.support import jmprint, EXIT_SUCCESS, utxo_to_utxostr, hextobin + """Wallet service diff --git a/jmclient/jmclient/wallet_utils.py b/jmclient/jmclient/wallet_utils.py index 8aa381829..4fb30e01f 100644 --- a/jmclient/jmclient/wallet_utils.py +++ b/jmclient/jmclient/wallet_utils.py @@ -1,4 +1,3 @@ -from future.utils import iteritems import json import os import sys @@ -17,7 +16,7 @@ is_native_segwit_mode, load_program_config, add_base_options, check_regtest) from jmclient.wallet_service import WalletService from jmbase.support import (get_password, jmprint, EXIT_FAILURE, - EXIT_ARGERROR, utxo_to_utxostr) + EXIT_ARGERROR, utxo_to_utxostr, hextobin) from .cryptoengine import TYPE_P2PKH, TYPE_P2SH_P2WPKH, TYPE_P2WPKH, \ TYPE_SEGWIT_LEGACY_WALLET_FIDELITY_BONDS @@ -324,9 +323,8 @@ def get_tx_info(txid): """ rpctx = jm_single().bc_interface.get_transaction(txid) txhex = str(rpctx['hex']) - txd = btc.deserialize(txhex) - output_script_values = {binascii.unhexlify(sv['script']): sv['value'] - for sv in txd['outs']} + tx = btc.CMutableTransaction.deserialize(hextobin(txhex)) + output_script_values = {x.scriptPubKey: x.nValue for x in tx.vout} value_freq_list = sorted( Counter(output_script_values.values()).most_common(), key=lambda x: -x[1]) @@ -338,7 +336,7 @@ def get_tx_info(txid): cj_amount = value_freq_list[0][0] cj_n = value_freq_list[0][1] return is_coinjoin, cj_amount, cj_n, output_script_values,\ - rpctx.get('blocktime', 0), txd + rpctx.get('blocktime', 0), tx def get_imported_privkey_branch(wallet_service, m, showprivkey): @@ -383,7 +381,7 @@ def wallet_showutxos(wallet, showprivkey): unsp[us]['privkey'] = wallet.get_wif_path(av['path']) used_commitments, external_commitments = podle.get_podle_commitments() - for u, ec in iteritems(external_commitments): + for u, ec in external_commitments.items(): success, us = utxo_to_utxostr(u) assert success tries = podle.get_podle_tries(utxo=u, max_tries=max_tries, @@ -404,7 +402,7 @@ def wallet_display(wallet_service, showprivkey, displayall=False, def get_addr_status(addr_path, utxos, is_new, is_internal): addr_balance = 0 status = [] - for utxo, utxodata in iteritems(utxos): + for utxo, utxodata in utxos.items(): if addr_path != utxodata['path']: continue addr_balance += utxodata['value'] @@ -415,7 +413,7 @@ def get_addr_status(addr_path, utxos, is_new, is_internal): # to bci if jm_single().bc_interface.__class__ == BitcoinCoreInterface: is_coinjoin, cj_amount, cj_n = \ - get_tx_info(binascii.hexlify(utxo[0]).decode('ascii'))[:3] + get_tx_info(utxo[0])[:3] if is_coinjoin and utxodata['value'] == cj_amount: status.append('cj-out') elif is_coinjoin: @@ -774,7 +772,7 @@ def print_row(index, time, tx_type, amount, delta, balance, cj_n, tx_number = 0 for tx in txes: is_coinjoin, cj_amount, cj_n, output_script_values, blocktime, txd =\ - get_tx_info(tx['txid']) + get_tx_info(hextobin(tx['txid'])) # unconfirmed transactions don't have blocktime, get_tx_info() returns # 0 in that case @@ -784,21 +782,21 @@ def print_row(index, time, tx_type, amount, delta, balance, cj_n, output_script_values.keys()) rpc_inputs = [] - for ins in txd['ins']: + for ins in txd.vin: wallet_tx = jm_single().bc_interface.get_transaction( - ins['outpoint']['hash']) + ins.prevout.hash[::-1]) if wallet_tx is None: continue - input_dict = btc.deserialize(str(wallet_tx['hex']))['outs'][ins[ - 'outpoint']['index']] + inp = btc.CMutableTransaction.deserialize(hextobin( + wallet_tx['hex'])).vout[ins.prevout.n] + input_dict = {"script": inp.scriptPubKey, "value": inp.nValue} rpc_inputs.append(input_dict) - rpc_input_scripts = set(binascii.unhexlify(ind['script']) - for ind in rpc_inputs) + rpc_input_scripts = set(ind['script'] for ind in rpc_inputs) our_input_scripts = wallet_script_set.intersection(rpc_input_scripts) our_input_values = [ ind['value'] for ind in rpc_inputs - if binascii.unhexlify(ind['script']) in our_input_scripts] + if ind['script'] in our_input_scripts] our_input_value = sum(our_input_values) utxos_consumed = len(our_input_values) @@ -866,7 +864,7 @@ def print_row(index, time, tx_type, amount, delta, balance, cj_n, amount = cj_amount delta_balance = out_value - our_input_value mixdepth_src = wallet.get_script_mixdepth(list(our_input_scripts)[0]) - cj_script = list(set([a for a, v in iteritems(output_script_values) + cj_script = list(set([a for a, v in output_script_values.items() if v == cj_amount]).intersection(our_output_scripts))[0] mixdepth_dst = wallet.get_script_mixdepth(cj_script) else: diff --git a/jmclient/jmclient/yieldgenerator.py b/jmclient/jmclient/yieldgenerator.py index f89af3604..b0a329939 100644 --- a/jmclient/jmclient/yieldgenerator.py +++ b/jmclient/jmclient/yieldgenerator.py @@ -1,5 +1,4 @@ #! /usr/bin/env python -from future.utils import iteritems import datetime import os @@ -78,7 +77,7 @@ def __init__(self, wallet_service, offerconfig): def create_my_orders(self): mix_balance = self.get_available_mixdepths() - if len([b for m, b in iteritems(mix_balance) if b > 0]) == 0: + if len([b for m, b in mix_balance.items() if b > 0]) == 0: jlog.error('do not have any coins left') return [] @@ -115,7 +114,7 @@ def oid_to_order(self, offer, amount): mix_balance = self.get_available_mixdepths() filtered_mix_balance = {m: b - for m, b in iteritems(mix_balance) + for m, b in mix_balance.items() if b >= total_amount} if not filtered_mix_balance: return None, None, None @@ -177,7 +176,7 @@ def select_input_mixdepth(self, available, offer, amount): inputs. available is a mixdepth/balance dict of all the mixdepths that can be chosen from, i.e. have enough balance. If there is no suitable input, the function can return None to abort the order.""" - available = sorted(iteritems(available), key=lambda entry: entry[0]) + available = sorted(available.items(), key=lambda entry: entry[0]) return available[0][0] def select_output_address(self, input_mixdepth, offer, amount): diff --git a/jmclient/setup.py b/jmclient/setup.py index 66c480df5..01238bb57 100644 --- a/jmclient/setup.py +++ b/jmclient/setup.py @@ -9,8 +9,7 @@ author_email='', license='GPL', packages=['jmclient'], - install_requires=['future', 'configparser;python_version<"3.2"', - 'joinmarketbase==0.7.0dev', 'mnemonic', 'argon2_cffi', + install_requires=['joinmarketbase==0.7.0dev', 'mnemonic', 'argon2_cffi', 'bencoder.pyx>=2.0.0', 'pyaes'], - python_requires='>=3.3', + python_requires='>=3.6', zip_safe=False) diff --git a/jmclient/test/commontest.py b/jmclient/test/commontest.py index 6295b1713..e06eaf0f5 100644 --- a/jmclient/test/commontest.py +++ b/jmclient/test/commontest.py @@ -6,12 +6,11 @@ import random from decimal import Decimal -from jmbase import (get_log, hextobin, utxostr_to_utxo, - utxo_to_utxostr, listchanger, dictchanger) +from jmbase import (get_log, hextobin, dictchanger) from jmclient import ( jm_single, open_test_wallet_maybe, estimate_tx_fee, - BlockchainInterface, get_p2sh_vbyte, BIP32Wallet, + BlockchainInterface, BIP32Wallet, SegwitLegacyWallet, WalletService, BTC_P2SH_P2WPKH) from jmbase.support import chunks import jmbitcoin as btc diff --git a/jmclient/test/test_client_protocol.py b/jmclient/test/test_client_protocol.py index 7583a555f..96ae0c309 100644 --- a/jmclient/test/test_client_protocol.py +++ b/jmclient/test/test_client_protocol.py @@ -1,7 +1,7 @@ #! /usr/bin/env python '''test client-protocol interfacae.''' -from jmbase import get_log, bintohex, hextobin +from jmbase import get_log, bintohex from jmbase.commands import * from jmclient import load_test_config, Taker,\ JMClientProtocolFactory, jm_single, Maker, WalletService diff --git a/jmclient/test/test_coinjoin.py b/jmclient/test/test_coinjoin.py index 810bf7151..7d93e4b9d 100644 --- a/jmclient/test/test_coinjoin.py +++ b/jmclient/test/test_coinjoin.py @@ -9,7 +9,7 @@ import copy from twisted.internet import reactor -from jmbase import get_log, hextobin, bintohex +from jmbase import get_log, hextobin from jmclient import load_test_config, jm_single,\ YieldGeneratorBasic, Taker, LegacyWallet, SegwitLegacyWallet,\ NO_ROUNDING diff --git a/jmclient/test/test_maker.py b/jmclient/test/test_maker.py index a7046f2ef..28750ec38 100644 --- a/jmclient/test/test_maker.py +++ b/jmclient/test/test_maker.py @@ -1,8 +1,7 @@ #!/usr/bin/env python import jmbitcoin as btc -from jmclient import Maker, get_p2sh_vbyte, get_p2pk_vbyte, \ - load_test_config, jm_single, WalletService +from jmclient import Maker, load_test_config, jm_single, WalletService import jmclient from commontest import DummyBlockchainInterface from test_taker import DummyWallet diff --git a/jmclient/test/test_payjoin.py b/jmclient/test/test_payjoin.py index 6d0ba142a..634a9f45c 100644 --- a/jmclient/test/test_payjoin.py +++ b/jmclient/test/test_payjoin.py @@ -11,7 +11,7 @@ from jmclient import cryptoengine from jmclient import (load_test_config, jm_single, P2EPMaker, P2EPTaker, - LegacyWallet, SegwitLegacyWallet, SegwitWallet) + SegwitLegacyWallet, SegwitWallet) from commontest import make_wallets from test_coinjoin import make_wallets_to_list, create_orderbook, sync_wallets diff --git a/jmclient/test/test_podle.py b/jmclient/test/test_podle.py index a643b7e19..9601ebc4b 100644 --- a/jmclient/test/test_podle.py +++ b/jmclient/test/test_podle.py @@ -2,7 +2,6 @@ '''Tests of Proof of discrete log equivalence commitments.''' import os import jmbitcoin as bitcoin -import binascii import struct import json import pytest diff --git a/jmclient/test/test_privkeys.py b/jmclient/test/test_privkeys.py index 2fb6884eb..8c18c6b91 100644 --- a/jmclient/test/test_privkeys.py +++ b/jmclient/test/test_privkeys.py @@ -2,10 +2,8 @@ '''Public and private key validity and formatting tests.''' import jmbitcoin as btc -from jmclient import (BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH, - jm_single, load_test_config) -import binascii -import struct +from jmbase import hextobin +from jmclient import BTCEngine, jm_single, load_test_config import json import pytest import os @@ -76,7 +74,7 @@ def test_wif_privkeys_valid(setup_keys): # we only handle compressed keys continue from_wif_key, keytype = BTCEngine.wif_to_privkey(key) - expected_key = binascii.unhexlify(hex_key) + b"\x01" + expected_key = hextobin(hex_key) + b"\x01" assert from_wif_key == expected_key, "Incorrect key decoding: " + \ str(from_wif_key) + ", should be: " + str(expected_key) jm_single().config.set("BLOCKCHAIN", "network", "testnet") diff --git a/jmclient/test/test_psbt_wallet.py b/jmclient/test/test_psbt_wallet.py index 154bd8d81..f317bb4d3 100644 --- a/jmclient/test/test_psbt_wallet.py +++ b/jmclient/test/test_psbt_wallet.py @@ -6,15 +6,12 @@ BIP174 are tested there, not here. ''' -import time -import binascii -import struct import copy from commontest import make_wallets, dummy_accept_callback, dummy_info_callback import jmbitcoin as bitcoin import pytest -from jmbase import get_log, bintohex, hextobin +from jmbase import get_log, bintohex from jmclient import (load_test_config, jm_single, direct_send, SegwitLegacyWallet, SegwitWallet, LegacyWallet) diff --git a/jmclient/test/test_snicker.py b/jmclient/test/test_snicker.py index 8417a78b7..9ed3c880b 100644 --- a/jmclient/test/test_snicker.py +++ b/jmclient/test/test_snicker.py @@ -1,18 +1,14 @@ #! /usr/bin/env python -from __future__ import (absolute_import, division, - print_function, unicode_literals) -from builtins import * # noqa: F401 -'''Test of unusual transaction types creation and push to -network to check validity.''' +'''Test of SNICKER functionality using Joinmarket + wallets as defined in jmclient.wallet.''' -import binascii from commontest import make_wallets, dummy_accept_callback, dummy_info_callback import jmbitcoin as btc import pytest -from jmbase import get_log, bintohex, hextobin -from jmclient import (load_test_config, jm_single, - estimate_tx_fee, SNICKERReceiver, direct_send) +from jmbase import get_log, bintohex +from jmclient import (load_test_config, estimate_tx_fee, SNICKERReceiver, + direct_send) log = get_log() diff --git a/jmclient/test/test_taker.py b/jmclient/test/test_taker.py index 3cb0bd304..9d6471b44 100644 --- a/jmclient/test/test_taker.py +++ b/jmclient/test/test_taker.py @@ -10,8 +10,7 @@ import json import struct from base64 import b64encode -from jmbase import (utxostr_to_utxo, utxo_to_utxostr, hextobin, - dictchanger, listchanger) +from jmbase import utxostr_to_utxo, hextobin from jmclient import load_test_config, jm_single, set_commitment_file,\ get_commitment_file, SegwitLegacyWallet, Taker, VolatileStorage,\ get_network, WalletService, NO_ROUNDING, BTC_P2PKH diff --git a/jmclient/test/test_tx_creation.py b/jmclient/test/test_tx_creation.py index 7c1e75c7c..75f4d8a60 100644 --- a/jmclient/test/test_tx_creation.py +++ b/jmclient/test/test_tx_creation.py @@ -5,8 +5,6 @@ p2(w)sh tests, these have been removed since Joinmarket does not use this feature.''' -import time -import binascii import struct from binascii import unhexlify from commontest import make_wallets, make_sign_and_push, ensure_bip65_activated @@ -14,8 +12,7 @@ import jmbitcoin as bitcoin import pytest from jmbase import get_log -from jmclient import load_test_config, jm_single,\ - get_p2pk_vbyte +from jmclient import load_test_config, jm_single log = get_log() #just a random selection of pubkeys for receiving multisigs; diff --git a/jmclient/test/test_wallet.py b/jmclient/test/test_wallet.py index e0e0e4a14..c19a8cf98 100644 --- a/jmclient/test/test_wallet.py +++ b/jmclient/test/test_wallet.py @@ -7,8 +7,7 @@ import pytest import jmbitcoin as btc from commontest import ensure_bip65_activated -from jmbase import (get_log, utxostr_to_utxo, utxo_to_utxostr, - hextobin, bintohex) +from jmbase import get_log, hextobin from jmclient import load_test_config, jm_single, \ SegwitLegacyWallet,BIP32Wallet, BIP49Wallet, LegacyWallet,\ VolatileStorage, get_network, cryptoengine, WalletError,\ diff --git a/jmclient/test/test_wallets.py b/jmclient/test/test_wallets.py index 7344ead3e..6450d3794 100644 --- a/jmclient/test/test_wallets.py +++ b/jmclient/test/test_wallets.py @@ -9,7 +9,6 @@ import pytest from jmbase import get_log, hextobin -import jmbitcoin as btc from jmclient import ( load_test_config, jm_single, estimate_tx_fee, BitcoinCoreInterface, Mnemonic) diff --git a/jmdaemon/jmdaemon/daemon_protocol.py b/jmdaemon/jmdaemon/daemon_protocol.py index 920e5e004..f6e754432 100644 --- a/jmdaemon/jmdaemon/daemon_protocol.py +++ b/jmdaemon/jmdaemon/daemon_protocol.py @@ -1,5 +1,4 @@ #! /usr/bin/env python -from future.utils import iteritems from .message_channel import MessageChannelCollection from .orderbookwatch import OrderbookWatch @@ -239,7 +238,7 @@ def on_JM_FILL(self, amount, commitment, revelation, filled_offers): #Reset utxo data to null for this new transaction self.ioauth_data = {} self.active_orders = json.loads(filled_offers) - for nick, offer_dict in iteritems(self.active_orders): + for nick, offer_dict in self.active_orders.items(): offer_fill_msg = " ".join([str(offer_dict["oid"]), str(amount), self.kp.hex_pk().decode('ascii'), str(commitment)]) self.mcc.prepare_privmsg(nick, "fill", offer_fill_msg) diff --git a/jmdaemon/jmdaemon/irc.py b/jmdaemon/jmdaemon/irc.py index 9e75037ce..c91fb83a3 100644 --- a/jmdaemon/jmdaemon/irc.py +++ b/jmdaemon/jmdaemon/irc.py @@ -17,7 +17,6 @@ def wlog(*x): """Simplifier to add lists to the debug log """ def conv(s): - # note: this only works because of the future package if isinstance(s, str): return s elif isinstance(s, bytes): diff --git a/jmdaemon/jmdaemon/message_channel.py b/jmdaemon/jmdaemon/message_channel.py index 70cd1e639..a5cbe6cfa 100644 --- a/jmdaemon/jmdaemon/message_channel.py +++ b/jmdaemon/jmdaemon/message_channel.py @@ -1,5 +1,4 @@ #! /usr/bin/env python -from future.utils import iteritems import abc import base64 import binascii @@ -334,7 +333,7 @@ def fill_orders(self, nick_order_dict, cj_amount, taker_pubkey, commitment): """ for mc in self.available_channels(): filtered_nick_order_dict = {k: v - for k, v in iteritems(nick_order_dict) + for k, v in nick_order_dict.items() if mc == self.active_channels[k]} mc.fill_orders(filtered_nick_order_dict, cj_amount, taker_pubkey, commitment) @@ -370,7 +369,7 @@ def send_tx(self, nick_list, txhex): tx_nick_sets[self.active_channels[nick]] = [nick] else: tx_nick_sets[self.active_channels[nick]].append(nick) - for mc, nl in iteritems(tx_nick_sets): + for mc, nl in tx_nick_sets.items(): self.prepare_send_tx(mc, nl, txhex) def prepare_send_tx(self, mc, nick_list, txhex): @@ -824,7 +823,7 @@ def request_orderbook(self): # Taker callbacks def fill_orders(self, nick_order_dict, cj_amount, taker_pubkey, commitment): - for c, order in iteritems(nick_order_dict): + for c, order in nick_order_dict.items(): msg = str(order['oid']) + ' ' + str(cj_amount) + ' ' + taker_pubkey msg += ' ' + commitment self.privmsg(c, 'fill', msg) diff --git a/jmdaemon/setup.py b/jmdaemon/setup.py index f93a61485..5279daea0 100644 --- a/jmdaemon/setup.py +++ b/jmdaemon/setup.py @@ -9,6 +9,6 @@ author_email='', license='GPL', packages=['jmdaemon'], - install_requires=['future', 'txtorcon', 'pyopenssl', 'libnacl', 'joinmarketbase==0.7.0dev'], - python_requires='>=3.3', + install_requires=['txtorcon', 'pyopenssl', 'libnacl', 'joinmarketbase==0.7.0dev'], + python_requires='>=3.6', zip_safe=False) diff --git a/jmdaemon/test/test_daemon_protocol.py b/jmdaemon/test/test_daemon_protocol.py index 321d548fd..cf82107c3 100644 --- a/jmdaemon/test/test_daemon_protocol.py +++ b/jmdaemon/test/test_daemon_protocol.py @@ -1,5 +1,4 @@ #! /usr/bin/env python -from future.utils import iteritems '''test daemon-protocol interfacae.''' from jmdaemon import MessageChannelCollection @@ -254,7 +253,7 @@ def on_JM_FILL(self, amount, commitment, revelation, filled_offers): "!push abcd abc def", "3", "4", str(list(tmpfo.keys())[0]), 6, 7, self.mcc.mchannels[0].hostid) #send "valid" onpubkey, onioauth messages - for k, v in iteritems(tmpfo): + for k, v in tmpfo.items(): reactor.callLater(1, self.on_pubkey, k, dummypub) reactor.callLater(2, self.on_ioauth, k, ['a', 'b'], "auth_pub", "cj_addr", "change_addr", "btc_sig") diff --git a/scripts/add-utxo.py b/scripts/add-utxo.py index ea6de4448..0b7bbc7e0 100755 --- a/scripts/add-utxo.py +++ b/scripts/add-utxo.py @@ -11,10 +11,9 @@ import json import binascii from pprint import pformat - from optparse import OptionParser -import jmbitcoin as btc -from jmclient import load_program_config, jm_single, get_p2pk_vbyte,\ + +from jmclient import load_program_config, jm_single,\ open_wallet, WalletService, add_external_commitments, update_commitments,\ PoDLE, get_podle_commitments, get_utxo_info, validate_utxo_data, quit,\ get_wallet_path, add_base_options, BTCEngine, BTC_P2SH_P2WPKH diff --git a/scripts/joinmarket-qt.py b/scripts/joinmarket-qt.py index 12eee3a92..46f033e3c 100755 --- a/scripts/joinmarket-qt.py +++ b/scripts/joinmarket-qt.py @@ -66,7 +66,7 @@ from jmbase import get_log from jmbase.support import DUST_THRESHOLD, EXIT_FAILURE, JM_CORE_VERSION from jmclient import load_program_config, get_network, update_persist_config,\ - open_test_wallet_maybe, get_wallet_path, get_p2sh_vbyte, get_p2pk_vbyte,\ + open_test_wallet_maybe, get_wallet_path,\ jm_single, validate_address, weighted_order_choose, Taker,\ JMClientProtocolFactory, start_reactor, get_schedule, schedule_to_text,\ get_blockchain_interface_instance, direct_send, WalletService,\ diff --git a/scripts/sendtomany.py b/scripts/sendtomany.py index 33118a08e..12037bb68 100755 --- a/scripts/sendtomany.py +++ b/scripts/sendtomany.py @@ -10,9 +10,8 @@ import jmbitcoin as btc from jmbase import get_log, jmprint, bintohex, utxostr_to_utxo from jmclient import load_program_config, estimate_tx_fee, jm_single,\ - get_p2pk_vbyte, validate_address, get_utxo_info, add_base_options,\ - validate_utxo_data, quit, BTCEngine, BTC_P2SH_P2WPKH, BTC_P2PKH - + validate_address, get_utxo_info, add_base_options,\ + validate_utxo_data, quit, BTCEngine log = get_log() diff --git a/test/test_segwit.py b/test/test_segwit.py index b5cc7a4c7..6287fade3 100644 --- a/test/test_segwit.py +++ b/test/test_segwit.py @@ -1,13 +1,12 @@ #! /usr/bin/env python '''Test creation of segwit transactions.''' -import binascii import json from common import make_wallets from pprint import pformat import jmbitcoin as btc import pytest -from jmbase import get_log, hextobin, bintohex +from jmbase import get_log, hextobin from jmclient import load_test_config, jm_single, LegacyWallet log = get_log() @@ -25,14 +24,6 @@ def test_segwit_valid_txs(setup_segwit): #TODO use bcinterface to decoderawtransaction #and compare the json values - -def binarize_tx(tx): - for o in tx['outs']: - o['script'] = binascii.unhexlify(o['script']) - for i in tx['ins']: - i['outpoint']['hash'] = binascii.unhexlify(i['outpoint']['hash']) - - @pytest.mark.parametrize( "wallet_structure, in_amt, amount, segwit_amt, segwit_ins, o_ins", [ ([[1, 0, 0, 0, 0]], 1, 1000000, 1, [0, 1, 2], []), From ad459d2fdb7b547116553a8e5ad7bb5c06918a10 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Fri, 1 May 2020 19:07:01 +0100 Subject: [PATCH 07/15] Add human readable representations of txs and PSBTs Human readable representation for CTransaction objects in jmbitcoin.secp256k1_transaction.py and for PartiallySignedTransaction objects in jmclient.wallet. PSBTWalletMixin, use of these in maker, taker, direct send and in tests. Users should note that PSBT human readable representations can in some cases be really huge. --- jmbitcoin/jmbitcoin/__init__.py | 3 +- jmbitcoin/jmbitcoin/secp256k1_transaction.py | 73 +++++++++++- jmbitcoin/test/test_tx_signing.py | 6 +- jmclient/jmclient/maker.py | 4 +- jmclient/jmclient/taker.py | 4 +- jmclient/jmclient/taker_utils.py | 11 +- jmclient/jmclient/wallet.py | 114 ++++++++++++++++++- jmclient/test/test_psbt_wallet.py | 61 +++++++++- jmclient/test/test_snicker.py | 2 +- 9 files changed, 253 insertions(+), 25 deletions(-) diff --git a/jmbitcoin/jmbitcoin/__init__.py b/jmbitcoin/jmbitcoin/__init__.py index b8efa5a6c..ae2e0e2bd 100644 --- a/jmbitcoin/jmbitcoin/__init__.py +++ b/jmbitcoin/jmbitcoin/__init__.py @@ -13,5 +13,6 @@ from bitcointx.core.key import KeyStore from bitcointx.core.script import (CScript, OP_0, SignatureHash, SIGHASH_ALL, SIGVERSION_WITNESS_V0, CScriptWitness) -from bitcointx.core.psbt import PartiallySignedTransaction +from bitcointx.core.psbt import (PartiallySignedTransaction, PSBT_Input, + PSBT_Output) diff --git a/jmbitcoin/jmbitcoin/secp256k1_transaction.py b/jmbitcoin/jmbitcoin/secp256k1_transaction.py index 0966f12e9..cf988fb00 100644 --- a/jmbitcoin/jmbitcoin/secp256k1_transaction.py +++ b/jmbitcoin/jmbitcoin/secp256k1_transaction.py @@ -2,16 +2,81 @@ # note, only used for non-cryptographic randomness: import random +import json from jmbitcoin.secp256k1_main import * - +from jmbase import bintohex, utxo_to_utxostr from bitcointx.core import (CMutableTransaction, Hash160, CTxInWitness, - CMutableOutPoint, CMutableTxIn, - CMutableTxOut, ValidationError) + CMutableOutPoint, CMutableTxIn, CTransaction, + CMutableTxOut, CTxIn, CTxOut, ValidationError) from bitcointx.core.script import * -from bitcointx.wallet import P2WPKHCoinAddress, CCoinAddress, P2PKHCoinAddress +from bitcointx.wallet import (P2WPKHCoinAddress, CCoinAddress, P2PKHCoinAddress, + CCoinAddressError) from bitcointx.core.scripteval import (VerifyScript, SCRIPT_VERIFY_WITNESS, SCRIPT_VERIFY_P2SH, SIGVERSION_WITNESS_V0) +def hrt(tx, jsonified=True): + """ Given a CTransaction object, output a human + readable json-formatted string (suitable for terminal + output or large GUI textbox display) containing + all details of that transaction. + If `jsonified` is False, the dict is returned, instead + of the json string. + """ + assert isinstance(tx, CTransaction) + outdict = {} + outdict["hex"] = bintohex(tx.serialize()) + outdict["inputs"]=[] + outdict["outputs"]=[] + outdict["txid"]= bintohex(tx.GetTxid()[::-1]) + outdict["nLockTime"] = tx.nLockTime + outdict["nVersion"] = tx.nVersion + for i, inp in enumerate(tx.vin): + if not tx.wit.vtxinwit: + # witness section is not initialized/empty + witarg = None + else: + witarg = tx.wit.vtxinwit[i] + outdict["inputs"].append(hrinp(inp, witarg)) + for i, out in enumerate(tx.vout): + outdict["outputs"].append(hrout(out)) + if not jsonified: + return outdict + return json.dumps(outdict, indent=4) + +def hrinp(txinput, txinput_witness): + """ Pass objects of type CTxIn and CTxInWitness (or None) + and a dict of human-readable entries for this input + is returned. + """ + assert isinstance(txinput, CTxIn) + outdict = {} + success, u = utxo_to_utxostr((txinput.prevout.hash[::-1], + txinput.prevout.n)) + assert success + outdict["outpoint"] = u + outdict["scriptSig"] = bintohex(txinput.scriptSig) + outdict["nSequence"] = txinput.nSequence + + if txinput_witness: + outdict["witness"] = bintohex( + txinput_witness.scriptWitness.serialize()) + return outdict + +def hrout(txoutput): + """ Returns a dict of human-readable entries + for this output. + """ + assert isinstance(txoutput, CTxOut) + outdict = {} + outdict["value_sats"] = txoutput.nValue + outdict["scriptPubKey"] = bintohex(txoutput.scriptPubKey) + try: + addr = CCoinAddress.from_scriptPubKey(txoutput.scriptPubKey) + outdict["address"] = str(addr) + except CCoinAddressError: + pass # non standard script + return outdict + def estimate_tx_size(ins, outs, txtype='p2pkh'): '''Estimate transaction size. The txtype field as detailed below is used to distinguish diff --git a/jmbitcoin/test/test_tx_signing.py b/jmbitcoin/test/test_tx_signing.py index 05cbbf521..d7f3d6f67 100644 --- a/jmbitcoin/test/test_tx_signing.py +++ b/jmbitcoin/test/test_tx_signing.py @@ -4,6 +4,7 @@ import binascii import hashlib +from jmbase import bintohex import jmbitcoin as btc @pytest.mark.parametrize( @@ -58,8 +59,9 @@ def test_sign_standard_txs(addrtype): if not sig: print(msg) raise - print("created signature: ", binascii.hexlify(sig)) - print("serialized transaction: {}".format(btc.b2x(tx.serialize()))) + print("created signature: ", bintohex(sig)) + print("serialized transaction: {}".format(bintohex(tx.serialize()))) + print("deserialized transaction: {}\n".format(btc.hrt(tx))) def test_mk_shuffled_tx(): # prepare two addresses for the outputs diff --git a/jmclient/jmclient/maker.py b/jmclient/jmclient/maker.py index b4b9ab6c3..40ffc3ad6 100644 --- a/jmclient/jmclient/maker.py +++ b/jmclient/jmclient/maker.py @@ -130,7 +130,9 @@ def on_tx_received(self, nick, tx_from_taker, offerinfo): tx = btc.CMutableTransaction.deserialize(tx_from_taker) except Exception as e: return (False, 'malformed txhex. ' + repr(e)) - jlog.info('obtained tx\n' + bintohex(tx.serialize())) + # if the above deserialization was successful, the human readable + # parsing will be also: + jlog.info('obtained tx\n' + btc.hrt(tx)) goodtx, errmsg = self.verify_unsigned_tx(tx, offerinfo) if not goodtx: jlog.info('not a good tx, reason=' + errmsg) diff --git a/jmclient/jmclient/taker.py b/jmclient/jmclient/taker.py index a4f13cba7..25d5d1a90 100644 --- a/jmclient/jmclient/taker.py +++ b/jmclient/jmclient/taker.py @@ -499,7 +499,7 @@ def receive_utxos(self, ioauth_data): self.outputs.append({'address': self.coinjoin_address(), 'value': self.cjamount}) self.latest_tx = btc.make_shuffled_tx(self.utxo_tx, self.outputs) - jlog.info('obtained tx\n' + bintohex(self.latest_tx.serialize())) + jlog.info('obtained tx\n' + btc.hrt(self.latest_tx)) for index, ins in enumerate(self.latest_tx.vin): utxo = (ins.prevout.hash[::-1], ins.prevout.n) @@ -1008,7 +1008,7 @@ def receive_utxos(self, ioauth_data): # contains only those. tx = btc.make_shuffled_tx(self.input_utxos, self.outputs, version=2, locktime=compute_tx_locktime()) - jlog.info('Created proposed fallback tx\n' + pprint.pformat(str(tx))) + jlog.info('Created proposed fallback tx:\n' + btc.hrt(tx)) # We now sign as a courtesy, because if we disappear the recipient # can still claim his coins with this. # sign our inputs before transfer diff --git a/jmclient/jmclient/taker_utils.py b/jmclient/jmclient/taker_utils.py index b6c488620..b2bed0e7d 100644 --- a/jmclient/jmclient/taker_utils.py +++ b/jmclient/jmclient/taker_utils.py @@ -11,7 +11,7 @@ from .wallet import BaseWallet, estimate_tx_fee, compute_tx_locktime, \ FidelityBondMixin from jmbitcoin import make_shuffled_tx, amount_to_str, mk_burn_script,\ - PartiallySignedTransaction, CMutableTxOut + PartiallySignedTransaction, CMutableTxOut, hrt from jmbase.support import EXIT_SUCCESS log = get_log() @@ -159,8 +159,7 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, return False new_psbt_signed = PartiallySignedTransaction.deserialize(serialized_psbt) print("Completed PSBT created: ") - print(pformat(new_psbt_signed)) - # TODO add more readable info here as for case below. + print(wallet_service.hr_psbt(new_psbt_signed)) return new_psbt_signed else: success, msg = wallet_service.sign_tx(tx, inscripts) @@ -168,9 +167,7 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, log.error("Failed to sign transaction, quitting. Error msg: " + msg) return log.info("Got signed transaction:\n") - log.info(pformat(str(tx))) - log.info("In serialized form (for copy-paste):") - log.info(bintohex(tx.serialize())) + log.info(hrt(tx)) actual_amount = amount if amount != 0 else total_inputs_val - fee_est log.info("Sends: " + amount_to_str(actual_amount) + " to destination: " + destination) if not answeryes: @@ -179,7 +176,7 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, log.info("You chose not to broadcast the transaction, quitting.") return False else: - accepted = accept_callback(pformat(str(tx)), destination, actual_amount, + accepted = accept_callback(hrt(tx), destination, actual_amount, fee_est) if not accepted: return False diff --git a/jmclient/jmclient/wallet.py b/jmclient/jmclient/wallet.py index cd15f5585..fb46a99ca 100644 --- a/jmclient/jmclient/wallet.py +++ b/jmclient/jmclient/wallet.py @@ -6,6 +6,7 @@ import numbers import random import base64 +import json from binascii import hexlify, unhexlify from datetime import datetime from calendar import timegm @@ -28,7 +29,7 @@ from .support import get_random_bytes from . import mn_encode, mn_decode import jmbitcoin as btc -from jmbase import JM_WALLET_NAME_PREFIX +from jmbase import JM_WALLET_NAME_PREFIX, bintohex """ @@ -1006,6 +1007,117 @@ class PSBTWalletMixin(object): def __init__(self, storage, **kwargs): super(PSBTWalletMixin, self).__init__(storage, **kwargs) + @staticmethod + def hr_psbt(in_psbt): + """ Returns a jsonified indented string with all relevant + information, in human readable form, contained in a PSBT. + Warning: the output can be very verbose in certain cases. + """ + assert isinstance(in_psbt, btc.PartiallySignedTransaction) + outdict = {} + outdict["psbt-version"] = in_psbt.version + + # human readable serialization of these three global fields is for + # now on a "best-effort" basis, i.e. just takes the representation + # provided by the underlying classes in bitcointx, though this may + # not be very readable. + # TODO: Improve proprietary/unknown as needed. + if in_psbt.xpubs: + outdict["xpubs"] = {str(k): bintohex( + v.serialize()) for k, v in in_psbt.xpubs.items()} + if in_psbt.proprietary_fields: + outdict["proprietary-fields"] = str(in_psbt.proprietary_fields) + if in_psbt.unknown_fields: + outdict["unknown-fields"] = str(in_psbt.unknown_fields) + + outdict["unsigned-tx"] = btc.hrt(in_psbt.unsigned_tx, jsonified=False) + outdict["psbt-inputs"] = [] + for inp in in_psbt.inputs: + outdict["psbt-inputs"].append(PSBTWalletMixin.hr_psbt_in(inp)) + outdict["psbt-outputs"] = [] + for out in in_psbt.outputs: + outdict["psbt-outputs"].append(PSBTWalletMixin.hr_psbt_out(out)) + return json.dumps(outdict, indent=4) + + @staticmethod + def hr_psbt_in(psbt_input): + """ Returns a dict containing human readable information + about a bitcointx.core.psbt.PSBT_Input object. + """ + assert isinstance(psbt_input, btc.PSBT_Input) + outdict = {} + if psbt_input.index is not None: + outdict["input-index"] = psbt_input.index + if psbt_input.utxo: + if isinstance(psbt_input.utxo, btc.CTxOut): + outdict["utxo"] = btc.hrout(psbt_input.utxo) + elif isinstance(psbt_input.utxo, btc.CTransaction): + # human readable full transaction is *too* verbose: + outdict["utxo"] = bintohex(psbt_input.utxo.serialize()) + else: + assert False, "invalid PSBT Input utxo field." + if psbt_input.sighash_type: + outdict["sighash-type"] = psbt_input.sighash_type + if psbt_input.redeem_script: + outdict["redeem-script"] = bintohex(psbt_input.redeem_script) + if psbt_input.witness_script: + outdict["witness-script"] = bintohex(psbt_input.witness_script) + if psbt_input.partial_sigs: + # convert the dict entries to hex: + outdict["partial-sigs"] = {bintohex(k): bintohex(v) for k,v in \ + psbt_input.partial_sigs.items()} + # Note we do not currently add derivation info to our own inputs, + # but probably will in future ( TODO ), still this is shown for + # externally generated PSBTs: + if psbt_input.derivation_map: + # TODO it would be more useful to print the indexes of the + # derivation path as integers, than 4 byte hex: + outdict["derivation-map"] = {bintohex(k): bintohex(v.serialize( + )) for k, v in psbt_input.derivation_map.items()} + + # we show these fields on a best-effort basis; same comment as for + # globals section as mentioned in hr_psbt() + if psbt_input.proprietary_fields: + outdict["proprietary-fields"] = str(psbt_input.proprietary_fields) + if psbt_input.unknown_fields: + outdict["unknown-fields"] = str(psbt_input.unknown_fields) + if psbt_input.proof_of_reserves_commitment: + outdict["proof-of-reserves-commitment"] = \ + str(psbt_input.proof_of_reserves_commitment) + + outdict["final-scriptSig"] = bintohex(psbt_input.final_script_sig) + outdict["final-scriptWitness"] = bintohex( + psbt_input.final_script_witness.serialize()) + + return outdict + + @staticmethod + def hr_psbt_out(psbt_output): + """ Returns a dict containing human readable information + about a PSBT_Output object. + """ + assert isinstance(psbt_output, btc.PSBT_Output) + outdict = {} + if psbt_output.index is not None: + outdict["output-index"] = psbt_output.index + + if psbt_output.derivation_map: + # See note to derivation map in hr_psbt_in() + outdict["derivation-map"] = {bintohex(k): bintohex(v.serialize( + )) for k, v in psbt_output.derivation_map.items()} + + if psbt_output.redeem_script: + outdict["redeem-script"] = bintohex(psbt_output.redeem_script) + if psbt_output.witness_script: + outdict["witness-script"] = bintohex(psbt_output.witness_script) + + if psbt_output.proprietary_fields: + outdict["proprietary-fields"] = str(psbt_output.proprietary_fields) + if psbt_output.unknown_fields: + outdict["unknown-fields"] = str(psbt_output.unknown_fields) + + return outdict + @staticmethod def witness_utxos_to_psbt_utxos(utxos): """ Given a dict of utxos as returned from select_utxos, diff --git a/jmclient/test/test_psbt_wallet.py b/jmclient/test/test_psbt_wallet.py index f317bb4d3..18f88d744 100644 --- a/jmclient/test/test_psbt_wallet.py +++ b/jmclient/test/test_psbt_wallet.py @@ -11,10 +11,10 @@ import jmbitcoin as bitcoin import pytest -from jmbase import get_log, bintohex +from jmbase import get_log, bintohex, hextobin from jmclient import (load_test_config, jm_single, direct_send, SegwitLegacyWallet, SegwitWallet, LegacyWallet) - +from jmclient.wallet import PSBTWalletMixin log = get_log() def test_create_and_sign_psbt_with_legacy(setup_psbt_wallet): @@ -130,6 +130,8 @@ def test_create_psbt_and_sign(setup_psbt_wallet, unowned_utxo, wallet_cls): if unowned_utxo: newpsbt.inputs[-1].redeem_script = redeem_script print(bintohex(newpsbt.serialize())) + print("human readable: ") + print(wallet_service.hr_psbt(newpsbt)) # we cannot compare with a fixed expected result due to wallet randomization, but we can # check psbt structure: expected_inputs_length = 3 if unowned_utxo else 2 @@ -164,9 +166,9 @@ def test_create_psbt_and_sign(setup_psbt_wallet, unowned_utxo, wallet_cls): @pytest.mark.parametrize('payment_amt, wallet_cls_sender, wallet_cls_receiver', [ (0.05, SegwitLegacyWallet, SegwitLegacyWallet), - (0.95, SegwitLegacyWallet, SegwitWallet), - (0.05, SegwitWallet, SegwitLegacyWallet), - (0.95, SegwitWallet, SegwitWallet), + #(0.95, SegwitLegacyWallet, SegwitWallet), + #(0.05, SegwitWallet, SegwitLegacyWallet), + #(0.95, SegwitWallet, SegwitWallet), ]) def test_payjoin_workflow(setup_psbt_wallet, payment_amt, wallet_cls_sender, wallet_cls_receiver): @@ -211,6 +213,8 @@ def test_payjoin_workflow(setup_psbt_wallet, payment_amt, wallet_cls_sender, info_callback=dummy_info_callback, with_final_psbt=True) + print("Initial payment PSBT created:\n{}".format(wallet_s.hr_psbt( + payment_psbt))) # ensure that the payemnt amount is what was intended: out_amts = [x.nValue for x in payment_psbt.unsigned_tx.vout] # NOTE this would have to change for more than 2 outputs: @@ -274,7 +278,7 @@ def test_payjoin_workflow(setup_psbt_wallet, payment_amt, wallet_cls_sender, version=payment_psbt.unsigned_tx.nVersion, locktime=payment_psbt.unsigned_tx.nLockTime) print("we created this unsigned tx: ") - print(unsigned_payjoin_tx) + print(bitcoin.hrt(unsigned_payjoin_tx)) # to create the PSBT we need the spent_outs for each input, # in the right order: spent_outs = [] @@ -301,6 +305,9 @@ def test_payjoin_workflow(setup_psbt_wallet, payment_amt, wallet_cls_sender, r_payjoin_psbt = wallet_r.create_psbt_from_tx(unsigned_payjoin_tx, spent_outs=spent_outs) + print("Receiver created payjoin PSBT:\n{}".format( + wallet_r.hr_psbt(r_payjoin_psbt))) + signresultandpsbt, err = wallet_r.sign_psbt(r_payjoin_psbt.serialize(), with_sign_result=True) assert not err, err @@ -308,6 +315,9 @@ def test_payjoin_workflow(setup_psbt_wallet, payment_amt, wallet_cls_sender, assert signresult.num_inputs_final == len(receiver_utxos) assert not signresult.is_final + print("Receiver signing successful. Payjoin PSBT is now:\n{}".format( + wallet_r.hr_psbt(receiver_signed_psbt))) + # *** STEP 3 *** # ************** @@ -317,11 +327,50 @@ def test_payjoin_workflow(setup_psbt_wallet, payment_amt, wallet_cls_sender, receiver_signed_psbt.serialize(), with_sign_result=True) assert not err, err signresult, sender_signed_psbt = signresultandpsbt + print("Sender's final signed PSBT is:\n{}".format( + wallet_s.hr_psbt(sender_signed_psbt))) assert signresult.is_final + # broadcast the tx extracted_tx = sender_signed_psbt.extract_transaction().serialize() assert jm_single().bc_interface.pushtx(extracted_tx) +""" test vector data for human readable parsing only, +they are taken from bitcointx/tests/test_psbt.py and in turn +taken from BIP174 test vectors. +TODO add more, but note we are not testing functionality here. +""" +hr_test_vectors = { +# PSBT with one P2PKH input. Outputs are empty +"one-p2pkh": '70736274ff0100750200000001268171371edff285e937adeea4b37b78000c0566cbb3ad64641713ca42171bf60000000000feffffff02d3dff505000000001976a914d0c59903c5bac2868760e90fd521a4665aa7652088ac00e1f5050000000017a9143545e6e33b832c47050f24d3eeb93c9c03948bc787b32e1300000100fda5010100000000010289a3c71eab4d20e0371bbba4cc698fa295c9463afa2e397f8533ccb62f9567e50100000017160014be18d152a9b012039daf3da7de4f53349eecb985ffffffff86f8aa43a71dff1448893a530a7237ef6b4608bbb2dd2d0171e63aec6a4890b40100000017160014fe3e9ef1a745e974d902c4355943abcb34bd5353ffffffff0200c2eb0b000000001976a91485cff1097fd9e008bb34af709c62197b38978a4888ac72fef84e2c00000017a914339725ba21efd62ac753a9bcd067d6c7a6a39d05870247304402202712be22e0270f394f568311dc7ca9a68970b8025fdd3b240229f07f8a5f3a240220018b38d7dcd314e734c9276bd6fb40f673325bc4baa144c800d2f2f02db2765c012103d2e15674941bad4a996372cb87e1856d3652606d98562fe39c5e9e7e413f210502483045022100d12b852d85dcd961d2f5f4ab660654df6eedcc794c0c33ce5cc309ffb5fce58d022067338a8e0e1725c197fb1a88af59f51e44e4255b20167c8684031c05d1f2592a01210223b72beef0965d10be0778efecd61fcac6f79a4ea169393380734464f84f2ab300000000000000', +# PSBT with one P2PKH input and one P2SH-P2WPKH input. +# First input is signed and finalized. Outputs are empty +"first-input-signed": '70736274ff0100a00200000002ab0949a08c5af7c49b8212f417e2f15ab3f5c33dcf153821a8139f877a5b7be40000000000feffffffab0949a08c5af7c49b8212f417e2f15ab3f5c33dcf153821a8139f877a5b7be40100000000feffffff02603bea0b000000001976a914768a40bbd740cbe81d988e71de2a4d5c71396b1d88ac8e240000000000001976a9146f4620b553fa095e721b9ee0efe9fa039cca459788ac000000000001076a47304402204759661797c01b036b25928948686218347d89864b719e1f7fcf57d1e511658702205309eabf56aa4d8891ffd111fdf1336f3a29da866d7f8486d75546ceedaf93190121035cdc61fc7ba971c0b501a646a2a83b102cb43881217ca682dc86e2d73fa882920001012000e1f5050000000017a9143545e6e33b832c47050f24d3eeb93c9c03948bc787010416001485d13537f2e265405a34dbafa9e3dda01fb82308000000', +# PSBT with one P2PKH input which has a non-final scriptSig +# and has a sighash type specified. Outputs are empty +"nonfinal-scriptsig": '70736274ff0100750200000001268171371edff285e937adeea4b37b78000c0566cbb3ad64641713ca42171bf60000000000feffffff02d3dff505000000001976a914d0c59903c5bac2868760e90fd521a4665aa7652088ac00e1f5050000000017a9143545e6e33b832c47050f24d3eeb93c9c03948bc787b32e1300000100fda5010100000000010289a3c71eab4d20e0371bbba4cc698fa295c9463afa2e397f8533ccb62f9567e50100000017160014be18d152a9b012039daf3da7de4f53349eecb985ffffffff86f8aa43a71dff1448893a530a7237ef6b4608bbb2dd2d0171e63aec6a4890b40100000017160014fe3e9ef1a745e974d902c4355943abcb34bd5353ffffffff0200c2eb0b000000001976a91485cff1097fd9e008bb34af709c62197b38978a4888ac72fef84e2c00000017a914339725ba21efd62ac753a9bcd067d6c7a6a39d05870247304402202712be22e0270f394f568311dc7ca9a68970b8025fdd3b240229f07f8a5f3a240220018b38d7dcd314e734c9276bd6fb40f673325bc4baa144c800d2f2f02db2765c012103d2e15674941bad4a996372cb87e1856d3652606d98562fe39c5e9e7e413f210502483045022100d12b852d85dcd961d2f5f4ab660654df6eedcc794c0c33ce5cc309ffb5fce58d022067338a8e0e1725c197fb1a88af59f51e44e4255b20167c8684031c05d1f2592a01210223b72beef0965d10be0778efecd61fcac6f79a4ea169393380734464f84f2ab30000000001030401000000000000', +# PSBT with one P2PKH input and one P2SH-P2WPKH input both with +# non-final scriptSigs. P2SH-P2WPKH input's redeemScript is available. +# Outputs filled. +"mixed-inputs-nonfinal": '70736274ff0100a00200000002ab0949a08c5af7c49b8212f417e2f15ab3f5c33dcf153821a8139f877a5b7be40000000000feffffffab0949a08c5af7c49b8212f417e2f15ab3f5c33dcf153821a8139f877a5b7be40100000000feffffff02603bea0b000000001976a914768a40bbd740cbe81d988e71de2a4d5c71396b1d88ac8e240000000000001976a9146f4620b553fa095e721b9ee0efe9fa039cca459788ac00000000000100df0200000001268171371edff285e937adeea4b37b78000c0566cbb3ad64641713ca42171bf6000000006a473044022070b2245123e6bf474d60c5b50c043d4c691a5d2435f09a34a7662a9dc251790a022001329ca9dacf280bdf30740ec0390422422c81cb45839457aeb76fc12edd95b3012102657d118d3357b8e0f4c2cd46db7b39f6d9c38d9a70abcb9b2de5dc8dbfe4ce31feffffff02d3dff505000000001976a914d0c59903c5bac2868760e90fd521a4665aa7652088ac00e1f5050000000017a9143545e6e33b832c47050f24d3eeb93c9c03948bc787b32e13000001012000e1f5050000000017a9143545e6e33b832c47050f24d3eeb93c9c03948bc787010416001485d13537f2e265405a34dbafa9e3dda01fb8230800220202ead596687ca806043edc3de116cdf29d5e9257c196cd055cf698c8d02bf24e9910b4a6ba670000008000000080020000800022020394f62be9df19952c5587768aeb7698061ad2c4a25c894f47d8c162b4d7213d0510b4a6ba6700000080010000800200008000', +# PSBT with one P2SH-P2WSH input of a 2-of-2 multisig, redeemScript, +# witnessScript, and keypaths are available. Contains one signature. +"2-2-multisig-p2wsh": '70736274ff0100550200000001279a2323a5dfb51fc45f220fa58b0fc13e1e3342792a85d7e36cd6333b5cbc390000000000ffffffff01a05aea0b000000001976a914ffe9c0061097cc3b636f2cb0460fa4fc427d2b4588ac0000000000010120955eea0b0000000017a9146345200f68d189e1adc0df1c4d16ea8f14c0dbeb87220203b1341ccba7683b6af4f1238cd6e97e7167d569fac47f1e48d47541844355bd4646304302200424b58effaaa694e1559ea5c93bbfd4a89064224055cdf070b6771469442d07021f5c8eb0fea6516d60b8acb33ad64ede60e8785bfb3aa94b99bdf86151db9a9a010104220020771fd18ad459666dd49f3d564e3dbc42f4c84774e360ada16816a8ed488d5681010547522103b1341ccba7683b6af4f1238cd6e97e7167d569fac47f1e48d47541844355bd462103de55d1e1dac805e3f8a58c1fbf9b94c02f3dbaafe127fefca4995f26f82083bd52ae220603b1341ccba7683b6af4f1238cd6e97e7167d569fac47f1e48d47541844355bd4610b4a6ba67000000800000008004000080220603de55d1e1dac805e3f8a58c1fbf9b94c02f3dbaafe127fefca4995f26f82083bd10b4a6ba670000008000000080050000800000', +# PSBT with unknown types in the inputs +"unknown-input-types": '70736274ff01003f0200000001ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000ffffffff010000000000000000036a010000000000000a0f0102030405060708090f0102030405060708090a0b0c0d0e0f0000', +# PSBT with `PSBT_GLOBAL_XPUB` +"global-xpub": '70736274ff01009d0100000002710ea76ab45c5cb6438e607e59cc037626981805ae9e0dfd9089012abb0be5350100000000ffffffff190994d6a8b3c8c82ccbcfb2fba4106aa06639b872a8d447465c0d42588d6d670000000000ffffffff0200e1f505000000001976a914b6bc2c0ee5655a843d79afedd0ccc3f7dd64340988ac605af405000000001600141188ef8e4ce0449eaac8fb141cbf5a1176e6a088000000004f010488b21e039e530cac800000003dbc8a5c9769f031b17e77fea1518603221a18fd18f2b9a54c6c8c1ac75cbc3502f230584b155d1c7f1cd45120a653c48d650b431b67c5b2c13f27d7142037c1691027569c503100008000000080000000800001011f00e1f5050000000016001433b982f91b28f160c920b4ab95e58ce50dda3a4a220203309680f33c7de38ea6a47cd4ecd66f1f5a49747c6ffb8808ed09039243e3ad5c47304402202d704ced830c56a909344bd742b6852dccd103e963bae92d38e75254d2bb424502202d86c437195df46c0ceda084f2a291c3da2d64070f76bf9b90b195e7ef28f77201220603309680f33c7de38ea6a47cd4ecd66f1f5a49747c6ffb8808ed09039243e3ad5c1827569c5031000080000000800000008000000000010000000001011f00e1f50500000000160014388fb944307eb77ef45197d0b0b245e079f011de220202c777161f73d0b7c72b9ee7bde650293d13f095bc7656ad1f525da5fd2e10b11047304402204cb1fb5f869c942e0e26100576125439179ae88dca8a9dc3ba08f7953988faa60220521f49ca791c27d70e273c9b14616985909361e25be274ea200d7e08827e514d01220602c777161f73d0b7c72b9ee7bde650293d13f095bc7656ad1f525da5fd2e10b1101827569c5031000080000000800000008000000000000000000000220202d20ca502ee289686d21815bd43a80637b0698e1fbcdbe4caed445f6c1a0a90ef1827569c50310000800000008000000080000000000400000000', +# PSBT with proprietary values +"proprietary-values": '70736274ff0100550200000001ab0949a08c5af7c49b8212f417e2f15ab3f5c33dcf153821a8139f877a5b7be40100000000feffffff018e240000000000001976a9146f4620b553fa095e721b9ee0efe9fa039cca459788ac0000000015fc0a676c6f62616c5f706678016d756c7469706c790563686965660001012000e1f5050000000017a9143545e6e33b832c47050f24d3eeb93c9c03948bc787010416001485d13537f2e265405a34dbafa9e3dda01fb823080ffc06696e5f706678fde80377686174056672616d650afc00fe40420f0061736b077361746f7368690012fc076f75745f706678feffffff01636f726e05746967657217fc076f75745f706678ffffffffffffffffff707570707905647269766500' +} + +def test_hr_psbt(setup_psbt_wallet): + bitcoin.select_chain_params("bitcoin") + for k, v in hr_test_vectors.items(): + print(PSBTWalletMixin.hr_psbt( + bitcoin.PartiallySignedTransaction.from_binary(hextobin(v)))) + bitcoin.select_chain_params("bitcoin/regtest") + @pytest.fixture(scope="module") def setup_psbt_wallet(): load_test_config() diff --git a/jmclient/test/test_snicker.py b/jmclient/test/test_snicker.py index 9ed3c880b..7d81e3373 100644 --- a/jmclient/test/test_snicker.py +++ b/jmclient/test/test_snicker.py @@ -45,7 +45,7 @@ def test_snicker_e2e(setup_snicker, nw, wallet_structures, assert tx, "Failed to spend from receiver wallet" print("Parent transaction OK. It was: ") - print(tx) + print(btc.hrt(tx)) wallet_r.process_new_tx(tx) # we must identify the receiver's output we're going to use; # it can be destination or change, that's up to the proposer From ca0de5c31200983ecf23ace12918971414ba36c1 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Mon, 4 May 2020 01:13:30 +0100 Subject: [PATCH 08/15] Add bip78 payjoin module and client-server test: See: https://github.com/bitcoin/bips/blob/master/bip-0078.mediawiki Adds a new module jmclient.payjoin which implements the full sender workflow from a BIP21 uri to a payjoin broadcast, state is managed in JMPayjoinManager, includes all checks as per documentation of btcpayserver (and later, BIP78). Added simple client and server implementations in test/payjoinclient.py and test/payjoinserver.py which allow a full end to end test on regtest. Add TLS support to payjoin tests: Note: the jmclient.payjoin module already supports TLS by default (Agent object), but here we add the ability to test without certificate verification. Both test/payjoinclient.py and test/payjoinserver.py now support TLS, but the server needs a key and certificate in its directory to run. Adds BIP78 payjoin option to sendpayment.py Users can use a bip21 uri with the "pj" field to send a payment to a remote server. Removes require_path_templates setting from KeyStore call in PSBTWalletMixin.sign_psbt --- jmbase/jmbase/__init__.py | 3 +- jmbase/jmbase/bytesprod.py | 22 ++ jmbitcoin/jmbitcoin/__init__.py | 2 + jmclient/jmclient/__init__.py | 2 +- jmclient/jmclient/payjoin.py | 503 +++++++++++++++++++++++++++++++ jmclient/jmclient/taker_utils.py | 8 +- jmclient/jmclient/wallet.py | 27 +- scripts/sendpayment.py | 32 +- test/payjoinclient.py | 49 +++ test/payjoinserver.py | 178 +++++++++++ 10 files changed, 816 insertions(+), 10 deletions(-) create mode 100644 jmbase/jmbase/bytesprod.py create mode 100644 jmclient/jmclient/payjoin.py create mode 100644 test/payjoinclient.py create mode 100644 test/payjoinserver.py diff --git a/jmbase/jmbase/__init__.py b/jmbase/jmbase/__init__.py index cdbbedb7b..e49ea0a7d 100644 --- a/jmbase/jmbase/__init__.py +++ b/jmbase/jmbase/__init__.py @@ -6,6 +6,7 @@ hextobin, lehextobin, utxostr_to_utxo, utxo_to_utxostr, EXIT_ARGERROR, EXIT_FAILURE, EXIT_SUCCESS, hexbin, dictchanger, listchanger, - cv, JM_WALLET_NAME_PREFIX, JM_APP_NAME) + JM_WALLET_NAME_PREFIX, JM_APP_NAME) +from .bytesprod import BytesProducer from .commands import * diff --git a/jmbase/jmbase/bytesprod.py b/jmbase/jmbase/bytesprod.py new file mode 100644 index 000000000..04e9fb703 --- /dev/null +++ b/jmbase/jmbase/bytesprod.py @@ -0,0 +1,22 @@ +""" See https://twistedmatrix.com/documents/current/web/howto/client.html +""" +from zope.interface import implementer + +from twisted.internet.defer import succeed +from twisted.web.iweb import IBodyProducer + +@implementer(IBodyProducer) +class BytesProducer(object): + def __init__(self, body): + self.body = body + self.length = len(body) + + def startProducing(self, consumer): + consumer.write(self.body) + return succeed(None) + + def pauseProducing(self): + pass + + def stopProducing(self): + pass \ No newline at end of file diff --git a/jmbitcoin/jmbitcoin/__init__.py b/jmbitcoin/jmbitcoin/__init__.py index ae2e0e2bd..6ab579f71 100644 --- a/jmbitcoin/jmbitcoin/__init__.py +++ b/jmbitcoin/jmbitcoin/__init__.py @@ -11,6 +11,8 @@ CMutableTransaction, Hash160, coins_to_satoshi, satoshi_to_coins) from bitcointx.core.key import KeyStore +from bitcointx.wallet import (P2SHCoinAddress, P2SHCoinAddressError, + P2WPKHCoinAddress, P2WPKHCoinAddressError) from bitcointx.core.script import (CScript, OP_0, SignatureHash, SIGHASH_ALL, SIGVERSION_WITNESS_V0, CScriptWitness) from bitcointx.core.psbt import (PartiallySignedTransaction, PSBT_Input, diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index f088b9455..d222478c2 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -56,7 +56,7 @@ from .maker import Maker, P2EPMaker from .yieldgenerator import YieldGenerator, YieldGeneratorBasic, ygmain from .snicker_receiver import SNICKERError, SNICKERReceiver - +from .payjoin import parse_payjoin_setup, send_payjoin # Set default logging handler to avoid "No handler found" warnings. try: diff --git a/jmclient/jmclient/payjoin.py b/jmclient/jmclient/payjoin.py new file mode 100644 index 000000000..9c116dde3 --- /dev/null +++ b/jmclient/jmclient/payjoin.py @@ -0,0 +1,503 @@ +from zope.interface import implementer +from twisted.internet import reactor +from twisted.web.client import (Agent, readBody, ResponseFailed, + BrowserLikePolicyForHTTPS) +from twisted.web.iweb import IPolicyForHTTPS +from twisted.internet.ssl import CertificateOptions +from twisted.web.http_headers import Headers + +import json +from pprint import pformat +from jmbase import BytesProducer +from .configure import get_log, jm_single +import jmbitcoin as btc +from .wallet import PSBTWalletMixin, SegwitLegacyWallet, SegwitWallet +from .wallet_service import WalletService +from .taker_utils import direct_send + +""" +For some documentation see: + https://github.com/btcpayserver/btcpayserver-doc/blob/master/Payjoin-spec.md + which is a delta to: + https://github.com/bitcoin/bips/blob/master/bip-0079.mediawiki +""" +log = get_log() + +""" This whitelister allows us to accept any cert for a specific + domain, and is to be used for testing only; the default Agent + behaviour of twisted.web.client.Agent for https URIs is + the correct one in production (i.e. uses local trust store). +""" +@implementer(IPolicyForHTTPS) +class WhitelistContextFactory(object): + def __init__(self, good_domains=None): + """ + :param good_domains: List of domains. The URLs must be in bytes + """ + if not good_domains: + self.good_domains = [] + else: + self.good_domains = good_domains + # by default, handle requests like a browser would + self.default_policy = BrowserLikePolicyForHTTPS() + + def creatorForNetloc(self, hostname, port): + # check if the hostname is in the the whitelist, + # otherwise return the default policy + if hostname in self.good_domains: + return CertificateOptions(verify=False) + return self.default_policy.creatorForNetloc(hostname, port) + +class JMPayjoinManager(object): + """ An encapsulation of state for an + ongoing Payjoin payment. Allows reporting + details of the outcome of a Payjoin attempt. + """ + + # enum such that progress can be + # reported + JM_PJ_NONE = 0 + JM_PJ_INIT = 1 + JM_PJ_PAYMENT_CREATED = 2 + JM_PJ_PAYMENT_SENT = 3 + JM_PJ_PARTIAL_RECEIVED = 4 + JM_PJ_PARTIAL_REJECTED = 5 + JM_PJ_PAYJOIN_COSIGNED = 6 + JM_PJ_PAYJOIN_BROADCAST = 7 + JM_PJ_PAYJOIN_BROADCAST_FAILED = 8 + + pj_state = JM_PJ_NONE + + def __init__(self, wallet_service, mixdepth, destination, + amount, server): + assert isinstance(wallet_service, WalletService) + # payjoin is not supported for non-segwit wallets: + assert isinstance(wallet_service.wallet, + (SegwitWallet, SegwitLegacyWallet)) + # our payjoin implementation requires PSBT + assert isinstance(wallet_service.wallet, PSBTWalletMixin) + self.wallet_service = wallet_service + # mixdepth from which payment is sourced + assert isinstance(mixdepth, int) + self.mixdepth = mixdepth + assert isinstance(destination, btc.CCoinAddress) + self.destination = destination + assert isinstance(amount, int) + assert amount > 0 + self.amount = amount + self.server = server + self.pj_state = self.JM_PJ_INIT + self.payment_tx = None + self.initial_psbt = None + self.payjoin_psbt = None + self.final_psbt = None + # change is initialized as None + # in case there is no change: + self.change_out = None + + def set_payment_tx_and_psbt(self, in_psbt): + assert isinstance(in_psbt, btc.PartiallySignedTransaction) + self.initial_psbt = in_psbt + # any failure here is a coding error, as it is fully + # under our control. + assert self.sanity_check_initial_payment() + self.pj_state = self.JM_PJ_PAYMENT_CREATED + + def sanity_check_initial_payment(self): + """ These checks are motivated by the checks specified + for the *receiver* in the btcpayserver implementation doc. + We want to make sure our payment isn't rejected. + We also sanity check that the payment details match + the initialization of this Manager object. + """ + # failure to extract tx should throw an error; + # this PSBT must be finalized and sane. + self.payment_tx = self.initial_psbt.extract_transaction() + + # inputs must all have witness utxo populated + for inp in self.initial_psbt.inputs: + if not inp.utxo and isinstance(inp.utxo, btc.CTxOut): + return False + + # check that there is no xpub or derivation info + if self.initial_psbt.xpubs: + return False + for inp in self.initial_psbt.inputs: + # derivation_map is an OrderedDict, if empty + # it will be counted as false: + if inp.derivation_map: + return False + for out in self.initial_psbt.outputs: + if out.derivation_map: + return False + + # TODO we can replicate the mempool check here for + # Core versions sufficiently high, also encapsulate + # it in bc_interface. + + # our logic requires no more than one change output + # for now: + found_payment = 0 + assert len(self.payment_tx.vout) in [1, 2] + for out in self.payment_tx.vout: + if out.nValue == self.amount and \ + btc.CCoinAddress.from_scriptPubKey( + out.scriptPubKey) == self.destination: + found_payment += 1 + else: + # store this for our balance check + # for receiver proposal + self.change_out = out + if not found_payment == 1: + return False + + return True + + def check_receiver_proposal(self, in_pbst, signed_psbt_for_fees): + """ This is the most security critical part of the + business logic of the payjoin. We must check in detail + that what the server proposes does not unfairly take money + from us, and also conforms to acceptable structure. + We perform the following checks of the receiver proposal: + 1. Check that there are more inputs (i.e. some were contributed). + 2. Does it contain our inputs, unchanged? + 3. Does it contain our payment output, with amount increased? + 4. Are the other inputs finalized, and of the correct type? + 5. Is the feerate unchanged within tolerance? + 6. Does it contain no xpub information or derivation information? + 7. Are the sequence numbers unchanged (and all the same) for the inputs? + 8. Is the nLockTime and version unchanged? + 9. Is the extra fee we pay in reduced change output less than a doubling? + + If all the above checks pass we will consider this valid, and cosign. + Returns: + (False, "reason for failure") + (True, None) + """ + assert isinstance(in_pbst, btc.PartiallySignedTransaction) + orig_psbt = self.initial_psbt + assert isinstance(orig_psbt, btc.PartiallySignedTransaction) + # 1 + if len(in_pbst.inputs) <= len(orig_psbt.inputs): + return (False, "Receiver did not contribute inputs to payjoin.") + # 2 + ourins = [(i.prevout.hash, i.prevout.n) for i in orig_psbt.unsigned_tx.vin] + found = [0] * len(ourins) + receiver_input_indices = [] + for i, inp in enumerate(in_pbst.unsigned_tx.vin): + for j, inp2 in enumerate(ourins): + if (inp.prevout.hash, inp.prevout.n) == inp2: + found[j] += 1 + else: + receiver_input_indices.append(i) + + if any([found[i] != 1 for i in range(len(found))]): + return (False, "Receiver proposed PSBT does not contain our inputs.") + # 3 + found = 0 + for out in in_pbst.unsigned_tx.vout: + if btc.CCoinAddress.from_scriptPubKey(out.scriptPubKey) == \ + self.destination and out.nValue >= self.amount: + found += 1 + if found != 1: + return (False, "Our payment output not found exactly once or " + "with wrong amount.") + # 4 + for ind in receiver_input_indices: + # check the input is finalized + if not self.wallet_service.is_input_finalized(in_pbst.inputs[ind]): + return (False, "receiver input is not finalized.") + # check the utxo field of the input and see if the + # scriptPubKey is of the right type. + spk = in_pbst.inputs[ind].utxo.scriptPubKey + if isinstance(self.wallet_service.wallet, SegwitLegacyWallet): + try: + btc.P2SHCoinAddress.from_scriptPubKey(spk) + except btc.P2SHCoinAddressError: + return (False, + "Receiver input type does not match ours.") + elif isinstance(self.wallet_service.wallet, SegwitWallet): + try: + btc.P2WPKHCoinAddress.from_scriptPubKey(spk) + except btc.P2WPKHCoinAddressError: + return (False, + "Receiver input type does not match ours.") + else: + assert False + # 5 + # To get the feerate of the psbt proposed, we use the already-signed + # version (so all witnesses filled in) to calculate its size, + # then compare that with the fee, and do the same for the + # pre-existing non-payjoin. + gffp = PSBTWalletMixin.get_fee_from_psbt + proposed_tx_fee = gffp(signed_psbt_for_fees) + nonpayjoin_tx_fee = gffp(self.initial_psbt) + proposed_tx_size = signed_psbt_for_fees.extract_transaction( + ).get_virtual_size() + nonpayjoin_tx_size = self.initial_psbt.extract_transaction( + ).get_virtual_size() + proposed_fee_rate = proposed_tx_fee / float(proposed_tx_size) + log.debug("proposed fee rate: " + str(proposed_fee_rate)) + nonpayjoin_fee_rate = nonpayjoin_tx_fee / float(nonpayjoin_tx_size) + log.debug("nonpayjoin fee rate: " + str(nonpayjoin_fee_rate)) + diff_rate = abs(proposed_fee_rate - nonpayjoin_fee_rate)/nonpayjoin_fee_rate + if diff_rate > 0.2: + log.error("Bad fee rate differential: " + str(diff_rate)) + return (False, "fee rate of payjoin tx is more than 20% different " + "from inital fee rate, rejecting.") + # 6 + if in_pbst.xpubs: + return (False, "Receiver proposal contains xpub information.") + # 7 + # we created all inputs with one sequence number, make sure everything + # agrees + # TODO - discussion with btcpayserver devs, docs will be updated, + # server will agree with client in future. For now disabling check + # (it's a very complicated issue, surprisingly!) + #seqno = self.initial_psbt.unsigned_tx.vin[0].nSequence + #for inp in in_pbst.unsigned_tx.vin: + # if inp.nSequence != seqno: + # return (False, "all sequence numbers are not the same.") + # 8 + if in_pbst.unsigned_tx.nLockTime != \ + self.initial_psbt.unsigned_tx.nLockTime: + return (False, "receiver proposal has altered nLockTime.") + if in_pbst.unsigned_tx.nVersion != \ + self.initial_psbt.unsigned_tx.nVersion: + return (False, "receiver proposal has altered nVersion.") + # 9 + if proposed_tx_fee >= 2 * nonpayjoin_tx_fee: + return (False, "receiver's tx fee is too large (possibly " + "too many extra inputs.") + # as well as the overall fee, check our pay-out specifically: + for out in in_pbst.unsigned_tx.vout: + if out.scriptPubKey == self.change_out.scriptPubKey: + found += 1 + if self.change_out.nValue - out.nValue > nonpayjoin_tx_fee: + return (False, "Our change output was reduced too much.") + return (True, None) + + def set_payjoin_psbt(self, in_psbt, signed_psbt_for_fees): + """ This is the PSBT as initially proposed + by the receiver, so we keep a copy of it in that + state. This must be a copy as the sig_psbt function + will update the mutable psbt it is given. + This must not be called until the psbt has passed + all sanity and validation checks. + """ + assert isinstance(in_psbt, btc.PartiallySignedTransaction) + assert isinstance(signed_psbt_for_fees, btc.PartiallySignedTransaction) + success, msg = self.check_receiver_proposal(in_psbt, + signed_psbt_for_fees) + if not success: + return (success, msg) + self.payjoin_psbt = in_psbt + self.pj_state = self.JM_PJ_PARTIAL_RECEIVED + return (True, None) + + def set_final_payjoin_psbt(self, in_psbt): + """ This is the PSBT after we have co-signed + it. If it is in a sane state, we update our state. + """ + assert isinstance(in_psbt, btc.PartiallySignedTransaction) + # note that this is the simplest way to check + # for finality and validity of PSBT: + assert in_psbt.extract_transaction() + self.final_psbt = in_psbt + self.pj_state = self.JM_PJ_PAYJOIN_COSIGNED + + def set_broadcast(self, success): + if success: + self.pj_state = self.JM_PJ_PAYJOIN_BROADCAST + else: + self.pj_state = self.JM_PJ_PAYJOIN_BROADCAST_FAILED + + def report(self, jsonified=False, verbose=False): + """ Returns a dict (optionally jsonified) containing + the following information (if they are + available): + * current status of Payjoin + * payment transaction (original, non payjoin) + * payjoin partial (PSBT) sent by receiver + * final payjoin transaction + * whether or not the payjoin transaction is + broadcast and/or confirmed. + If verbose is True, we include the full deserialization + of transactions and PSBTs, which is too verbose for GUI + display. + """ + reportdict = {"name:", "PAYJOIN STATUS REPORT"} + reportdict["status"] = self.pj_state # TODO: string + if self.payment_tx: + txdata = btc.hrt(self.payment_tx) + if verbose: + txdata = txdata["hex"] + reportdict["payment-tx"] = txdata + if self.payjoin_psbt: + psbtdata = PSBTWalletMixin.hr_psbt( + self.payjoin_psbt) if verbose else self.payjoin_psbt.to_base64() + reportdict["payjoin-proposed"] = psbtdata + if self.final_psbt: + finaldata = PSBTWalletMixin.hr_psbt( + self.final_psbt) if verbose else self.final_psbt.to_base64() + reportdict["payjoin-final"] = finaldata + if jsonified: + return json.dumps(reportdict, indent=4) + else: + return reportdict + +def parse_payjoin_setup(bip21_uri, wallet_service, mixdepth): + """ Takes the payment request data from the uri and returns a + JMPayjoinManager object initialised for that payment. + """ + assert btc.is_bip21_uri(bip21_uri), "invalid bip21 uri: " + bip21_uri + decoded = btc.decode_bip21_uri(bip21_uri) + + assert "amount" in decoded + assert "address" in decoded + assert "pj" in decoded + + amount = decoded["amount"] + destaddr = decoded["address"] + # this will throw for any invalid address: + destaddr = btc.CCoinAddress(destaddr) + server = decoded["pj"] + + return JMPayjoinManager(wallet_service, mixdepth, destaddr, amount, server) + +def send_payjoin(manager, accept_callback=None, + info_callback=None, tls_whitelist=None): + """ Given a JMPayjoinManager object `manager`, initialised with the + payment request data from the server, use its wallet_service to construct + a payment transaction, with coins sourced from mixdepth `mixdepth`, + then wait for the server response, parse the PSBT, perform checks and complete sign. + The info and accept callbacks are to ask the user to confirm the creation of + the original payment transaction (None defaults to terminal/CLI processing), + and are as defined in `taker_utils.direct_send`. + + If `tls_whitelist` is a list of bytestrings, they are treated as hostnames + for which tls certificate verification is ignored. Obviously this is ONLY for + testing. + + Returns: + (True, None) in case of payment setup successful (response will be delivered + asynchronously) - the `manager` object can be inspected for more detail. + (False, errormsg) in case of failure. + """ + + # wallet should already be synced before calling here; + # we can create a standard payment, but have it returned as a PSBT. + assert isinstance(manager, JMPayjoinManager) + assert manager.wallet_service.synced + payment_psbt = direct_send(manager.wallet_service, manager.amount, manager.mixdepth, + str(manager.destination), accept_callback=accept_callback, + info_callback=info_callback, + with_final_psbt=True) + if not payment_psbt: + return (False, "could not create non-payjoin payment") + + manager.set_payment_tx_and_psbt(payment_psbt) + # TODO add delayed call to broadcast this after 1 minute + + # Now we send the request to the server, with the encoded + # payment PSBT + if not tls_whitelist: + agent = Agent(reactor) + else: + agent = Agent(reactor, + contextFactory=WhitelistContextFactory(tls_whitelist)) + + body = BytesProducer(payment_psbt.to_base64().encode("utf-8")) + # TODO what to use as user agent? + d = agent.request(b"POST", manager.server.encode("utf-8"), + Headers({"Content-Type": ['text/plain']}), + bodyProducer=body) + + d.addCallback(receive_payjoin_proposal_from_server, manager) + # note that the errback (here "noresponse") is *not* triggered + # by a server rejection (which is accompanied by a non-200 + # status code returned), but by failure to communicate. + def noResponse(failure): + failure.trap(ResponseFailed) + log.error(failure.value.reasons[0].getTraceback()) + reactor.stop() + d.addErrback(noResponse) + return (True, None) + +def fallback_nonpayjoin_broadcast(manager, err): + assert isinstance(manager, JMPayjoinManager) + log.warn("Payjoin did not succeed, falling back to non-payjoin payment.") + log.warn("Error message was: " + str(err)) + original_tx = manager.initial_psbt.extract_transaction() + if not jm_single().bc_interface.pushtx(original_tx.serialize()): + log.error("Unable to broadcast original payment. The payment is NOT made.") + log.info("We paid without coinjoin. Transaction: ") + log.info(btc.hrt(original_tx)) + reactor.stop() + +def receive_payjoin_proposal_from_server(response, manager): + assert isinstance(manager, JMPayjoinManager) + + # if the response code is not 200 OK, we must assume payjoin + # attempt has failed, and revert to standard payment. + if int(response.code) != 200: + fallback_nonpayjoin_broadcast(manager, err=response.phrase) + return + # for debugging; will be removed in future: + log.debug("Response headers:") + log.debug(pformat(list(response.headers.getAllRawHeaders()))) + # no attempt at chunking or handling incrementally is needed + # here. The body should be a byte string containing the + # new PSBT. + d = readBody(response) + d.addCallback(process_payjoin_proposal_from_server, manager) + +def process_payjoin_proposal_from_server(response_body, manager): + assert isinstance(manager, JMPayjoinManager) + try: + payjoin_proposal_psbt = \ + btc.PartiallySignedTransaction.from_base64(response_body) + except Exception as e: + log.error("Payjoin tx from server could not be parsed: " + repr(e)) + fallback_nonpayjoin_broadcast(manager, err="Server sent invalid psbt") + return + + log.debug("Receiver sent us this PSBT: ") + log.debug(manager.wallet_service.hr_psbt(payjoin_proposal_psbt)) + # we need to add back in our utxo information to the received psbt, + # since the servers remove it (not sure why?) + for i, inp in enumerate(payjoin_proposal_psbt.unsigned_tx.vin): + for j, inp2 in enumerate(manager.initial_psbt.unsigned_tx.vin): + if (inp.prevout.hash, inp.prevout.n) == ( + inp2.prevout.hash, inp2.prevout.n): + payjoin_proposal_psbt.inputs[i].utxo = \ + manager.initial_psbt.inputs[j].utxo + signresultandpsbt, err = manager.wallet_service.sign_psbt( + payjoin_proposal_psbt.serialize(), with_sign_result=True) + if err: + log.error("Failed to sign PSBT from the receiver, error: " + err) + fallback_nonpayjoin_broadcast(manager, err="Failed to sign receiver PSBT") + return + + signresult, sender_signed_psbt = signresultandpsbt + assert signresult.is_final + success, msg = manager.set_payjoin_psbt(payjoin_proposal_psbt, sender_signed_psbt) + if not success: + log.error(msg) + fallback_nonpayjoin_broadcast(manager, err="Receiver PSBT checks failed.") + return + # All checks have passed. We can use the already signed transaction in + # sender_signed_psbt. + log.info("Our final signed PSBT is:\n{}".format( + manager.wallet_service.hr_psbt(sender_signed_psbt))) + manager.set_final_payjoin_psbt(sender_signed_psbt) + + # broadcast the tx + extracted_tx = sender_signed_psbt.extract_transaction() + log.info("Here is the final payjoin transaction:") + log.info(btc.hrt(extracted_tx)) + if not jm_single().bc_interface.pushtx(extracted_tx.serialize()): + log.info("The above transaction failed to broadcast.") + else: + log.info("Payjoin transactoin broadcast successfully.") + reactor.stop() diff --git a/jmclient/jmclient/taker_utils.py b/jmclient/jmclient/taker_utils.py index b2bed0e7d..d8e8a2d0f 100644 --- a/jmclient/jmclient/taker_utils.py +++ b/jmclient/jmclient/taker_utils.py @@ -22,10 +22,12 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, accept_callback=None, info_callback=None, - return_transaction=False, with_final_psbt=False): + return_transaction=False, with_final_psbt=False, + optin_rbf=False): """Send coins directly from one mixdepth to one destination address; does not need IRC. Sweep as for normal sendpayment (set amount=0). If answeryes is True, callback/command line query is not performed. + If optin_rbf is True, the nSequence values are changed as appropriate. If accept_callback is None, command line input for acceptance is assumed, else this callback is called: accept_callback: @@ -142,6 +144,10 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, log.info("Using a change value of: " + amount_to_str(changeval) + ".") tx = make_shuffled_tx(list(utxos.keys()), outs, 2, tx_locktime) + if optin_rbf: + for inp in tx.vin: + inp.nSequence = 0xffffffff - 2 + inscripts = {} spent_outs = [] for i, txinp in enumerate(tx.vin): diff --git a/jmclient/jmclient/wallet.py b/jmclient/jmclient/wallet.py index fb46a99ca..518511332 100644 --- a/jmclient/jmclient/wallet.py +++ b/jmclient/jmclient/wallet.py @@ -1007,6 +1007,30 @@ class PSBTWalletMixin(object): def __init__(self, storage, **kwargs): super(PSBTWalletMixin, self).__init__(storage, **kwargs) + @staticmethod + def get_fee_from_psbt(in_psbt): + assert isinstance(in_psbt, btc.PartiallySignedTransaction) + spent = sum(in_psbt.get_input_amounts()) + paid = sum((x.nValue for x in in_psbt.unsigned_tx.vout)) + return spent - paid + + def is_input_finalized(self, psbt_input): + """ This should be a convenience method in python-bitcointx. + However note: this is not a static method and tacitly + assumes that the input under examination is of the wallet's + type. + """ + assert isinstance(psbt_input, btc.PSBT_Input) + if not psbt_input.utxo: + return False + if isinstance(self, (LegacyWallet, SegwitLegacyWallet)): + if not psbt_input.final_script_sig: + return False + if isinstance(self, (SegwitLegacyWallet, SegwitWallet)): + if not psbt_input.final_script_witness: + return False + return True + @staticmethod def hr_psbt(in_psbt): """ Returns a jsonified indented string with all relevant @@ -1196,8 +1220,7 @@ def sign_psbt(self, in_psbt, with_sign_result=False): for k2, v2 in v.items(): privkeys.append(self._get_priv_from_path(v2[0])) jmckeys = list(btc.JMCKey(x[0][:-1]) for x in privkeys) - new_keystore = btc.KeyStore.from_iterable(jmckeys, - require_path_templates=False) + new_keystore = btc.KeyStore.from_iterable(jmckeys) # for p2sh inputs that we want to sign, the redeem_script # field must be populated by us, as the counterparty did not diff --git a/scripts/sendpayment.py b/scripts/sendpayment.py index f71bef972..af401a527 100755 --- a/scripts/sendpayment.py +++ b/scripts/sendpayment.py @@ -15,7 +15,8 @@ JMClientProtocolFactory, start_reactor, validate_address, is_burn_destination, \ jm_single, estimate_tx_fee, direct_send, WalletService,\ open_test_wallet_maybe, get_wallet_path, NO_ROUNDING, \ - get_sendpayment_parser, get_max_cj_fee_values, check_regtest + get_sendpayment_parser, get_max_cj_fee_values, check_regtest, \ + parse_payjoin_setup, send_payjoin from twisted.python.log import startLogging from jmbase.support import get_log, set_logging_level, jmprint, \ EXIT_FAILURE, EXIT_ARGERROR, DUST_THRESHOLD @@ -51,8 +52,8 @@ def main(): (options, args) = parser.parse_args() load_program_config(config_path=options.datadir) if options.p2ep and len(args) != 3: - parser.error("PayJoin requires exactly three arguments: " - "wallet, amount and destination address.") + parser.error("Joinmarket peer-to-peer PayJoin requires exactly three " + "arguments: wallet, amount and destination address.") sys.exit(EXIT_ARGERROR) elif options.schedule == '': if ((len(args) < 2) or @@ -65,6 +66,7 @@ def main(): #without schedule file option, use the arguments to create a schedule #of a single transaction sweeping = False + bip79 = False if options.schedule == '': if btc.is_bip21_uri(args[1]): parsed = btc.decode_bip21_uri(args[1]) @@ -75,7 +77,15 @@ def main(): sys.exit(EXIT_ARGERROR) destaddr = parsed['address'] if 'jmnick' in parsed: + if "pj" in parsed: + parser.error("Cannot specify both BIP79 and Joinmarket " + "peer-to-peer payjoin at the same time!") + sys.exit(EXIT_ARGERROR) options.p2ep = parsed['jmnick'] + elif "pj" in parsed: + # note that this is a URL; its validity + # checking is deferred to twisted.web.client.Agent + bip79 = parsed["pj"] else: amount = btc.amount_to_sat(args[1]) if amount == 0: @@ -107,6 +117,9 @@ def main(): schedule = [[options.mixdepth, amount, options.makercount, destaddr, 0.0, NO_ROUNDING, 0]] else: + if btc.is_bip21_uri(args[1]): + parser.error("Schedule files are not compatible with bip21 uris.") + sys.exit(EXIT_ARGERROR) if options.p2ep: parser.error("Schedule files are not compatible with PayJoin") sys.exit(EXIT_FAILURE) @@ -146,7 +159,8 @@ def main(): fee_per_cp_guess)) maxcjfee = (1, float('inf')) - if not options.p2ep and not options.pickorders and options.makercount != 0: + if not (options.p2ep or bip79) and not options.pickorders and \ + options.makercount != 0: maxcjfee = get_max_cj_fee_values(jm_single().config, options) log.info("Using maximum coinjoin fee limits per maker of {:.4%}, {} " "".format(maxcjfee[0], btc.amount_to_str(maxcjfee[1]))) @@ -189,7 +203,7 @@ def main(): log.info("Estimated miner/tx fees for this coinjoin amount: {:.1%}" .format(exp_tx_fees_ratio)) - if options.makercount == 0 and not options.p2ep: + if options.makercount == 0 and not options.p2ep and not bip79: tx = direct_send(wallet_service, amount, mixdepth, destaddr, options.answeryes, with_final_psbt=options.with_psbt) if options.with_psbt: @@ -299,6 +313,14 @@ def p2ep_on_finished_callback(res, fromtx=False, waittime=0.0, reactor.stop() taker = P2EPTaker(options.p2ep, wallet_service, schedule, callbacks=(None, None, p2ep_on_finished_callback)) + + elif bip79: + # TODO sanity check wallet type is segwit + manager = parse_payjoin_setup(args[1], wallet_service, options.mixdepth) + reactor.callWhenRunning(send_payjoin, manager) + reactor.run() + return + else: taker = Taker(wallet_service, schedule, diff --git a/test/payjoinclient.py b/test/payjoinclient.py new file mode 100644 index 000000000..764b8b601 --- /dev/null +++ b/test/payjoinclient.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +import sys +from twisted.internet import reactor +from jmclient.cli_options import check_regtest +from jmclient import (get_wallet_path, WalletService, open_test_wallet_maybe, + jm_single, load_test_config, + SegwitLegacyWallet, SegwitWallet) +from jmclient.payjoin import send_payjoin, parse_payjoin_setup + +if __name__ == "__main__": + wallet_name = sys.argv[1] + mixdepth = int(sys.argv[2]) + usessl = int(sys.argv[3]) + bip21uri = None + if len(sys.argv) > 4: + bip21uri = sys.argv[4] + load_test_config() + jm_single().datadir = "." + check_regtest() + if not bip21uri: + if usessl == 0: + pjurl = "http://127.0.0.1:8080" + else: + pjurl = "https://127.0.0.1:8080" + bip21uri = "bitcoin:2N7CAdEUjJW9tUHiPhDkmL9ukPtcukJMoxK?amount=0.3&pj=" + pjurl + wallet_path = get_wallet_path(wallet_name, None) + if jm_single().config.get("POLICY", "native") == "true": + walletclass = SegwitWallet + else: + walletclass = SegwitLegacyWallet + wallet = open_test_wallet_maybe( + wallet_path, wallet_name, 4, + wallet_password_stdin=False, + test_wallet_cls=walletclass, + gap_limit=6) + wallet_service = WalletService(wallet) + # in this script, we need the wallet synced before + # logic processing for some paths, so do it now: + while not wallet_service.synced: + wallet_service.sync_wallet(fast=True) + # the sync call here will now be a no-op: + wallet_service.startService() + manager = parse_payjoin_setup(bip21uri, wallet_service, mixdepth) + if usessl == 0: + tlshostnames = None + else: + tlshostnames = [b"127.0.0.1"] + reactor.callWhenRunning(send_payjoin, manager, tls_whitelist=tlshostnames) + reactor.run() diff --git a/test/payjoinserver.py b/test/payjoinserver.py new file mode 100644 index 000000000..cda5e5c0b --- /dev/null +++ b/test/payjoinserver.py @@ -0,0 +1,178 @@ +#! /usr/bin/env python +""" Creates a very simple server for payjoin + payment requests; uses regtest and a single + JM wallet, provides a hex seed for the sender + side of the test. + Use the same command line setup as for ygrunner.py, + except you needn't specify --nirc= + NOTE: to run this test you will need a `key.pem` + and a `cert.pem` in this (test/) directory, + created in the standard way for ssl certificates. + Note that (in test) the client will not verify + them. +""" +import os +from twisted.web.server import Site +from twisted.web.resource import Resource +from twisted.internet import ssl +from twisted.internet import reactor, endpoints +from io import BytesIO +from common import make_wallets +import pytest +from jmbase import jmprint +import jmbitcoin as btc +from jmclient import load_test_config, jm_single,\ + SegwitWallet, SegwitLegacyWallet, cryptoengine + +# TODO change test for arbitrary payment requests +payment_amt = 30000000 + +dir_path = os.path.dirname(os.path.realpath(__file__)) + +def get_ssl_context(): + """Construct an SSL context factory from the user's privatekey/cert. + Here just hardcoded for tests. + Note this is off by default since the cert needs setting up. + """ + return ssl.DefaultOpenSSLContextFactory(os.path.join(dir_path, "key.pem"), + os.path.join(dir_path, "cert.pem")) + +class PayjoinServer(Resource): + def __init__(self, wallet_service): + self.wallet_service = wallet_service + super().__init__() + isLeaf = True + def render_GET(self, request): + return "Only for testing.".encode("utf-8") + def render_POST(self, request): + """ The sender will use POST to send the initial + payment transaction. + """ + jmprint("The server got this POST request: ") + print(request) + print(request.method) + print(request.uri) + print(request.args) + print(request.path) + print(request.content) + proposed_tx = request.content + assert isinstance(proposed_tx, BytesIO) + payment_psbt_base64 = proposed_tx.read() + payment_psbt = btc.PartiallySignedTransaction.from_base64( + payment_psbt_base64) + all_receiver_utxos = self.wallet_service.get_all_utxos() + # TODO is there a less verbose way to get any 2 utxos from the dict? + receiver_utxos_keys = list(all_receiver_utxos.keys())[:2] + receiver_utxos = {k: v for k, v in all_receiver_utxos.items( + ) if k in receiver_utxos_keys} + + # receiver will do other checks as discussed above, including payment + # amount; as discussed above, this is out of the scope of this PSBT test. + + # construct unsigned tx for payjoin-psbt: + payjoin_tx_inputs = [(x.prevout.hash[::-1], + x.prevout.n) for x in payment_psbt.unsigned_tx.vin] + payjoin_tx_inputs.extend(receiver_utxos.keys()) + # find payment output and change output + pay_out = None + change_out = None + for o in payment_psbt.unsigned_tx.vout: + jm_out_fmt = {"value": o.nValue, + "address": str(btc.CCoinAddress.from_scriptPubKey( + o.scriptPubKey))} + if o.nValue == payment_amt: + assert pay_out is None + pay_out = jm_out_fmt + else: + assert change_out is None + change_out = jm_out_fmt + + # we now know there were two outputs and know which is payment. + # bump payment output with our input: + outs = [pay_out, change_out] + our_inputs_val = sum([v["value"] for _, v in receiver_utxos.items()]) + pay_out["value"] += our_inputs_val + print("we bumped the payment output value by: ", our_inputs_val) + print("It is now: ", pay_out["value"]) + unsigned_payjoin_tx = btc.make_shuffled_tx(payjoin_tx_inputs, outs, + version=payment_psbt.unsigned_tx.nVersion, + locktime=payment_psbt.unsigned_tx.nLockTime) + print("we created this unsigned tx: ") + print(btc.hrt(unsigned_payjoin_tx)) + # to create the PSBT we need the spent_outs for each input, + # in the right order: + spent_outs = [] + for i, inp in enumerate(unsigned_payjoin_tx.vin): + input_found = False + for j, inp2 in enumerate(payment_psbt.unsigned_tx.vin): + if inp.prevout == inp2.prevout: + spent_outs.append(payment_psbt.inputs[j].utxo) + input_found = True + break + if input_found: + continue + # if we got here this input is ours, we must find + # it from our original utxo choice list: + for ru in receiver_utxos.keys(): + if (inp.prevout.hash[::-1], inp.prevout.n) == ru: + spent_outs.append( + self.wallet_service.witness_utxos_to_psbt_utxos( + {ru: receiver_utxos[ru]})[0]) + input_found = True + break + # there should be no other inputs: + assert input_found + + r_payjoin_psbt = self.wallet_service.create_psbt_from_tx(unsigned_payjoin_tx, + spent_outs=spent_outs) + print("Receiver created payjoin PSBT:\n{}".format( + self.wallet_service.hr_psbt(r_payjoin_psbt))) + + signresultandpsbt, err = self.wallet_service.sign_psbt(r_payjoin_psbt.serialize(), + with_sign_result=True) + assert not err, err + signresult, receiver_signed_psbt = signresultandpsbt + assert signresult.num_inputs_final == len(receiver_utxos) + assert not signresult.is_final + + print("Receiver signing successful. Payjoin PSBT is now:\n{}".format( + self.wallet_service.hr_psbt(receiver_signed_psbt))) + content = receiver_signed_psbt.to_base64() + request.setHeader(b"content-length", ("%d" % len(content)).encode("ascii")) + return content.encode("ascii") + + + +def test_start_payjoin_server(setup_payjoin_server): + # set up the wallet that the server owns, and the wallet for + # the sender too (print the seed): + if jm_single().config.get("POLICY", "native") == "true": + walletclass = SegwitWallet + else: + walletclass = SegwitLegacyWallet + + wallet_services = make_wallets(2, + wallet_structures=[[1, 3, 0, 0, 0]] * 2, + mean_amt=2, + walletclass=walletclass) + #the server bot uses the first wallet, the sender the second + server_wallet_service = wallet_services[0]['wallet'] + jmprint("\n\nTaker wallet seed : " + wallet_services[1]['seed']) + jmprint("\n") + server_wallet_service.sync_wallet(fast=True) + + site = Site(PayjoinServer(server_wallet_service)) + # TODO for now, just sticking with TLS test as non-encrypted + # is unlikely to be used, but add that option. + reactor.listenSSL(8080, site, contextFactory=get_ssl_context()) + #endpoint = endpoints.TCP4ServerEndpoint(reactor, 8080) + #endpoint.listen(site) + reactor.run() + +@pytest.fixture(scope="module") +def setup_payjoin_server(): + load_test_config() + jm_single().bc_interface.tick_forward_chain_interval = 10 + jm_single().bc_interface.simulate_blocks() + # handles the custom regtest hrp for bech32 + cryptoengine.BTC_P2WPKH.VBYTE = 100 From 4cf77ed77417e11a56742fbb819917d31d7dafbb Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Wed, 6 May 2020 21:04:28 +0100 Subject: [PATCH 09/15] Various bugfixes: Fix bug in Coins tab, get_utxos_by_mixdepth calls Fix bug in jmbitcoin.mktx, now respects nVersion choice Fix bug in tumbler restart wait Fix bug in makercount for payjoin for fee check, set to 0: The value of `options.makercount` is set to zero so that the fee sanity check in the sendpayment script operates approximately correctly (the receiver will bump the fee to keep the fee rate the same if necessary). Also the `bip79` variable is better named `payjoinurl`. Fix bug in "freeze" context menu function in Qt --- jmbitcoin/jmbitcoin/secp256k1_transaction.py | 2 +- jmclient/jmclient/taker_utils.py | 4 ++-- jmclient/jmclient/wallet_utils.py | 10 ++++---- scripts/joinmarket-qt.py | 13 +++++++---- scripts/sendpayment.py | 24 +++++++++++++------- 5 files changed, 33 insertions(+), 20 deletions(-) diff --git a/jmbitcoin/jmbitcoin/secp256k1_transaction.py b/jmbitcoin/jmbitcoin/secp256k1_transaction.py index cf988fb00..705302c65 100644 --- a/jmbitcoin/jmbitcoin/secp256k1_transaction.py +++ b/jmbitcoin/jmbitcoin/secp256k1_transaction.py @@ -294,7 +294,7 @@ def mktx(ins, outs, version=1, locktime=0): out = CMutableTxOut(o["value"], CCoinAddress(o["address"]).to_scriptPubKey()) vout.append(out) - return CMutableTransaction(vin, vout, nLockTime=locktime) + return CMutableTransaction(vin, vout, nLockTime=locktime, nVersion=version) def make_shuffled_tx(ins, outs, version=1, locktime=0): """ Simple wrapper to ensure transaction diff --git a/jmclient/jmclient/taker_utils.py b/jmclient/jmclient/taker_utils.py index d8e8a2d0f..875b3433f 100644 --- a/jmclient/jmclient/taker_utils.py +++ b/jmclient/jmclient/taker_utils.py @@ -4,7 +4,7 @@ import sys import time import numbers -from jmbase import get_log, jmprint, bintohex +from jmbase import get_log, jmprint, bintohex, hextobin from .configure import jm_single, validate_address, is_burn_destination from .schedule import human_readable_schedule_entry, tweak_tumble_schedule,\ schedule_to_text @@ -209,7 +209,7 @@ def restart_wait(txid): and confirmed (it must be an in-wallet transaction since it always spends coins from the wallet). """ - res = jm_single().bc_interface.get_transaction(txid) + res = jm_single().bc_interface.get_transaction(hextobin(txid)) if not res: return False if res["confirmations"] == 0: diff --git a/jmclient/jmclient/wallet_utils.py b/jmclient/jmclient/wallet_utils.py index 4fb30e01f..ee7b33686 100644 --- a/jmclient/jmclient/wallet_utils.py +++ b/jmclient/jmclient/wallet_utils.py @@ -345,8 +345,8 @@ def get_imported_privkey_branch(wallet_service, m, showprivkey): addr = wallet_service.get_address_from_path(path) script = wallet_service.get_script_from_path(path) balance = 0.0 - for data in wallet_service.get_utxos_by_mixdepth(include_disabled=True, - hexfmt=False)[m].values(): + for data in wallet_service.get_utxos_by_mixdepth( + include_disabled=True)[m].values(): if script == data['script']: balance += data['value'] used = ('used' if balance > 0.0 else 'empty') @@ -962,7 +962,7 @@ def f(r, deposits, deposit_times, now, final_balance): 'history (%s)') % (btc.sat_to_str(total_wallet_balance), btc.sat_to_str(balance))) wallet_utxo_count = sum(map(len, wallet.get_utxos_by_mixdepth( - include_disabled=True, hexfmt=False).values())) + include_disabled=True).values())) if utxo_count + unconfirmed_utxo_count != wallet_utxo_count: jmprint(('BUG ERROR: wallet utxo count (%d) does not match utxo count from ' + 'history (%s)') % (wallet_utxo_count, utxo_count)) @@ -1087,8 +1087,8 @@ def output_utxos(utxos, status, start=0): def get_utxos_enabled_disabled(wallet_service, md): """ Returns dicts for enabled and disabled separately """ - utxos_enabled = wallet_service.get_utxos_by_mixdepth(hexfmt=False)[md] - utxos_all = wallet_service.get_utxos_by_mixdepth(include_disabled=True, hexfmt=False)[md] + utxos_enabled = wallet_service.get_utxos_by_mixdepth()[md] + utxos_all = wallet_service.get_utxos_by_mixdepth(include_disabled=True)[md] utxos_disabled_keyset = set(utxos_all).difference(set(utxos_enabled)) utxos_disabled = {} for u in utxos_disabled_keyset: diff --git a/scripts/joinmarket-qt.py b/scripts/joinmarket-qt.py index 46f033e3c..cbfa6d5f6 100755 --- a/scripts/joinmarket-qt.py +++ b/scripts/joinmarket-qt.py @@ -64,7 +64,8 @@ JM_GUI_VERSION = '15dev' from jmbase import get_log -from jmbase.support import DUST_THRESHOLD, EXIT_FAILURE, JM_CORE_VERSION +from jmbase.support import DUST_THRESHOLD, EXIT_FAILURE, utxo_to_utxostr,\ + bintohex, hextobin, JM_CORE_VERSION from jmclient import load_program_config, get_network, update_persist_config,\ open_test_wallet_maybe, get_wallet_path,\ jm_single, validate_address, weighted_order_choose, Taker,\ @@ -627,6 +628,7 @@ def startMultiple(self): self.waitingtxid=txid self.restartTimer.timeout.connect(self.restartWaitWrap) self.restartTimer.start(5000) + self.updateSchedView() return self.updateSchedView() self.startJoin() @@ -1174,7 +1176,9 @@ def show_blank(): else: for k, v in um.items(): # txid:index, btc, address - t = btc.safe_hexlify(k[0])+":"+str(k[1]) + success, t = utxo_to_utxostr(k) + # keys must be utxo format else a coding error: + assert success s = "{0:.08f}".format(v['value']/1e8) a = mainWindow.wallet_service.script_to_addr(v["script"]) item = QTreeWidgetItem([t, s, a]) @@ -1187,7 +1191,7 @@ def show_blank(): def toggle_utxo_disable(self, txids, idxs): for i in range(0, len(txids)): txid = txids[i] - txid_bytes = btc.safe_from_hex(txid) + txid_bytes = hextobin(txid) mainWindow.wallet_service.toggle_disable_utxo(txid_bytes, idxs[i]) self.updateUtxos() @@ -1206,7 +1210,8 @@ def create_menu(self, position): assert idx >= 0 txids.append(txid) idxs.append(idx) - except: + except Exception as e: + log.error("Error retrieving txids in Coins tab: " + repr(e)) return # current item item = self.cTW.currentItem() diff --git a/scripts/sendpayment.py b/scripts/sendpayment.py index af401a527..79dd5217f 100755 --- a/scripts/sendpayment.py +++ b/scripts/sendpayment.py @@ -66,7 +66,7 @@ def main(): #without schedule file option, use the arguments to create a schedule #of a single transaction sweeping = False - bip79 = False + payjoinurl = None if options.schedule == '': if btc.is_bip21_uri(args[1]): parsed = btc.decode_bip21_uri(args[1]) @@ -78,14 +78,22 @@ def main(): destaddr = parsed['address'] if 'jmnick' in parsed: if "pj" in parsed: - parser.error("Cannot specify both BIP79 and Joinmarket " + parser.error("Cannot specify both BIP79++ and Joinmarket " "peer-to-peer payjoin at the same time!") sys.exit(EXIT_ARGERROR) options.p2ep = parsed['jmnick'] elif "pj" in parsed: # note that this is a URL; its validity # checking is deferred to twisted.web.client.Agent - bip79 = parsed["pj"] + payjoinurl = parsed["pj"] + # setting makercount only for fee sanity check. + # note we ignore any user setting and enforce N=0, + # as this is a flag in the code for a non-JM coinjoin; + # for the fee sanity check, note that BIP79++ currently + # will only allow very small fee changes, so N=0 won't + # be very inaccurate. + jmprint("Attempting to pay via payjoin.", "info") + options.makercount = 0 else: amount = btc.amount_to_sat(args[1]) if amount == 0: @@ -159,13 +167,13 @@ def main(): fee_per_cp_guess)) maxcjfee = (1, float('inf')) - if not (options.p2ep or bip79) and not options.pickorders and \ + if not options.p2ep and not options.pickorders and \ options.makercount != 0: maxcjfee = get_max_cj_fee_values(jm_single().config, options) log.info("Using maximum coinjoin fee limits per maker of {:.4%}, {} " "".format(maxcjfee[0], btc.amount_to_str(maxcjfee[1]))) - log.debug('starting sendpayment') + log.info('starting sendpayment') max_mix_depth = max([mixdepth, options.amtmixdepths - 1]) @@ -203,7 +211,7 @@ def main(): log.info("Estimated miner/tx fees for this coinjoin amount: {:.1%}" .format(exp_tx_fees_ratio)) - if options.makercount == 0 and not options.p2ep and not bip79: + if options.makercount == 0 and not options.p2ep and not payjoinurl: tx = direct_send(wallet_service, amount, mixdepth, destaddr, options.answeryes, with_final_psbt=options.with_psbt) if options.with_psbt: @@ -314,10 +322,10 @@ def p2ep_on_finished_callback(res, fromtx=False, waittime=0.0, taker = P2EPTaker(options.p2ep, wallet_service, schedule, callbacks=(None, None, p2ep_on_finished_callback)) - elif bip79: + elif payjoinurl: # TODO sanity check wallet type is segwit manager = parse_payjoin_setup(args[1], wallet_service, options.mixdepth) - reactor.callWhenRunning(send_payjoin, manager) + reactor.callWhenRunning(send_payjoin, manager, tls_whitelist=["127.0.0.1"]) reactor.run() return From 53ef79bf3727fb7b17d48c7b0e6465130010c55a Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Sat, 6 Jun 2020 15:14:19 +0100 Subject: [PATCH 10/15] Updates to account for code changes in #544 Note in particular that: bitcoin.mktx in this PR now does support script entries in outputs to account for nonstandard destinations (as is needed for burn). bitcoin.sign now supports p2wsh (as is needed for timelocks). --- jmbitcoin/jmbitcoin/secp256k1_transaction.py | 69 +++++++++++++------- jmbitcoin/test/test_tx_signing.py | 2 +- jmclient/jmclient/cryptoengine.py | 16 ++--- jmclient/jmclient/taker_utils.py | 6 +- jmclient/jmclient/wallet.py | 6 +- jmclient/test/test_taker.py | 2 +- jmclient/test/test_tx_creation.py | 26 ++++---- jmclient/test/test_wallet.py | 12 ++-- 8 files changed, 77 insertions(+), 62 deletions(-) diff --git a/jmbitcoin/jmbitcoin/secp256k1_transaction.py b/jmbitcoin/jmbitcoin/secp256k1_transaction.py index 705302c65..4b5f65bae 100644 --- a/jmbitcoin/jmbitcoin/secp256k1_transaction.py +++ b/jmbitcoin/jmbitcoin/secp256k1_transaction.py @@ -3,6 +3,10 @@ # note, only used for non-cryptographic randomness: import random import json +# needed for single sha256 evaluation, which is used +# in bitcoin (p2wsh) but not exposed in python-bitcointx: +import hashlib + from jmbitcoin.secp256k1_main import * from jmbase import bintohex, utxo_to_utxostr from bitcointx.core import (CMutableTransaction, Hash160, CTxInWitness, @@ -154,10 +158,12 @@ def pubkey_to_p2sh_p2wpkh_script(pub): return pubkey_to_p2wpkh_script(pub).to_p2sh_scriptPubKey() def redeem_script_to_p2wsh_script(redeem_script): - return P2WSH_PRE + bin_sha256(binascii.unhexlify(redeem_script)) - -def redeem_script_to_p2wsh_address(redeem_script, vbyte, witver=0): - return script_to_address(redeem_script_to_p2wsh_script(redeem_script), vbyte, witver) + """ Given redeem script of type CScript (or bytes) + returns the corresponding segwit v0 scriptPubKey as + for the case pay-to-witness-scripthash. + """ + return standard_witness_v0_scriptpubkey( + hashlib.sha256(redeem_script).digest()) def mk_freeze_script(pub, locktime): """ @@ -169,27 +175,30 @@ def mk_freeze_script(pub, locktime): if not isinstance(pub, bytes): raise TypeError("pubkey must be in bytes") usehex = False - if not is_valid_pubkey(pub, usehex, require_compressed=True): + if not is_valid_pubkey(pub, require_compressed=True): raise ValueError("not a valid public key") - scr = [locktime, btc.OP_CHECKLOCKTIMEVERIFY, btc.OP_DROP, pub, - btc.OP_CHECKSIG] - return binascii.hexlify(serialize_script(scr)).decode() + return CScript([locktime, OP_CHECKLOCKTIMEVERIFY, OP_DROP, pub, + OP_CHECKSIG]) def mk_burn_script(data): + """ For a given bytestring (data), + returns a scriptPubKey which is an OP_RETURN + of that data. + """ if not isinstance(data, bytes): raise TypeError("data must be in bytes") - data = binascii.hexlify(data).decode() - scr = [btc.OP_RETURN, data] - return serialize_script(scr) + return CScript([btc.OP_RETURN, data]) def sign(tx, i, priv, hashcode=SIGHASH_ALL, amount=None, native=False): """ Given a transaction tx of type CMutableTransaction, an input index i, and a raw privkey in bytes, updates the CMutableTransaction to contain the newly appended signature. - Only three scriptPubKey types supported: p2pkh, p2wpkh, p2sh-p2wpkh. + Only four scriptPubKey types supported: p2pkh, p2wpkh, p2sh-p2wpkh, p2wsh. Note that signing multisig must be done outside this function, using the wrapped library. + If native is not the default (False), and if native != "p2wpkh", + then native must be a CScript object containing the redeemscript needed to sign. Returns: (signature, "signing succeeded") or: (None, errormsg) in case of failure """ @@ -226,11 +235,17 @@ def return_err(e): # see line 1256 of bitcointx.core.scripteval.py: flags.add(SCRIPT_VERIFY_P2SH) - input_scriptPubKey = pubkey_to_p2wpkh_script(pub) - # only created for convenience access to scriptCode: - input_address = P2WPKHCoinAddress.from_scriptPubKey(input_scriptPubKey) - # function name is misleading here; redeemScript only applies to p2sh. - scriptCode = input_address.to_redeemScript() + if native and native != "p2wpkh": + scriptCode = native + input_scriptPubKey = redeem_script_to_p2wsh_script(native) + else: + # this covers both p2wpkh and p2sh-p2wpkh case: + input_scriptPubKey = pubkey_to_p2wpkh_script(pub) + # only created for convenience access to scriptCode: + input_address = P2WPKHCoinAddress.from_scriptPubKey( + input_scriptPubKey) + # function name is misleading here; redeemScript only applies to p2sh. + scriptCode = input_address.to_redeemScript() sighash = SignatureHash(scriptCode, tx, i, hashcode, amount=amount, sigversion=SIGVERSION_WITNESS_V0) @@ -243,7 +258,10 @@ def return_err(e): else: tx.vin[i].scriptSig = CScript([input_scriptPubKey]) - witness = [sig, pub] + if native and native != "p2wpkh": + witness = [sig, scriptCode] + else: + witness = [sig, pub] ctxwitness = CTxInWitness(CScriptWitness(witness)) tx.wit.vtxinwit[i] = ctxwitness # Verify the signature worked. @@ -268,7 +286,9 @@ def apply_freeze_signature(tx, i, redeem_script, sig): def mktx(ins, outs, version=1, locktime=0): """ Given a list of input tuples (txid(bytes), n(int)), and a list of outputs which are dicts with - keys "address" (value should be *str* not CCoinAddress), + keys "address" (value should be *str* not CCoinAddress) ( + or alternately "script" (for nonstandard outputs, value + should be CScript)), "value" (value should be integer satoshis), outputs a CMutableTransaction object. Tx version and locktime are optionally set, for non-default @@ -289,10 +309,13 @@ def mktx(ins, outs, version=1, locktime=0): inp = CMutableTxIn(prevout=outpoint, nSequence=sequence) vin.append(inp) for o in outs: - # note the to_scriptPubKey method is only available for standard - # address types - out = CMutableTxOut(o["value"], - CCoinAddress(o["address"]).to_scriptPubKey()) + if "script" in o: + sPK = o["script"] + else: + # note the to_scriptPubKey method is only available for standard + # address types + sPK = CCoinAddress(o["address"]).to_scriptPubKey() + out = CMutableTxOut(o["value"], sPK) vout.append(out) return CMutableTransaction(vin, vout, nLockTime=locktime, nVersion=version) diff --git a/jmbitcoin/test/test_tx_signing.py b/jmbitcoin/test/test_tx_signing.py index d7f3d6f67..e2ccf21f2 100644 --- a/jmbitcoin/test/test_tx_signing.py +++ b/jmbitcoin/test/test_tx_signing.py @@ -49,7 +49,7 @@ def test_sign_standard_txs(addrtype): # Calculate the signature hash for the transaction. This is then signed by the # private key that controls the UTXO being spent here at this txin_index. if addrtype == "p2wpkh": - sig, msg = btc.sign(tx, 0, priv, amount=amount, native=True) + sig, msg = btc.sign(tx, 0, priv, amount=amount, native="p2wpkh") elif addrtype == "p2sh-p2wpkh": sig, msg = btc.sign(tx, 0, priv, amount=amount, native=False) elif addrtype == "p2pkh": diff --git a/jmclient/jmclient/cryptoengine.py b/jmclient/jmclient/cryptoengine.py index 17f114870..9262d3bfd 100644 --- a/jmclient/jmclient/cryptoengine.py +++ b/jmclient/jmclient/cryptoengine.py @@ -314,7 +314,7 @@ def sign_transaction(cls, tx, index, privkey, amount, hashcode=btc.SIGHASH_ALL, **kwargs): assert amount is not None return btc.sign(tx, index, privkey, - hashcode=hashcode, amount=amount, native=True) + hashcode=hashcode, amount=amount, native="p2wpkh") class BTC_Timelocked_P2WSH(BTCEngine): @@ -362,16 +362,10 @@ def privkey_to_wif(cls, privkey_locktime): def sign_transaction(cls, tx, index, privkey_locktime, amount, hashcode=btc.SIGHASH_ALL, **kwargs): assert amount is not None - - privkey, locktime = privkey_locktime - privkey = hexlify(privkey).decode() - pubkey = btc.privkey_to_pubkey(privkey) - pubkey = unhexlify(pubkey) - redeem_script = cls.pubkey_to_script_code((pubkey, locktime)) - tx = btc.serialize(tx) - sig = btc.get_p2sh_signature(tx, index, redeem_script, privkey, - amount) - return btc.apply_freeze_signature(tx, index, redeem_script, sig) + priv, locktime = privkey_locktime + pub = cls.privkey_to_pubkey(priv) + redeem_script = cls.pubkey_to_script_code((pub, locktime)) + return btc.sign(tx, index, priv, amount=amount, native=redeem_script) class BTC_Watchonly_Timelocked_P2WSH(BTC_Timelocked_P2WSH): diff --git a/jmclient/jmclient/taker_utils.py b/jmclient/jmclient/taker_utils.py index 875b3433f..f57935673 100644 --- a/jmclient/jmclient/taker_utils.py +++ b/jmclient/jmclient/taker_utils.py @@ -11,7 +11,7 @@ from .wallet import BaseWallet, estimate_tx_fee, compute_tx_locktime, \ FidelityBondMixin from jmbitcoin import make_shuffled_tx, amount_to_str, mk_burn_script,\ - PartiallySignedTransaction, CMutableTxOut, hrt + PartiallySignedTransaction, CMutableTxOut, hrt, Hash160 from jmbase.support import EXIT_SUCCESS log = get_log() @@ -94,10 +94,10 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, path = wallet_service.wallet.get_path(mixdepth, address_type, index) privkey, engine = wallet_service.wallet._get_key_from_path(path) pubkey = engine.privkey_to_pubkey(privkey) - pubkeyhash = bin_hash160(pubkey) + pubkeyhash = Hash160(pubkey) #size of burn output is slightly different from regular outputs - burn_script = mk_burn_script(pubkeyhash) #in hex + burn_script = mk_burn_script(pubkeyhash) fee_est = estimate_tx_fee(len(utxos), 0, txtype=txtype, extra_bytes=len(burn_script)/2) outs = [{"script": burn_script, "value": total_inputs_val - fee_est}] diff --git a/jmclient/jmclient/wallet.py b/jmclient/jmclient/wallet.py index 518511332..2797e3c04 100644 --- a/jmclient/jmclient/wallet.py +++ b/jmclient/jmclient/wallet.py @@ -1193,7 +1193,7 @@ def create_psbt_from_tx(self, tx, spent_outs=None): # this happens when an input is provided but it's not in # this wallet; in this case, we cannot set the redeem script. continue - privkey, _ = self._get_priv_from_path(path) + privkey, _ = self._get_key_from_path(path) txinput.redeem_script = btc.pubkey_to_p2wpkh_script( btc.privkey_to_pubkey(privkey)) return new_psbt @@ -1218,7 +1218,7 @@ def sign_psbt(self, in_psbt, with_sign_result=False): privkeys = [] for k, v in self._utxos._utxo.items(): for k2, v2 in v.items(): - privkeys.append(self._get_priv_from_path(v2[0])) + privkeys.append(self._get_key_from_path(v2[0])) jmckeys = list(btc.JMCKey(x[0][:-1]) for x in privkeys) new_keystore = btc.KeyStore.from_iterable(jmckeys) @@ -1241,7 +1241,7 @@ def sign_psbt(self, in_psbt, with_sign_result=False): # this happens when an input is provided but it's not in # this wallet; in this case, we cannot set the redeem script. continue - privkey, _ = self._get_priv_from_path(path) + privkey, _ = self._get_key_from_path(path) txinput.redeem_script = btc.pubkey_to_p2wpkh_script( btc.privkey_to_pubkey(privkey)) # no else branch; any other form of scriptPubKey will just be diff --git a/jmclient/test/test_taker.py b/jmclient/test/test_taker.py index 9d6471b44..a31607841 100644 --- a/jmclient/test/test_taker.py +++ b/jmclient/test/test_taker.py @@ -412,7 +412,7 @@ def test_on_sig(setup_taker, dummyaddr, schedule): utxos = [(struct.pack(b"B", x) * 32, 1) for x in range(5)] #create 2 privkey + utxos that are to be ours privs = [x*32 + b"\x01" for x in [struct.pack(b'B', y) for y in range(1,6)]] - scripts = [BTC_P2PKH.privkey_to_script(privs[x]) for x in range(5)] + scripts = [BTC_P2PKH.key_to_script(privs[x]) for x in range(5)] addrs = [BTC_P2PKH.privkey_to_address(privs[x]) for x in range(5)] fake_query_results = [{'value': 200000000, 'utxo': utxos[x], 'address': addrs[x], 'script': scripts[x], 'confirms': 20} for x in range(5)] diff --git a/jmclient/test/test_tx_creation.py b/jmclient/test/test_tx_creation.py index 75f4d8a60..4c0aafe56 100644 --- a/jmclient/test/test_tx_creation.py +++ b/jmclient/test/test_tx_creation.py @@ -6,7 +6,6 @@ does not use this feature.''' import struct -from binascii import unhexlify from commontest import make_wallets, make_sign_and_push, ensure_bip65_activated import jmbitcoin as bitcoin @@ -132,7 +131,7 @@ def test_spend_p2wpkh(setup_tx_creation): for i, priv in enumerate(privs): # sign each of 3 inputs; note that bitcoin.sign # automatically validates each signature it creates. - sig, msg = bitcoin.sign(tx, i, priv, amount=amount, native=True) + sig, msg = bitcoin.sign(tx, i, priv, amount=amount, native="p2wpkh") if not sig: assert False, msg txid = jm_single().bc_interface.pushtx(tx.serialize()) @@ -150,13 +149,14 @@ def test_spend_freeze_script(setup_tx_creation): for timeoffset, required_success in timeoffset_success_tests: #generate keypair - priv = "aa"*32 + "01" - pub = unhexlify(bitcoin.privkey_to_pubkey(priv)) + priv = b"\xaa"*32 + b"\x01" + pub = bitcoin.privkey_to_pubkey(priv) addr_locktime = mediantime + timeoffset redeem_script = bitcoin.mk_freeze_script(pub, addr_locktime) script_pub_key = bitcoin.redeem_script_to_p2wsh_script(redeem_script) - regtest_vbyte = 100 - addr = bitcoin.script_to_address(script_pub_key, vbyte=regtest_vbyte) + # cannot convert to address within wallet service, as not known + # to wallet; use engine directly: + addr = wallet_service._ENGINE.script_to_address(script_pub_key) #fund frozen funds address amount = 100000000 @@ -165,20 +165,18 @@ def test_spend_freeze_script(setup_tx_creation): assert funding_txid #spend frozen funds - frozen_in = funding_txid + ":0" + frozen_in = (funding_txid, 0) output_addr = wallet_service.get_internal_addr(1) miner_fee = 5000 outs = [{'value': amount - miner_fee, 'address': output_addr}] tx = bitcoin.mktx([frozen_in], outs, locktime=addr_locktime+1) i = 0 - sig = bitcoin.get_p2sh_signature(tx, i, redeem_script, priv, amount) - - assert bitcoin.verify_tx_input(tx, i, script_pub_key, sig, pub, - scriptCode=redeem_script, amount=amount) - tx = bitcoin.apply_freeze_signature(tx, i, redeem_script, sig) - push_success = jm_single().bc_interface.pushtx(tx) - + sig, success = bitcoin.sign(tx, i, priv, amount=amount, + native=redeem_script) + assert success + push_success = jm_single().bc_interface.pushtx(tx.serialize()) assert push_success == required_success + @pytest.fixture(scope="module") def setup_tx_creation(): load_test_config() diff --git a/jmclient/test/test_wallet.py b/jmclient/test/test_wallet.py index c19a8cf98..15fed468e 100644 --- a/jmclient/test/test_wallet.py +++ b/jmclient/test/test_wallet.py @@ -416,12 +416,12 @@ def test_timelocked_output_signing(setup_wallet): utxo = fund_wallet_addr(wallet, wallet.script_to_addr(script)) timestamp = wallet._time_number_to_timestamp(timenumber) - tx = btc.deserialize(btc.mktx(['{}:{}'.format( - hexlify(utxo[0]).decode('ascii'), utxo[1])], - [btc.p2sh_scriptaddr(b"\x00",magicbyte=196) + ':' + str(10**8 - 9000)], - locktime=timestamp+1)) - tx = wallet.sign_tx(tx, {0: (script, 10**8)}) - txout = jm_single().bc_interface.pushtx(btc.serialize(tx)) + tx = btc.mktx([utxo], [{"address": str(btc.CCoinAddress.from_scriptPubKey( + btc.standard_scripthash_scriptpubkey(btc.Hash160(b"\x00")))), + "value":10**8 - 9000}], locktime=timestamp+1) + success, msg = wallet.sign_tx(tx, {0: (script, 10**8)}) + assert success, msg + txout = jm_single().bc_interface.pushtx(tx.serialize()) assert txout def test_get_bbm(setup_wallet): From d34c53bc05200da9edb49230a9990415d0aed9ac Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Tue, 9 Jun 2020 20:27:57 +0100 Subject: [PATCH 11/15] Various fixups: Upgrade python-bitcointx to 1.1.0: Address requirements of python-bitcointx 1.1.0: Specifically, the witness `utxo` field can no longer be assumed to be of type CTxOut, so we should access the CTxOut with the field witness_utxo and also when updating the `utxo` field we now use `set_utxo()`. Use PartiallySignedTransaction.get_fee() method. Use PartiallySignedTransaction.set_utxo. Additionally some minor typos/comment corrections and removal of the now defunct `apply_freeze_signature`. Add custom load location for libsecp where needed; falls back to system installation if Joinmarket custom installation is not found. Decode error msg from server in payjoin Cleanup test file test_proposals.txt (delete after test) Human readable function names (names for human readable conversions are now themselves human readable). Remove unused get_*_vbyte functions and cleanup Removes old unused files (electrum*.py). Fixes core nohistory sync test to use both standard wallet types, and fixes address import counter. Fixes that test to use the right chain params so that native segwit wallets can work in regtest with nohistory mode. Removes some now unneeded imports. Fixes commontest.create_wallet_for_sync to hash all parameters, including optional ones. Replaces usage of binascii.hexlify with bintohex. --- .gitignore | 1 + jmbitcoin/jmbitcoin/__init__.py | 11 + jmbitcoin/jmbitcoin/secp256k1_transaction.py | 22 +- jmbitcoin/setup.py | 2 +- jmbitcoin/test/test_tx_signing.py | 3 +- jmclient/jmclient/__init__.py | 5 +- jmclient/jmclient/configure.py | 12 +- jmclient/jmclient/electrum_data.py | 261 --------- jmclient/jmclient/electruminterface.py | 549 ------------------- jmclient/jmclient/maker.py | 2 +- jmclient/jmclient/payjoin.py | 34 +- jmclient/jmclient/taker.py | 6 +- jmclient/jmclient/taker_utils.py | 11 +- jmclient/jmclient/wallet.py | 34 +- jmclient/test/commontest.py | 13 +- jmclient/test/test_configure.py | 10 +- jmclient/test/test_core_nohistory_sync.py | 26 +- jmclient/test/test_psbt_wallet.py | 16 +- jmclient/test/test_snicker.py | 18 +- test/payjoinserver.py | 6 +- 20 files changed, 122 insertions(+), 920 deletions(-) delete mode 100644 jmclient/jmclient/electrum_data.py delete mode 100644 jmclient/jmclient/electruminterface.py diff --git a/.gitignore b/.gitignore index ce381d709..63520648d 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ miniircd/ miniircd.tar.gz nums_basepoints.txt schedulefortesting +test_proposals.txt scripts/commitmentlist tmp/ wallets/ diff --git a/jmbitcoin/jmbitcoin/__init__.py b/jmbitcoin/jmbitcoin/__init__.py index 6ab579f71..2a19f7f16 100644 --- a/jmbitcoin/jmbitcoin/__init__.py +++ b/jmbitcoin/jmbitcoin/__init__.py @@ -1,4 +1,15 @@ import coincurve as secp256k1 + +# If user has compiled and installed libsecp256k1 via +# JM installation script install.sh, use that; +# if not, it is assumed to be present at the system level +# See: https://github.com/Simplexum/python-bitcointx/commit/79333106eeb55841df2935781646369b186d99f7#diff-1ea6586127522e62d109ec5893a18850R301-R310 +import os, sys +expected_secp_location = os.path.join(sys.prefix, "lib", "libsecp256k1.so") +if os.path.exists(expected_secp_location): + import bitcointx + bitcointx.set_custom_secp256k1_path(expected_secp_location) + from jmbitcoin.secp256k1_main import * from jmbitcoin.secp256k1_transaction import * from jmbitcoin.secp256k1_deterministic import * diff --git a/jmbitcoin/jmbitcoin/secp256k1_transaction.py b/jmbitcoin/jmbitcoin/secp256k1_transaction.py index 4b5f65bae..7cf9fb56e 100644 --- a/jmbitcoin/jmbitcoin/secp256k1_transaction.py +++ b/jmbitcoin/jmbitcoin/secp256k1_transaction.py @@ -18,7 +18,7 @@ from bitcointx.core.scripteval import (VerifyScript, SCRIPT_VERIFY_WITNESS, SCRIPT_VERIFY_P2SH, SIGVERSION_WITNESS_V0) -def hrt(tx, jsonified=True): +def human_readable_transaction(tx, jsonified=True): """ Given a CTransaction object, output a human readable json-formatted string (suitable for terminal output or large GUI textbox display) containing @@ -40,14 +40,14 @@ def hrt(tx, jsonified=True): witarg = None else: witarg = tx.wit.vtxinwit[i] - outdict["inputs"].append(hrinp(inp, witarg)) + outdict["inputs"].append(human_readable_input(inp, witarg)) for i, out in enumerate(tx.vout): - outdict["outputs"].append(hrout(out)) + outdict["outputs"].append(human_readable_output(out)) if not jsonified: return outdict return json.dumps(outdict, indent=4) -def hrinp(txinput, txinput_witness): +def human_readable_input(txinput, txinput_witness): """ Pass objects of type CTxIn and CTxInWitness (or None) and a dict of human-readable entries for this input is returned. @@ -66,7 +66,7 @@ def hrinp(txinput, txinput_witness): txinput_witness.scriptWitness.serialize()) return outdict -def hrout(txoutput): +def human_readable_output(txoutput): """ Returns a dict of human-readable entries for this output. """ @@ -232,7 +232,7 @@ def return_err(e): else: # segwit case; we currently support p2wpkh native or under p2sh. - # see line 1256 of bitcointx.core.scripteval.py: + # https://github.com/Simplexum/python-bitcointx/blob/648ad8f45ff853bf9923c6498bfa0648b3d7bcbd/bitcointx/core/scripteval.py#L1250-L1252 flags.add(SCRIPT_VERIFY_P2SH) if native and native != "p2wpkh": @@ -273,16 +273,6 @@ def return_err(e): return sig, "signing succeeded" -def apply_freeze_signature(tx, i, redeem_script, sig): - if isinstance(redeem_script, str): - redeem_script = binascii.unhexlify(redeem_script) - if isinstance(sig, str): - sig = binascii.unhexlify(sig) - txobj = deserialize(tx) - txobj["ins"][i]["script"] = "" - txobj["ins"][i]["txinwitness"] = [sig, redeem_script] - return serialize(txobj) - def mktx(ins, outs, version=1, locktime=0): """ Given a list of input tuples (txid(bytes), n(int)), and a list of outputs which are dicts with diff --git a/jmbitcoin/setup.py b/jmbitcoin/setup.py index 9330fc002..00adc0ae6 100644 --- a/jmbitcoin/setup.py +++ b/jmbitcoin/setup.py @@ -10,5 +10,5 @@ license='GPL', packages=['jmbitcoin'], python_requires='>=3.6', - install_requires=['coincurve', 'python-bitcointx>=1.0.5', 'pyaes', 'urldecode'], + install_requires=['coincurve', 'python-bitcointx>=1.1.0', 'pyaes', 'urldecode'], zip_safe=False) diff --git a/jmbitcoin/test/test_tx_signing.py b/jmbitcoin/test/test_tx_signing.py index e2ccf21f2..28ca2bae2 100644 --- a/jmbitcoin/test/test_tx_signing.py +++ b/jmbitcoin/test/test_tx_signing.py @@ -61,7 +61,8 @@ def test_sign_standard_txs(addrtype): raise print("created signature: ", bintohex(sig)) print("serialized transaction: {}".format(bintohex(tx.serialize()))) - print("deserialized transaction: {}\n".format(btc.hrt(tx))) + print("deserialized transaction: {}\n".format( + btc.human_readable_transaction(tx))) def test_mk_shuffled_tx(): # prepare two addresses for the outputs diff --git a/jmclient/jmclient/__init__.py b/jmclient/jmclient/__init__.py index d222478c2..0a486580e 100644 --- a/jmclient/jmclient/__init__.py +++ b/jmclient/jmclient/__init__.py @@ -21,13 +21,12 @@ from .cryptoengine import (BTCEngine, BTC_P2PKH, BTC_P2SH_P2WPKH, EngineError, TYPE_P2PKH, TYPE_P2SH_P2WPKH, TYPE_P2WPKH) from .configure import (load_test_config, - load_program_config, get_p2pk_vbyte, jm_single, get_network, update_persist_config, + load_program_config, jm_single, get_network, update_persist_config, validate_address, is_burn_destination, get_irc_mchannels, - get_blockchain_interface_instance, get_p2sh_vbyte, set_config, is_segwit_mode, + get_blockchain_interface_instance, set_config, is_segwit_mode, is_native_segwit_mode) from .blockchaininterface import (BlockchainInterface, RegtestBitcoinCoreInterface, BitcoinCoreInterface) -from .electruminterface import ElectrumInterface from .client_protocol import (JMTakerClientProtocol, JMClientProtocolFactory, start_reactor) from .podle import (set_commitment_file, get_commitment_file, diff --git a/jmclient/jmclient/configure.py b/jmclient/jmclient/configure.py index 17c885c3a..c2cb92030 100644 --- a/jmclient/jmclient/configure.py +++ b/jmclient/jmclient/configure.py @@ -366,15 +366,6 @@ def get_network(): """Returns network name""" return global_singleton.config.get("BLOCKCHAIN", "network") - -def get_p2sh_vbyte(): - return btc.BTC_P2SH_VBYTE[get_network()] - - -def get_p2pk_vbyte(): - return btc.BTC_P2PK_VBYTE[get_network()] - - def validate_address(addr): try: # automatically respects the network @@ -570,7 +561,8 @@ def get_blockchain_interface_instance(_config): elif source == "bitcoin-rpc-no-history": bc_interface = BitcoinCoreNoHistoryInterface(rpc, network) if testnet or network == "regtest": - # TODO will not work for bech32 regtest addresses: + # in tests, for bech32 regtest addresses, for bc-no-history, + # this will have to be reset manually: btc.select_chain_params("bitcoin/testnet") else: btc.select_chain_params("bitcoin") diff --git a/jmclient/jmclient/electrum_data.py b/jmclient/jmclient/electrum_data.py deleted file mode 100644 index 393f49c0b..000000000 --- a/jmclient/jmclient/electrum_data.py +++ /dev/null @@ -1,261 +0,0 @@ -# Default server list from electrum client -# https://github.com/spesmilo/electrum, file https://github.com/spesmilo/electrum/blob/7dbd612d5dad13cd6f1c0df32534a578bad331ad/lib/servers.json - -#Edit this to 't' instead of 's' to use TCP; -#This is specifically not exposed in joinmarket.cfg -#since there is no good reason to prefer TCP over SSL -#unless the latter simply doesn't work. -DEFAULT_PROTO = 's' - -DEFAULT_PORTS = {'t':'50001', 's':'50002'} - -DEFAULT_SERVERS = { - "E-X.not.fyi": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "ELECTRUMX.not.fyi": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "ELEX01.blackpole.online": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "VPS.hsmiths.com": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "bitcoin.freedomnode.com": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "btc.smsys.me": { - "pruning": "-", - "s": "995", - "version": "1.1" - }, - "currentlane.lovebitco.in": { - "pruning": "-", - "t": "50001", - "version": "1.1" - }, - "daedalus.bauerj.eu": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "de01.hamster.science": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "ecdsa.net": { - "pruning": "-", - "s": "110", - "t": "50001", - "version": "1.1" - }, - "elec.luggs.co": { - "pruning": "-", - "s": "443", - "version": "1.1" - }, - "electrum.akinbo.org": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "electrum.antumbra.se": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "electrum.be": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "electrum.coinucopia.io": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "electrum.cutie.ga": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "electrum.festivaldelhumor.org": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "electrum.hsmiths.com": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "electrum.qtornado.com": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "electrum.vom-stausee.de": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "electrum3.hachre.de": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "electrumx.bot.nu": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "electrumx.westeurope.cloudapp.azure.com": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "elx01.knas.systems": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "ex-btc.server-on.net": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "helicarrier.bauerj.eu": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "mooo.not.fyi": { - "pruning": "-", - "s": "50012", - "t": "50011", - "version": "1.1" - }, - "ndnd.selfhost.eu": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "node.arihanc.com": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "node.xbt.eu": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "node1.volatilevictory.com": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "noserver4u.de": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "qmebr.spdns.org": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "raspi.hsmiths.com": { - "pruning": "-", - "s": "51002", - "t": "51001", - "version": "1.1" - }, - "s2.noip.pl": { - "pruning": "-", - "s": "50102", - "version": "1.1" - }, - "s5.noip.pl": { - "pruning": "-", - "s": "50105", - "version": "1.1" - }, - "songbird.bauerj.eu": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "us.electrum.be": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - }, - "us01.hamster.science": { - "pruning": "-", - "s": "50002", - "t": "50001", - "version": "1.1" - } -} - -def set_electrum_testnet(): - global DEFAULT_PORTS, DEFAULT_SERVERS - DEFAULT_PORTS = {'t':'51001', 's':'51002'} - DEFAULT_SERVERS = { - 'testnetnode.arihanc.com': {'t':'51001', 's':'51002'}, - 'testnet1.bauerj.eu': {'t':'51001', 's':'51002'}, - #'14.3.140.101': {'t':'51001', 's':'51002'}, #non-responsive? - 'testnet.hsmiths.com': {'t':'53011', 's':'53012'}, - 'electrum.akinbo.org': {'t':'51001', 's':'51002'}, - 'ELEX05.blackpole.online': {'t':'52011', 's':'52002'},} - #Replace with for regtest: - #'localhost': {'t': '50001', 's': '51002'},} - -def get_default_servers(): - return DEFAULT_SERVERS - -def get_default_ports(): - return DEFAULT_PORTS \ No newline at end of file diff --git a/jmclient/jmclient/electruminterface.py b/jmclient/jmclient/electruminterface.py deleted file mode 100644 index 773d83042..000000000 --- a/jmclient/jmclient/electruminterface.py +++ /dev/null @@ -1,549 +0,0 @@ -import jmbitcoin as btc -import json -import queue as Queue -import os -import pprint -import random -import socket -import threading -import ssl -import binascii -from twisted.internet.protocol import ClientFactory -from twisted.internet.ssl import ClientContextFactory -from twisted.protocols.basic import LineReceiver -from twisted.internet import reactor, task, defer -from .blockchaininterface import BlockchainInterface -from .configure import get_p2sh_vbyte -from jmbase import get_log, jmprint -from .electrum_data import get_default_servers, set_electrum_testnet,\ - DEFAULT_PROTO - -log = get_log() - -class ElectrumConnectionError(Exception): - pass - -class TxElectrumClientProtocol(LineReceiver): - #map deferreds to msgids to correctly link response with request - deferreds = {} - delimiter = b"\n" - - def __init__(self, factory): - self.factory = factory - - def connectionMade(self): - log.debug('connection to Electrum succesful') - self.msg_id = 0 - if self.factory.bci.wallet: - #Use connectionMade as a trigger to start wallet sync, - #if the reactor start happened after the call to wallet sync - #(in Qt, the reactor starts before wallet sync, so we make - #this call manually instead). - self.factory.bci.sync_addresses(self.factory.bci.wallet) - #these server calls must always be done to keep the connection open - self.start_ping() - self.call_server_method('blockchain.numblocks.subscribe') - - def start_ping(self): - pingloop = task.LoopingCall(self.ping) - pingloop.start(60.0) - - def ping(self): - #We dont bother tracking response to this; - #just for keeping connection active - self.call_server_method('server.version') - - def send_json(self, json_data): - data = json.dumps(json_data).encode() - self.sendLine(data) - - def call_server_method(self, method, params=[]): - self.msg_id = self.msg_id + 1 - current_id = self.msg_id - self.deferreds[current_id] = defer.Deferred() - method_dict = { - 'id': current_id, - 'method': method, - 'params': params - } - self.send_json(method_dict) - return self.deferreds[current_id] - - def lineReceived(self, line): - try: - parsed = json.loads(line.decode()) - msgid = parsed['id'] - linked_deferred = self.deferreds[msgid] - except: - log.debug("Ignored response from Electrum server: " + str(line)) - return - linked_deferred.callback(parsed) - -class TxElectrumClientProtocolFactory(ClientFactory): - - def __init__(self, bci): - self.bci = bci - def buildProtocol(self,addr): - self.client = TxElectrumClientProtocol(self) - return self.client - - def clientConnectionLost(self, connector, reason): - log.debug('Electrum connection lost, reason: ' + str(reason)) - self.bci.start_electrum_proto(None) - - def clientConnectionFailed(self, connector, reason): - jmprint('connection failed', "warning") - self.bci.start_electrum_proto(None) - -class ElectrumConn(threading.Thread): - - def __init__(self, server, port, proto): - threading.Thread.__init__(self) - self.daemon = True - self.msg_id = 0 - self.RetQueue = Queue.Queue() - try: - if proto == 't': - self.s = socket.create_connection((server,int(port))) - elif proto == 's': - self.raw_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - #reads are sometimes quite slow, so conservative, but we must - #time out a completely hanging connection. - self.raw_socket.settimeout(60) - self.raw_socket.connect((server, int(port))) - self.s = ssl.wrap_socket(self.raw_socket) - else: - #Wrong proto is not accepted for restarts - log.error("Failure to connect to Electrum, " - "protocol must be TCP or SSL.") - os._exit(1) - except Exception as e: - log.error("Error connecting to electrum server; trying again.") - raise ElectrumConnectionError - self.ping() - - def run(self): - while True: - all_data = None - while True: - data = self.s.recv(1024) - if data is None: - continue - if all_data is None: - all_data = data - else: - all_data = all_data + data - if b'\n' in all_data: - break - data_json = json.loads(all_data[:-1].decode()) - self.RetQueue.put(data_json) - - def ping(self): - log.debug('Sending Electrum server ping') - self.send_json({'id':0,'method':'server.version','params':[]}) - t = threading.Timer(60, self.ping) - t.daemon = True - t.start() - - def send_json(self, json_data): - data = json.dumps(json_data).encode() - self.s.send(data + b'\n') - - def call_server_method(self, method, params=[]): - self.msg_id = self.msg_id + 1 - current_id = self.msg_id - method_dict = { - 'id': current_id, - 'method': method, - 'params': params - } - self.send_json(method_dict) - while True: - ret_data = self.RetQueue.get() - if ret_data.get('id', None) == current_id: - return ret_data - else: - log.debug(json.dumps(ret_data)) - -class ElectrumInterface(BlockchainInterface): - BATCH_SIZE = 8 - def __init__(self, testnet=False, electrum_server=None): - self.synctype = "sync-only" - if testnet: - set_electrum_testnet() - self.start_electrum_proto() - self.electrum_conn = None - self.start_connection_thread() - #task.LoopingCall objects that track transactions, keyed by txids. - #Format: {"txid": (loop, unconfirmed true/false, confirmed true/false, - #spent true/false), ..} - self.tx_watcher_loops = {} - self.wallet = None - self.wallet_synced = False - - def start_electrum_proto(self, electrum_server=None): - self.server, self.port = self.get_server(electrum_server) - self.factory = TxElectrumClientProtocolFactory(self) - if DEFAULT_PROTO == 's': - ctx = ClientContextFactory() - reactor.connectSSL(self.server, self.port, self.factory, ctx) - elif DEFAULT_PROTO == 't': - reactor.connectTCP(self.server, self.port, self.factory) - else: - raise Exception("Unrecognized connection protocol to Electrum, " - "should be one of 't' or 's' (TCP or SSL), " - "critical error, quitting.") - - def start_connection_thread(self): - """Initiate a thread that serves blocking, single - calls to an Electrum server. This won't usually be the - same server that's used to do sync (which, confusingly, - is asynchronous). - """ - try: - s, p = self.get_server(None) - self.electrum_conn = ElectrumConn(s, p, DEFAULT_PROTO) - except ElectrumConnectionError: - reactor.callLater(1.0, self.start_connection_thread) - return - self.electrum_conn.start() - #used to hold open server conn - self.electrum_conn.call_server_method('blockchain.numblocks.subscribe') - - def sync_wallet(self, wallet, fast=False, restart_cb=False): - """This triggers the start of syncing, wiping temporary state - and starting the reactor for wallet-tool runs. The 'fast' - and 'restart_cb' parameters are ignored and included only - for compatibility; they are both only used by Core. - """ - self.wallet = wallet - #wipe the temporary cache of address histories - self.temp_addr_history = {} - #mark as not currently synced - self.wallet_synced = False - if self.synctype == "sync-only": - if not reactor.running: - reactor.run() - - def get_server(self, electrum_server): - if not electrum_server: - while True: - electrum_server = random.choice(list(get_default_servers().keys())) - if DEFAULT_PROTO in get_default_servers()[electrum_server]: - break - s = electrum_server - p = int(get_default_servers()[electrum_server][DEFAULT_PROTO]) - log.debug('Trying to connect to Electrum server: ' + str(electrum_server)) - return (s, p) - - def get_from_electrum(self, method, params=[], blocking=False): - params = [params] if type(params) is not list else params - if blocking: - return self.electrum_conn.call_server_method(method, params) - else: - return self.factory.client.call_server_method(method, params) - - def sync_addresses(self, wallet, restart_cb=None): - if not self.electrum_conn: - #wait until we have some connection up before starting - reactor.callLater(0.2, self.sync_addresses, wallet, restart_cb) - return - log.debug("downloading wallet history from Electrum server ...") - for mixdepth in range(wallet.max_mixdepth + 1): - for forchange in [0, 1]: - #start from a clean index - wallet.set_next_index(mixdepth, forchange, 0) - self.synchronize_batch(wallet, mixdepth, forchange, 0) - - def synchronize_batch(self, wallet, mixdepth, forchange, start_index): - #for debugging only: - #log.debug("Syncing address batch, m, fc, i: " + ",".join( - # [str(x) for x in [mixdepth, forchange, start_index]])) - if mixdepth not in self.temp_addr_history: - self.temp_addr_history[mixdepth] = {} - if forchange not in self.temp_addr_history[mixdepth]: - self.temp_addr_history[mixdepth][forchange] = {"finished": False} - for i in range(start_index, start_index + self.BATCH_SIZE): - #get_new_addr is OK here, as guaranteed to be sequential *on this branch* - a = wallet.get_new_addr(mixdepth, forchange) - d = self.get_from_electrum('blockchain.address.get_history', a) - #makes sure entries in temporary address history are ready - #to be accessed. - if i not in self.temp_addr_history[mixdepth][forchange]: - self.temp_addr_history[mixdepth][forchange][i] = {'synced': False, - 'addr': a, - 'used': False} - d.addCallback(self.process_address_history, wallet, - mixdepth, forchange, i, a, start_index) - - def process_address_history(self, history, wallet, mixdepth, forchange, i, - addr, start_index): - """Given the history data for an address from Electrum, update the current view - of the wallet's usage at mixdepth mixdepth and account forchange, address addr at - index i. Once all addresses from index start_index to start_index + self.BATCH_SIZE - have been thus updated, trigger either continuation to the next batch, or, if - conditions are fulfilled, end syncing for this (mixdepth, forchange) branch, and - if all such branches are finished, proceed to the sync_unspent step. - """ - tah = self.temp_addr_history[mixdepth][forchange] - if len(history['result']) > 0: - tah[i]['used'] = True - tah[i]['synced'] = True - #Having updated this specific record, check if the entire batch from start_index - #has been synchronized - if all([tah[j]['synced'] for j in range(start_index, start_index + self.BATCH_SIZE)]): - #check if unused goes back as much as gaplimit *and* we are ahead of any - #existing index_cache from the wallet file; if both true, end, else, continue - #to next batch - if all([tah[j]['used'] is False for j in range( - start_index + self.BATCH_SIZE - wallet.gap_limit, - start_index + self.BATCH_SIZE)]): - last_used_addr = None - #to find last used, note that it may be in the *previous* batch; - #may as well just search from the start, since it takes no time. - for j in range(start_index + self.BATCH_SIZE): - if tah[j]['used']: - last_used_addr = tah[j]['addr'] - if last_used_addr: - wallet.set_next_index( - mixdepth, forchange, - wallet.get_next_unused_index(mixdepth, forchange)) - else: - wallet.set_next_index(mixdepth, forchange, 0) - tah["finished"] = True - #check if all branches are finished to trigger next stage of sync. - addr_sync_complete = True - for m in range(wallet.max_mix_depth): - for fc in [0, 1]: - if not self.temp_addr_history[m][fc]["finished"]: - addr_sync_complete = False - if addr_sync_complete: - self.sync_unspent(wallet) - else: - #continue search forwards on this branch - self.synchronize_batch(wallet, mixdepth, forchange, start_index + self.BATCH_SIZE) - - def sync_unspent(self, wallet): - # finds utxos in the wallet - wallet.reset_utxos() - #Prepare list of all used addresses - addrs = set() - for m in range(wallet.max_mixdepth): - for fc in [0, 1]: - branch_list = [] - for k, v in self.temp_addr_history[m][fc].items(): - if k == "finished": - continue - if v["used"]: - branch_list.append(v["addr"]) - addrs.update(branch_list) - if len(addrs) == 0: - log.debug('no tx used') - self.wallet_synced = True - if self.synctype == 'sync-only': - reactor.stop() - return - #make sure to add any addresses during the run (a subset of those - #added to the address cache) - for md in range(wallet.max_mixdepth): - for internal in (True, False): - for index in range(wallet.get_next_unused_index(md, internal)): - addrs.add(wallet.get_addr(md, internal, index)) - for path in wallet.yield_imported_paths(md): - addrs.add(wallet.get_address_from_path(path)) - - self.listunspent_calls = len(addrs) - for a in addrs: - # FIXME: update to protocol version 1.1 and use scripthash instead - script = wallet.addr_to_script(a) - d = self.get_from_electrum('blockchain.address.listunspent', a) - d.addCallback(self.process_listunspent_data, wallet, script) - - def process_listunspent_data(self, unspent_info, wallet, script): - res = unspent_info['result'] - for u in res: - txid = binascii.unhexlify(u['tx_hash']) - wallet.add_utxo(txid, int(u['tx_pos']), script, int(u['value'])) - - self.listunspent_calls -= 1 - if self.listunspent_calls == 0: - self.wallet_synced = True - if self.synctype == "sync-only": - reactor.stop() - - def pushtx(self, txhex): - brcst_res = self.get_from_electrum('blockchain.transaction.broadcast', - txhex, blocking=True) - brcst_status = brcst_res['result'] - if isinstance(brcst_status, str) and len(brcst_status) == 64: - return (True, brcst_status) - log.debug(brcst_status) - return (False, None) - - def query_utxo_set(self, txout, includeconf=False): - self.current_height = self.get_from_electrum( - "blockchain.numblocks.subscribe", blocking=True)['result'] - if not isinstance(txout, list): - txout = [txout] - utxos = [[t[:64],int(t[65:])] for t in txout] - result = [] - for ut in utxos: - address = self.get_from_electrum("blockchain.utxo.get_address", - ut, blocking=True)['result'] - utxo_info = self.get_from_electrum("blockchain.address.listunspent", - address, blocking=True)['result'] - utxo = None - for u in utxo_info: - if u['tx_hash'] == ut[0] and u['tx_pos'] == ut[1]: - utxo = u - if utxo is None: - result.append(None) - else: - r = { - 'value': utxo['value'], - 'address': address, - 'script': btc.address_to_script(address) - } - if includeconf: - if int(utxo['height']) in [0, -1]: - #-1 means unconfirmed inputs - r['confirms'] = 0 - else: - #+1 because if current height = tx height, that's 1 conf - r['confirms'] = int(self.current_height) - int( - utxo['height']) + 1 - result.append(r) - return result - - def estimate_fee_per_kb(self, N): - if super(ElectrumInterface, self).fee_per_kb_has_been_manually_set(N): - return int(random.uniform(N * float(0.8), N * float(1.2))) - fee_info = self.get_from_electrum('blockchain.estimatefee', N, blocking=True) - jmprint('got fee info result: ' + str(fee_info), "debug") - fee = fee_info.get('result') - fee_per_kb_sat = int(float(fee) * 100000000) - return fee_per_kb_sat - - def outputs_watcher(self, wallet_name, notifyaddr, tx_output_set, - unconfirmfun, confirmfun, timeoutfun): - """Given a key for the watcher loop (notifyaddr), a wallet name (account), - a set of outputs, and unconfirm, confirm and timeout callbacks, - check to see if a transaction matching that output set has appeared in - the wallet. Call the callbacks and update the watcher loop state. - End the loop when the confirmation has been seen (no spent monitoring here). - """ - wl = self.tx_watcher_loops[notifyaddr] - jmprint('txoutset=' + pprint.pformat(tx_output_set), "debug") - unconftx = self.get_from_electrum('blockchain.address.get_mempool', - notifyaddr, blocking=True).get('result') - unconftxs = set([str(t['tx_hash']) for t in unconftx]) - if len(unconftxs): - txdatas = [] - for txid in unconftxs: - txdatas.append({'id': txid, - 'hex':str(self.get_from_electrum( - 'blockchain.transaction.get',txid, - blocking=True).get('result'))}) - unconfirmed_txid = None - for txdata in txdatas: - txhex = txdata['hex'] - outs = set([(sv['script'], sv['value']) for sv in btc.deserialize( - txhex)['outs']]) - jmprint('unconfirm query outs = ' + str(outs), "debug") - if outs == tx_output_set: - unconfirmed_txid = txdata['id'] - unconfirmed_txhex = txhex - break - #call unconf callback if it was found in the mempool - if unconfirmed_txid and not wl[1]: - jmprint("Tx: " + str(unconfirmed_txid) + " seen on network.", "info") - unconfirmfun(btc.deserialize(unconfirmed_txhex), unconfirmed_txid) - wl[1] = True - return - - conftx = self.get_from_electrum('blockchain.address.listunspent', - notifyaddr, blocking=True).get('result') - conftxs = set([str(t['tx_hash']) for t in conftx]) - if len(conftxs): - txdatas = [] - for txid in conftxs: - txdata = str(self.get_from_electrum('blockchain.transaction.get', - txid, blocking=True).get('result')) - txdatas.append({'hex':txdata,'id':txid}) - confirmed_txid = None - for txdata in txdatas: - txhex = txdata['hex'] - outs = set([(sv['script'], sv['value']) for sv in btc.deserialize( - txhex)['outs']]) - jmprint('confirm query outs = ' + str(outs), "info") - if outs == tx_output_set: - confirmed_txid = txdata['id'] - confirmed_txhex = txhex - break - if confirmed_txid and not wl[2]: - confirmfun(btc.deserialize(confirmed_txhex), confirmed_txid, 1) - wl[2] = True - wl[0].stop() - return - - def tx_watcher(self, txd, unconfirmfun, confirmfun, spentfun, c, n): - """Called at a polling interval, checks if the given deserialized - transaction (which must be fully signed) is (a) broadcast, (b) confirmed - and (c) spent from. (c, n ignored in electrum version, just supports - registering first confirmation). - TODO: There is no handling of conflicts here. - """ - txid = btc.txhash(btc.serialize(txd)) - wl = self.tx_watcher_loops[txid] - #first check if in mempool (unconfirmed) - #choose an output address for the query. Filter out - #p2pkh addresses, assume p2sh (thus would fail to find tx on - #some nonstandard script type) - addr = None - for i in range(len(txd['outs'])): - if not btc.is_p2pkh_script(txd['outs'][i]['script']): - addr = btc.script_to_address(txd['outs'][i]['script'], get_p2sh_vbyte()) - break - if not addr: - log.error("Failed to find any p2sh output, cannot be a standard " - "joinmarket transaction, fatal error!") - reactor.stop() - return - unconftxs_res = self.get_from_electrum('blockchain.address.get_mempool', - addr, blocking=True).get('result') - unconftxs = [str(t['tx_hash']) for t in unconftxs_res] - - if not wl[1] and txid in unconftxs: - jmprint("Tx: " + str(txid) + " seen on network.", "info") - unconfirmfun(txd, txid) - wl[1] = True - return - conftx = self.get_from_electrum('blockchain.address.listunspent', - addr, blocking=True).get('result') - conftxs = [str(t['tx_hash']) for t in conftx] - if not wl[2] and len(conftxs) and txid in conftxs: - jmprint("Tx: " + str(txid) + " is confirmed.", "info") - confirmfun(txd, txid, 1) - wl[2] = True - #Note we do not stop the monitoring loop when - #confirmations occur, since we are also monitoring for spending. - return - if not spentfun or wl[3]: - return - - def rpc(self, method, args): - # FIXME: this is very poorly written code - if method == 'gettransaction': - assert len(args) == 1 - return self._gettransaction(args[0]) - else: - raise NotImplementedError(method) - - def _gettransaction(self, txid): - # FIXME: this is not complete and only implemented to work with - # wallet_utils - return { - 'hex': str(self.get_from_electrum('blockchain.transaction.get', - txid, blocking=True) - .get('result')) - } diff --git a/jmclient/jmclient/maker.py b/jmclient/jmclient/maker.py index 40ffc3ad6..3838ec354 100644 --- a/jmclient/jmclient/maker.py +++ b/jmclient/jmclient/maker.py @@ -132,7 +132,7 @@ def on_tx_received(self, nick, tx_from_taker, offerinfo): return (False, 'malformed txhex. ' + repr(e)) # if the above deserialization was successful, the human readable # parsing will be also: - jlog.info('obtained tx\n' + btc.hrt(tx)) + jlog.info('obtained tx\n' + btc.human_readable_transaction(tx)) goodtx, errmsg = self.verify_unsigned_tx(tx, offerinfo) if not goodtx: jlog.info('not a good tx, reason=' + errmsg) diff --git a/jmclient/jmclient/payjoin.py b/jmclient/jmclient/payjoin.py index 9c116dde3..37bb4da4e 100644 --- a/jmclient/jmclient/payjoin.py +++ b/jmclient/jmclient/payjoin.py @@ -116,7 +116,7 @@ def sanity_check_initial_payment(self): # inputs must all have witness utxo populated for inp in self.initial_psbt.inputs: - if not inp.utxo and isinstance(inp.utxo, btc.CTxOut): + if not isinstance(inp.witness_utxo, btc.CTxOut): return False # check that there is no xpub or derivation info @@ -191,7 +191,7 @@ def check_receiver_proposal(self, in_pbst, signed_psbt_for_fees): else: receiver_input_indices.append(i) - if any([found[i] != 1 for i in range(len(found))]): + if any([f != 1 for f in found]): return (False, "Receiver proposed PSBT does not contain our inputs.") # 3 found = 0 @@ -229,9 +229,11 @@ def check_receiver_proposal(self, in_pbst, signed_psbt_for_fees): # version (so all witnesses filled in) to calculate its size, # then compare that with the fee, and do the same for the # pre-existing non-payjoin. - gffp = PSBTWalletMixin.get_fee_from_psbt - proposed_tx_fee = gffp(signed_psbt_for_fees) - nonpayjoin_tx_fee = gffp(self.initial_psbt) + try: + proposed_tx_fee = signed_psbt_for_fees.get_fee() + except ValueError: + return (False, "receiver proposed tx has negative fee.") + nonpayjoin_tx_fee = self.initial_psbt.get_fee() proposed_tx_size = signed_psbt_for_fees.extract_transaction( ).get_virtual_size() nonpayjoin_tx_size = self.initial_psbt.extract_transaction( @@ -329,16 +331,16 @@ def report(self, jsonified=False, verbose=False): reportdict = {"name:", "PAYJOIN STATUS REPORT"} reportdict["status"] = self.pj_state # TODO: string if self.payment_tx: - txdata = btc.hrt(self.payment_tx) + txdata = btc.human_readable_transaction(self.payment_tx) if verbose: txdata = txdata["hex"] reportdict["payment-tx"] = txdata if self.payjoin_psbt: - psbtdata = PSBTWalletMixin.hr_psbt( + psbtdata = PSBTWalletMixin.human_readable_psbt( self.payjoin_psbt) if verbose else self.payjoin_psbt.to_base64() reportdict["payjoin-proposed"] = psbtdata if self.final_psbt: - finaldata = PSBTWalletMixin.hr_psbt( + finaldata = PSBTWalletMixin.human_readable_psbt( self.final_psbt) if verbose else self.final_psbt.to_base64() reportdict["payjoin-final"] = finaldata if jsonified: @@ -427,12 +429,12 @@ def noResponse(failure): def fallback_nonpayjoin_broadcast(manager, err): assert isinstance(manager, JMPayjoinManager) log.warn("Payjoin did not succeed, falling back to non-payjoin payment.") - log.warn("Error message was: " + str(err)) + log.warn("Error message was: " + err.decode("utf-8")) original_tx = manager.initial_psbt.extract_transaction() if not jm_single().bc_interface.pushtx(original_tx.serialize()): log.error("Unable to broadcast original payment. The payment is NOT made.") log.info("We paid without coinjoin. Transaction: ") - log.info(btc.hrt(original_tx)) + log.info(btc.human_readable_transaction(original_tx)) reactor.stop() def receive_payjoin_proposal_from_server(response, manager): @@ -463,15 +465,15 @@ def process_payjoin_proposal_from_server(response_body, manager): return log.debug("Receiver sent us this PSBT: ") - log.debug(manager.wallet_service.hr_psbt(payjoin_proposal_psbt)) + log.debug(manager.wallet_service.human_readable_psbt(payjoin_proposal_psbt)) # we need to add back in our utxo information to the received psbt, # since the servers remove it (not sure why?) for i, inp in enumerate(payjoin_proposal_psbt.unsigned_tx.vin): for j, inp2 in enumerate(manager.initial_psbt.unsigned_tx.vin): if (inp.prevout.hash, inp.prevout.n) == ( inp2.prevout.hash, inp2.prevout.n): - payjoin_proposal_psbt.inputs[i].utxo = \ - manager.initial_psbt.inputs[j].utxo + payjoin_proposal_psbt.set_utxo( + manager.initial_psbt.inputs[j].utxo, i) signresultandpsbt, err = manager.wallet_service.sign_psbt( payjoin_proposal_psbt.serialize(), with_sign_result=True) if err: @@ -489,15 +491,15 @@ def process_payjoin_proposal_from_server(response_body, manager): # All checks have passed. We can use the already signed transaction in # sender_signed_psbt. log.info("Our final signed PSBT is:\n{}".format( - manager.wallet_service.hr_psbt(sender_signed_psbt))) + manager.wallet_service.human_readable_psbt(sender_signed_psbt))) manager.set_final_payjoin_psbt(sender_signed_psbt) # broadcast the tx extracted_tx = sender_signed_psbt.extract_transaction() log.info("Here is the final payjoin transaction:") - log.info(btc.hrt(extracted_tx)) + log.info(btc.human_readable_transaction(extracted_tx)) if not jm_single().bc_interface.pushtx(extracted_tx.serialize()): log.info("The above transaction failed to broadcast.") else: - log.info("Payjoin transactoin broadcast successfully.") + log.info("Payjoin transaction broadcast successfully.") reactor.stop() diff --git a/jmclient/jmclient/taker.py b/jmclient/jmclient/taker.py index 25d5d1a90..3650923ef 100644 --- a/jmclient/jmclient/taker.py +++ b/jmclient/jmclient/taker.py @@ -499,7 +499,8 @@ def receive_utxos(self, ioauth_data): self.outputs.append({'address': self.coinjoin_address(), 'value': self.cjamount}) self.latest_tx = btc.make_shuffled_tx(self.utxo_tx, self.outputs) - jlog.info('obtained tx\n' + btc.hrt(self.latest_tx)) + jlog.info('obtained tx\n' + btc.human_readable_transaction( + self.latest_tx)) for index, ins in enumerate(self.latest_tx.vin): utxo = (ins.prevout.hash[::-1], ins.prevout.n) @@ -1008,7 +1009,8 @@ def receive_utxos(self, ioauth_data): # contains only those. tx = btc.make_shuffled_tx(self.input_utxos, self.outputs, version=2, locktime=compute_tx_locktime()) - jlog.info('Created proposed fallback tx:\n' + btc.hrt(tx)) + jlog.info('Created proposed fallback tx:\n' + \ + btc.human_readable_transaction(tx)) # We now sign as a courtesy, because if we disappear the recipient # can still claim his coins with this. # sign our inputs before transfer diff --git a/jmclient/jmclient/taker_utils.py b/jmclient/jmclient/taker_utils.py index f57935673..17523856c 100644 --- a/jmclient/jmclient/taker_utils.py +++ b/jmclient/jmclient/taker_utils.py @@ -11,7 +11,8 @@ from .wallet import BaseWallet, estimate_tx_fee, compute_tx_locktime, \ FidelityBondMixin from jmbitcoin import make_shuffled_tx, amount_to_str, mk_burn_script,\ - PartiallySignedTransaction, CMutableTxOut, hrt, Hash160 + PartiallySignedTransaction, CMutableTxOut,\ + human_readable_transaction, Hash160 from jmbase.support import EXIT_SUCCESS log = get_log() @@ -165,7 +166,7 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, return False new_psbt_signed = PartiallySignedTransaction.deserialize(serialized_psbt) print("Completed PSBT created: ") - print(wallet_service.hr_psbt(new_psbt_signed)) + print(wallet_service.human_readable_psbt(new_psbt_signed)) return new_psbt_signed else: success, msg = wallet_service.sign_tx(tx, inscripts) @@ -173,7 +174,7 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, log.error("Failed to sign transaction, quitting. Error msg: " + msg) return log.info("Got signed transaction:\n") - log.info(hrt(tx)) + log.info(human_readable_transaction(tx)) actual_amount = amount if amount != 0 else total_inputs_val - fee_est log.info("Sends: " + amount_to_str(actual_amount) + " to destination: " + destination) if not answeryes: @@ -182,8 +183,8 @@ def direct_send(wallet_service, amount, mixdepth, destination, answeryes=False, log.info("You chose not to broadcast the transaction, quitting.") return False else: - accepted = accept_callback(hrt(tx), destination, actual_amount, - fee_est) + accepted = accept_callback(human_readable_transaction(tx), + destination, actual_amount, fee_est) if not accepted: return False jm_single().bc_interface.pushtx(tx.serialize()) diff --git a/jmclient/jmclient/wallet.py b/jmclient/jmclient/wallet.py index 2797e3c04..c013a8973 100644 --- a/jmclient/jmclient/wallet.py +++ b/jmclient/jmclient/wallet.py @@ -1007,13 +1007,6 @@ class PSBTWalletMixin(object): def __init__(self, storage, **kwargs): super(PSBTWalletMixin, self).__init__(storage, **kwargs) - @staticmethod - def get_fee_from_psbt(in_psbt): - assert isinstance(in_psbt, btc.PartiallySignedTransaction) - spent = sum(in_psbt.get_input_amounts()) - paid = sum((x.nValue for x in in_psbt.unsigned_tx.vout)) - return spent - paid - def is_input_finalized(self, psbt_input): """ This should be a convenience method in python-bitcointx. However note: this is not a static method and tacitly @@ -1032,7 +1025,7 @@ def is_input_finalized(self, psbt_input): return True @staticmethod - def hr_psbt(in_psbt): + def human_readable_psbt(in_psbt): """ Returns a jsonified indented string with all relevant information, in human readable form, contained in a PSBT. Warning: the output can be very verbose in certain cases. @@ -1054,17 +1047,20 @@ def hr_psbt(in_psbt): if in_psbt.unknown_fields: outdict["unknown-fields"] = str(in_psbt.unknown_fields) - outdict["unsigned-tx"] = btc.hrt(in_psbt.unsigned_tx, jsonified=False) + outdict["unsigned-tx"] = btc.human_readable_transaction( + in_psbt.unsigned_tx, jsonified=False) outdict["psbt-inputs"] = [] for inp in in_psbt.inputs: - outdict["psbt-inputs"].append(PSBTWalletMixin.hr_psbt_in(inp)) + outdict["psbt-inputs"].append( + PSBTWalletMixin.human_readable_psbt_in(inp)) outdict["psbt-outputs"] = [] for out in in_psbt.outputs: - outdict["psbt-outputs"].append(PSBTWalletMixin.hr_psbt_out(out)) + outdict["psbt-outputs"].append( + PSBTWalletMixin.human_readable_psbt_out(out)) return json.dumps(outdict, indent=4) @staticmethod - def hr_psbt_in(psbt_input): + def human_readable_psbt_in(psbt_input): """ Returns a dict containing human readable information about a bitcointx.core.psbt.PSBT_Input object. """ @@ -1074,7 +1070,7 @@ def hr_psbt_in(psbt_input): outdict["input-index"] = psbt_input.index if psbt_input.utxo: if isinstance(psbt_input.utxo, btc.CTxOut): - outdict["utxo"] = btc.hrout(psbt_input.utxo) + outdict["utxo"] = btc.human_readable_output(psbt_input.utxo) elif isinstance(psbt_input.utxo, btc.CTransaction): # human readable full transaction is *too* verbose: outdict["utxo"] = bintohex(psbt_input.utxo.serialize()) @@ -1116,7 +1112,7 @@ def hr_psbt_in(psbt_input): return outdict @staticmethod - def hr_psbt_out(psbt_output): + def human_readable_psbt_out(psbt_output): """ Returns a dict containing human readable information about a PSBT_Output object. """ @@ -1175,13 +1171,15 @@ def create_psbt_from_tx(self, tx, spent_outs=None): continue if isinstance(spent_outs[i], (btc.CTransaction, btc.CTxOut)): # note that we trust the caller to choose Tx vs TxOut as according - # to non-witness/witness: - txinput.utxo = spent_outs[i] + # to non-witness/witness. Note also that for now this mixin does + # not attempt to provide unsigned-tx(second argument) for witness + # case. + txinput.set_utxo(spent_outs[i], None) else: assert False, "invalid spent output type passed into PSBT creator" # we now insert redeemscripts where that is possible and necessary: for i, txinput in enumerate(new_psbt.inputs): - if isinstance(txinput.utxo, btc.CTxOut): + if isinstance(txinput.witness_utxo, btc.CTxOut): # witness if txinput.utxo.scriptPubKey.is_witness_scriptpubkey(): # nothing needs inserting; the scriptSig is empty. @@ -1228,7 +1226,7 @@ def sign_psbt(self, in_psbt, with_sign_result=False): # then overwriting it is harmless (preimage resistance). if isinstance(self, SegwitLegacyWallet): for i, txinput in enumerate(new_psbt.inputs): - tu = txinput.utxo + tu = txinput.witness_utxo if isinstance(tu, btc.CTxOut): # witness if tu.scriptPubKey.is_witness_scriptpubkey(): diff --git a/jmclient/test/commontest.py b/jmclient/test/commontest.py index e06eaf0f5..f4c849c26 100644 --- a/jmclient/test/commontest.py +++ b/jmclient/test/commontest.py @@ -6,7 +6,7 @@ import random from decimal import Decimal -from jmbase import (get_log, hextobin, dictchanger) +from jmbase import (get_log, hextobin, bintohex, dictchanger) from jmclient import ( jm_single, open_test_wallet_maybe, estimate_tx_fee, @@ -120,8 +120,10 @@ def estimate_fee_per_kb(self, N): def create_wallet_for_sync(wallet_structure, a, **kwargs): #We need a distinct seed for each run so as not to step over each other; - #make it through a deterministic hash - seedh = btc.b2x(btc.Hash("".join([str(x) for x in a]).encode("utf-8")))[:32] + #make it through a deterministic hash of all parameters including optionals. + preimage = "".join([str(x) for x in a] + [str(y) for y in kwargs.values()]).encode("utf-8") + print("using preimage: ", preimage) + seedh = bintohex(btc.Hash(preimage))[:32] return make_wallets( 1, [wallet_structure], fixed_seeds=[seedh], **kwargs)[0]['wallet'] @@ -191,8 +193,9 @@ def make_wallets(n, if len(wallet_structures) != n: raise Exception("Number of wallets doesn't match wallet structures") if not fixed_seeds: - seeds = chunks(binascii.hexlify(os.urandom(BIP32Wallet.ENTROPY_BYTES * n)).decode('ascii'), - BIP32Wallet.ENTROPY_BYTES * 2) + seeds = chunks(bintohex(os.urandom( + BIP32Wallet.ENTROPY_BYTES * n)), + BIP32Wallet.ENTROPY_BYTES * 2) else: seeds = fixed_seeds wallets = {} diff --git a/jmclient/test/test_configure.py b/jmclient/test/test_configure.py index cb488d2fe..2c5eb030a 100644 --- a/jmclient/test/test_configure.py +++ b/jmclient/test/test_configure.py @@ -3,8 +3,8 @@ import pytest import struct from jmclient import load_test_config, jm_single, get_irc_mchannels -from jmclient.configure import (get_config_irc_channel, get_p2sh_vbyte, - get_p2pk_vbyte, get_blockchain_interface_instance) +from jmclient.configure import (get_config_irc_channel, + get_blockchain_interface_instance) def test_attribute_dict(): @@ -35,12 +35,6 @@ def test_config_get_irc_channel(): load_test_config() -def test_net_byte(): - load_test_config() - assert struct.unpack(b'B', get_p2pk_vbyte())[0] == 0x6f - assert struct.unpack(b'B', get_p2sh_vbyte())[0] == 196 - - def test_blockchain_sources(): load_test_config() for src in ["electrum", "dummy"]: diff --git a/jmclient/test/test_core_nohistory_sync.py b/jmclient/test/test_core_nohistory_sync.py index ded8eb1c6..7c8c083fa 100644 --- a/jmclient/test/test_core_nohistory_sync.py +++ b/jmclient/test/test_core_nohistory_sync.py @@ -7,29 +7,35 @@ import pytest from jmbase import get_log -from jmclient import load_test_config +from jmclient import (load_test_config, SegwitLegacyWallet, + SegwitWallet, jm_single) +from jmbitcoin import select_chain_params log = get_log() def test_fast_sync_unavailable(setup_sync): - load_test_config(bs="bitcoin-rpc-no-history") wallet_service = create_wallet_for_sync([0, 0, 0, 0, 0], ['test_fast_sync_unavailable']) with pytest.raises(RuntimeError) as e_info: wallet_service.sync_wallet(fast=True) -@pytest.mark.parametrize('internal', (False, True)) -def test_sync(setup_sync, internal): - load_test_config(bs="bitcoin-rpc-no-history") +@pytest.mark.parametrize('internal, wallet_cls', [(False, SegwitLegacyWallet), + (True, SegwitLegacyWallet), + (False, SegwitWallet), + (True, SegwitWallet)]) +def test_sync(setup_sync, internal, wallet_cls): used_count = [1, 3, 6, 2, 23] wallet_service = create_wallet_for_sync(used_count, ['test_sync'], - populate_internal=internal) + populate_internal=internal, wallet_cls=wallet_cls) ##the gap limit should be not zero before sync assert wallet_service.gap_limit > 0 for md in range(len(used_count)): ##obtaining an address should be possible without error before sync wallet_service.get_new_script(md, internal) + # TODO bci should probably not store this state globally, + # in case syncing is needed for multiple wallets (as in this test): + jm_single().bc_interface.import_addresses_call_count = 0 wallet_service.sync_wallet(fast=False) for md in range(len(used_count)): @@ -45,4 +51,10 @@ def test_sync(setup_sync, internal): @pytest.fixture(scope='module') def setup_sync(): - pass + load_test_config(bs="bitcoin-rpc-no-history") + # a special case needed for the bitcoin core + # no history interface: it does not use + # 'blockchain_source' to distinguish regtest, + # so it must be set specifically for the test + # here: + select_chain_params("bitcoin/regtest") diff --git a/jmclient/test/test_psbt_wallet.py b/jmclient/test/test_psbt_wallet.py index 18f88d744..5c59e00f6 100644 --- a/jmclient/test/test_psbt_wallet.py +++ b/jmclient/test/test_psbt_wallet.py @@ -131,7 +131,7 @@ def test_create_psbt_and_sign(setup_psbt_wallet, unowned_utxo, wallet_cls): newpsbt.inputs[-1].redeem_script = redeem_script print(bintohex(newpsbt.serialize())) print("human readable: ") - print(wallet_service.hr_psbt(newpsbt)) + print(wallet_service.human_readable_psbt(newpsbt)) # we cannot compare with a fixed expected result due to wallet randomization, but we can # check psbt structure: expected_inputs_length = 3 if unowned_utxo else 2 @@ -213,8 +213,8 @@ def test_payjoin_workflow(setup_psbt_wallet, payment_amt, wallet_cls_sender, info_callback=dummy_info_callback, with_final_psbt=True) - print("Initial payment PSBT created:\n{}".format(wallet_s.hr_psbt( - payment_psbt))) + print("Initial payment PSBT created:\n{}".format( + wallet_s.human_readable_psbt(payment_psbt))) # ensure that the payemnt amount is what was intended: out_amts = [x.nValue for x in payment_psbt.unsigned_tx.vout] # NOTE this would have to change for more than 2 outputs: @@ -278,7 +278,7 @@ def test_payjoin_workflow(setup_psbt_wallet, payment_amt, wallet_cls_sender, version=payment_psbt.unsigned_tx.nVersion, locktime=payment_psbt.unsigned_tx.nLockTime) print("we created this unsigned tx: ") - print(bitcoin.hrt(unsigned_payjoin_tx)) + print(bitcoin.human_readable_transaction(unsigned_payjoin_tx)) # to create the PSBT we need the spent_outs for each input, # in the right order: spent_outs = [] @@ -306,7 +306,7 @@ def test_payjoin_workflow(setup_psbt_wallet, payment_amt, wallet_cls_sender, r_payjoin_psbt = wallet_r.create_psbt_from_tx(unsigned_payjoin_tx, spent_outs=spent_outs) print("Receiver created payjoin PSBT:\n{}".format( - wallet_r.hr_psbt(r_payjoin_psbt))) + wallet_r.human_readable_psbt(r_payjoin_psbt))) signresultandpsbt, err = wallet_r.sign_psbt(r_payjoin_psbt.serialize(), with_sign_result=True) @@ -316,7 +316,7 @@ def test_payjoin_workflow(setup_psbt_wallet, payment_amt, wallet_cls_sender, assert not signresult.is_final print("Receiver signing successful. Payjoin PSBT is now:\n{}".format( - wallet_r.hr_psbt(receiver_signed_psbt))) + wallet_r.human_readable_psbt(receiver_signed_psbt))) # *** STEP 3 *** # ************** @@ -328,7 +328,7 @@ def test_payjoin_workflow(setup_psbt_wallet, payment_amt, wallet_cls_sender, assert not err, err signresult, sender_signed_psbt = signresultandpsbt print("Sender's final signed PSBT is:\n{}".format( - wallet_s.hr_psbt(sender_signed_psbt))) + wallet_s.human_readable_psbt(sender_signed_psbt))) assert signresult.is_final # broadcast the tx @@ -367,7 +367,7 @@ def test_payjoin_workflow(setup_psbt_wallet, payment_amt, wallet_cls_sender, def test_hr_psbt(setup_psbt_wallet): bitcoin.select_chain_params("bitcoin") for k, v in hr_test_vectors.items(): - print(PSBTWalletMixin.hr_psbt( + print(PSBTWalletMixin.human_readable_psbt( bitcoin.PartiallySignedTransaction.from_binary(hextobin(v)))) bitcoin.select_chain_params("bitcoin/regtest") diff --git a/jmclient/test/test_snicker.py b/jmclient/test/test_snicker.py index 7d81e3373..7eeca551b 100644 --- a/jmclient/test/test_snicker.py +++ b/jmclient/test/test_snicker.py @@ -2,14 +2,16 @@ '''Test of SNICKER functionality using Joinmarket wallets as defined in jmclient.wallet.''' -from commontest import make_wallets, dummy_accept_callback, dummy_info_callback +import pytest +import os +from commontest import make_wallets, dummy_accept_callback, dummy_info_callback import jmbitcoin as btc -import pytest from jmbase import get_log, bintohex from jmclient import (load_test_config, estimate_tx_fee, SNICKERReceiver, direct_send) +TEST_PROPOSALS_FILE = "test_proposals.txt" log = get_log() @pytest.mark.parametrize( @@ -45,7 +47,7 @@ def test_snicker_e2e(setup_snicker, nw, wallet_structures, assert tx, "Failed to spend from receiver wallet" print("Parent transaction OK. It was: ") - print(btc.hrt(tx)) + print(btc.human_readable_transaction(tx)) wallet_r.process_new_tx(tx) # we must identify the receiver's output we're going to use; # it can be destination or change, that's up to the proposer @@ -97,10 +99,10 @@ def test_snicker_e2e(setup_snicker, nw, wallet_structures, prop_utxo['script'], change_spk, version_byte=1) + b"," + bintohex(p).encode('utf-8')) - with open("test_proposals.txt", "wb") as f: + with open(TEST_PROPOSALS_FILE, "wb") as f: f.write(b"\n".join(encrypted_proposals)) sR = SNICKERReceiver(wallet_r) - sR.proposals_source = "test_proposals.txt" # avoid clashing with mainnet + sR.proposals_source = TEST_PROPOSALS_FILE # avoid clashing with mainnet sR.poll_for_proposals() assert len(sR.successful_txs) == 1 wallet_r.process_new_tx(sR.successful_txs[0]) @@ -110,5 +112,9 @@ def test_snicker_e2e(setup_snicker, nw, wallet_structures, assert receiver_end_bal == receiver_start_bal + net_transfer @pytest.fixture(scope="module") -def setup_snicker(): +def setup_snicker(request): load_test_config() + def teardown(): + if os.path.exists(TEST_PROPOSALS_FILE): + os.remove(TEST_PROPOSALS_FILE) + request.addfinalizer(teardown) diff --git a/test/payjoinserver.py b/test/payjoinserver.py index cda5e5c0b..b18c99afe 100644 --- a/test/payjoinserver.py +++ b/test/payjoinserver.py @@ -98,7 +98,7 @@ def render_POST(self, request): version=payment_psbt.unsigned_tx.nVersion, locktime=payment_psbt.unsigned_tx.nLockTime) print("we created this unsigned tx: ") - print(btc.hrt(unsigned_payjoin_tx)) + print(btc.human_readable_transaction(unsigned_payjoin_tx)) # to create the PSBT we need the spent_outs for each input, # in the right order: spent_outs = [] @@ -126,7 +126,7 @@ def render_POST(self, request): r_payjoin_psbt = self.wallet_service.create_psbt_from_tx(unsigned_payjoin_tx, spent_outs=spent_outs) print("Receiver created payjoin PSBT:\n{}".format( - self.wallet_service.hr_psbt(r_payjoin_psbt))) + self.wallet_service.human_readable_psbt(r_payjoin_psbt))) signresultandpsbt, err = self.wallet_service.sign_psbt(r_payjoin_psbt.serialize(), with_sign_result=True) @@ -136,7 +136,7 @@ def render_POST(self, request): assert not signresult.is_final print("Receiver signing successful. Payjoin PSBT is now:\n{}".format( - self.wallet_service.hr_psbt(receiver_signed_psbt))) + self.wallet_service.human_readable_psbt(receiver_signed_psbt))) content = receiver_signed_psbt.to_base64() request.setHeader(b"content-length", ("%d" % len(content)).encode("ascii")) return content.encode("ascii") From 55295e8ea9a1f755a4879e9ee7065ba1c048f254 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Mon, 29 Jun 2020 16:52:07 +0100 Subject: [PATCH 12/15] first waypoint on bip78 --- jmclient/jmclient/payjoin.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/jmclient/jmclient/payjoin.py b/jmclient/jmclient/payjoin.py index 37bb4da4e..07bb5dc9d 100644 --- a/jmclient/jmclient/payjoin.py +++ b/jmclient/jmclient/payjoin.py @@ -17,8 +17,10 @@ """ For some documentation see: + https://github.com/bitcoin/bips/blob/master/bip-0078.mediawiki + and an earlier document: https://github.com/btcpayserver/btcpayserver-doc/blob/master/Payjoin-spec.md - which is a delta to: + and even earlier: https://github.com/bitcoin/bips/blob/master/bip-0079.mediawiki """ log = get_log() @@ -69,7 +71,7 @@ class JMPayjoinManager(object): pj_state = JM_PJ_NONE def __init__(self, wallet_service, mixdepth, destination, - amount, server): + amount, server, output_sub_allowed=True): assert isinstance(wallet_service, WalletService) # payjoin is not supported for non-segwit wallets: assert isinstance(wallet_service.wallet, @@ -86,6 +88,7 @@ def __init__(self, wallet_service, mixdepth, destination, assert amount > 0 self.amount = amount self.server = server + self.output_sub_allowed = output_sub_allowed self.pj_state = self.JM_PJ_INIT self.payment_tx = None self.initial_psbt = None @@ -364,8 +367,11 @@ def parse_payjoin_setup(bip21_uri, wallet_service, mixdepth): # this will throw for any invalid address: destaddr = btc.CCoinAddress(destaddr) server = decoded["pj"] - - return JMPayjoinManager(wallet_service, mixdepth, destaddr, amount, server) + os_allowed = True + if "pjos" in decoded and decoded["pjos"] == "0": + os_allowed = False + return JMPayjoinManager(wallet_service, mixdepth, destaddr, amount, server, + output_sub_allowed=os_allowed) def send_payjoin(manager, accept_callback=None, info_callback=None, tls_whitelist=None): From 6e6bf0a99c5e95b7fe940a3ce92923ac1eb3389e Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Tue, 7 Jul 2020 10:01:44 +0100 Subject: [PATCH 13/15] Use VERIFY_STRICTENC flag for Script verification Without this flag set, the python-bitcointx Script verification will use openssl for non-strict encoding, which requires a libopenssl dependency. Moreover non- strict encoding is now out of consensus so is not needed for our purpose. --- jmbitcoin/jmbitcoin/secp256k1_transaction.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/jmbitcoin/jmbitcoin/secp256k1_transaction.py b/jmbitcoin/jmbitcoin/secp256k1_transaction.py index 7cf9fb56e..91bb4f50b 100644 --- a/jmbitcoin/jmbitcoin/secp256k1_transaction.py +++ b/jmbitcoin/jmbitcoin/secp256k1_transaction.py @@ -16,7 +16,9 @@ from bitcointx.wallet import (P2WPKHCoinAddress, CCoinAddress, P2PKHCoinAddress, CCoinAddressError) from bitcointx.core.scripteval import (VerifyScript, SCRIPT_VERIFY_WITNESS, - SCRIPT_VERIFY_P2SH, SIGVERSION_WITNESS_V0) + SCRIPT_VERIFY_P2SH, + SCRIPT_VERIFY_STRICTENC, + SIGVERSION_WITNESS_V0) def human_readable_transaction(tx, jsonified=True): """ Given a CTransaction object, output a human @@ -203,7 +205,7 @@ def sign(tx, i, priv, hashcode=SIGHASH_ALL, amount=None, native=False): or: (None, errormsg) in case of failure """ # script verification flags - flags = set() + flags = set([SCRIPT_VERIFY_STRICTENC]) def return_err(e): return None, "Error in signing: " + repr(e) @@ -320,7 +322,7 @@ def make_shuffled_tx(ins, outs, version=1, locktime=0): def verify_tx_input(tx, i, scriptSig, scriptPubKey, amount=None, witness=None, native=False): - flags = set() + flags = set([SCRIPT_VERIFY_STRICTENC]) if witness: flags.add(SCRIPT_VERIFY_P2SH) if native: From 3ed4e889f2f323dd5f6c2bc26a45b246b915a9f9 Mon Sep 17 00:00:00 2001 From: Jules Comte Date: Wed, 8 Jul 2020 10:54:06 -0400 Subject: [PATCH 14/15] Search for correct library extension on mac os --- jmbitcoin/jmbitcoin/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/jmbitcoin/jmbitcoin/__init__.py b/jmbitcoin/jmbitcoin/__init__.py index 2a19f7f16..2ca1e1308 100644 --- a/jmbitcoin/jmbitcoin/__init__.py +++ b/jmbitcoin/jmbitcoin/__init__.py @@ -5,7 +5,11 @@ # if not, it is assumed to be present at the system level # See: https://github.com/Simplexum/python-bitcointx/commit/79333106eeb55841df2935781646369b186d99f7#diff-1ea6586127522e62d109ec5893a18850R301-R310 import os, sys -expected_secp_location = os.path.join(sys.prefix, "lib", "libsecp256k1.so") +if sys.platform == "darwin": + secp_name = "libsecp256k1.dylib" +else: + secp_name = "libsecp256k1.so" +expected_secp_location = os.path.join(sys.prefix, "lib", secp_name) if os.path.exists(expected_secp_location): import bitcointx bitcointx.set_custom_secp256k1_path(expected_secp_location) From 41540ab53e4ae86ffdd02688e689e56df3356229 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Fri, 10 Jul 2020 12:32:42 +0100 Subject: [PATCH 15/15] Modify Payjoin code for BIP78 changes. This is now tested as compatible with BIP78 as implemented by BTCPayServer. An additional config section [PAYJOIN] is added to manage settings for fee control in payjoin as described in the BIP. These settings are marked as advanced usage as they're rather complex for users to understand and the defaults should be very safe. --- jmclient/jmclient/configure.py | 29 +++++ jmclient/jmclient/payjoin.py | 195 ++++++++++++++++++++++----------- scripts/sendpayment.py | 14 +-- test/payjoinclient.py | 2 +- test/regtest_joinmarket.cfg | 29 +++++ 5 files changed, 200 insertions(+), 69 deletions(-) diff --git a/jmclient/jmclient/configure.py b/jmclient/jmclient/configure.py index c2cb92030..8a63ead83 100644 --- a/jmclient/jmclient/configure.py +++ b/jmclient/jmclient/configure.py @@ -294,6 +294,35 @@ def jm_single(): #Location of your commitments.json file (stores commitments you've used #and those you want to use in future), relative to the scripts directory. commit_file_location = cmtdata/commitments.json + +[PAYJOIN] +# for the majority of situations, the defaults +# need not be altered - they will ensure you don't pay +# a significantly higher fee. +# MODIFICATION OF THESE SETTINGS IS DISADVISED. + +# Payjoin protocol version; currently only '1' is supported. +payjoin_version = 1 + +# servers can change their destination address by default (0). +# if '1', they cannot. Note that servers can explicitly request +# that this is activated, in which case we respect that choice. +disable_output_substitution = 0 + +# "default" here indicates that we will allow the receiver to +# increase the fee we pay by: +# 1.2 * (our_fee_rate_per_vbyte * vsize_of_our_input_type) +# (see https://github.com/bitcoin/bips/blob/master/bip-0078.mediawiki#span_idfeeoutputspanFee_output) +# (and 1.2 to give breathing room) +# which indicates we are allowing roughly one extra input's fee. +# If it is instead set to an integer, then that many satoshis are allowed. +# Additionally, note that we will also set the parameter additionafeeoutputindex +# to that of our change output, unless there is none in which case this is disabled. +max_additional_fee_contribution = default + +# this is the minimum satoshis per vbyte we allow in the payjoin +# transaction; note it is decimal, not integer. +min_fee_rate = 1.1 """ #This allows use of the jmclient package with a diff --git a/jmclient/jmclient/payjoin.py b/jmclient/jmclient/payjoin.py index 07bb5dc9d..9a0ec234a 100644 --- a/jmclient/jmclient/payjoin.py +++ b/jmclient/jmclient/payjoin.py @@ -5,7 +5,8 @@ from twisted.web.iweb import IPolicyForHTTPS from twisted.internet.ssl import CertificateOptions from twisted.web.http_headers import Headers - +import urllib.parse as urlparse +from urllib.parse import urlencode import json from pprint import pformat from jmbase import BytesProducer @@ -71,7 +72,7 @@ class JMPayjoinManager(object): pj_state = JM_PJ_NONE def __init__(self, wallet_service, mixdepth, destination, - amount, server, output_sub_allowed=True): + amount, server, disable_output_substitution=False): assert isinstance(wallet_service, WalletService) # payjoin is not supported for non-segwit wallets: assert isinstance(wallet_service.wallet, @@ -88,7 +89,7 @@ def __init__(self, wallet_service, mixdepth, destination, assert amount > 0 self.amount = amount self.server = server - self.output_sub_allowed = output_sub_allowed + self.disable_output_substitution = disable_output_substitution self.pj_state = self.JM_PJ_INIT self.payment_tx = None self.initial_psbt = None @@ -97,6 +98,7 @@ def __init__(self, wallet_service, mixdepth, destination, # change is initialized as None # in case there is no change: self.change_out = None + self.change_out_index = None def set_payment_tx_and_psbt(self, in_psbt): assert isinstance(in_psbt, btc.PartiallySignedTransaction) @@ -142,7 +144,7 @@ def sanity_check_initial_payment(self): # for now: found_payment = 0 assert len(self.payment_tx.vout) in [1, 2] - for out in self.payment_tx.vout: + for i, out in enumerate(self.payment_tx.vout): if out.nValue == self.amount and \ btc.CCoinAddress.from_scriptPubKey( out.scriptPubKey) == self.destination: @@ -151,6 +153,7 @@ def sanity_check_initial_payment(self): # store this for our balance check # for receiver proposal self.change_out = out + self.change_out_index = i if not found_payment == 1: return False @@ -161,16 +164,29 @@ def check_receiver_proposal(self, in_pbst, signed_psbt_for_fees): business logic of the payjoin. We must check in detail that what the server proposes does not unfairly take money from us, and also conforms to acceptable structure. + See https://github.com/bitcoin/bips/blob/master/bip-0078.mediawiki#senders-payjoin-proposal-checklist We perform the following checks of the receiver proposal: - 1. Check that there are more inputs (i.e. some were contributed). - 2. Does it contain our inputs, unchanged? - 3. Does it contain our payment output, with amount increased? - 4. Are the other inputs finalized, and of the correct type? - 5. Is the feerate unchanged within tolerance? - 6. Does it contain no xpub information or derivation information? - 7. Are the sequence numbers unchanged (and all the same) for the inputs? - 8. Is the nLockTime and version unchanged? - 9. Is the extra fee we pay in reduced change output less than a doubling? + 1. Does it contain our inputs, unchanged? + 2. if output substitution was disabled: + check that the payment output (same scriptPubKey) has + amount equal to or greater than original tx. + if output substition is not disabled: + no check here (all of index, sPK and amount may be altered) + 3. Are the other inputs (if they exist) finalized, and of the correct type? + 4. Is the absolute fee >= fee of original tx? + 5. Check that the feerate of the transaction is not less than our minfeerate + (after signing - we have the signed version here). + 6. If we have a change output, check that: + - the change output still exists, exactly once + - amount subtracted from self.change_out is less than or equal to + maxadditionalfeecontribution. + - Check that the MAFC is only going to fee: check difference between + new fee and old fee is >= MAFC + We do not need to further check against number of new inputs, since + we already insisted on only paying for one. + 7. Does it contain no xpub information or derivation information? + 8. Are the sequence numbers unchanged (and all the same) for the inputs? + 9. Is the nLockTime and version unchanged? If all the above checks pass we will consider this valid, and cosign. Returns: @@ -181,9 +197,6 @@ def check_receiver_proposal(self, in_pbst, signed_psbt_for_fees): orig_psbt = self.initial_psbt assert isinstance(orig_psbt, btc.PartiallySignedTransaction) # 1 - if len(in_pbst.inputs) <= len(orig_psbt.inputs): - return (False, "Receiver did not contribute inputs to payjoin.") - # 2 ourins = [(i.prevout.hash, i.prevout.n) for i in orig_psbt.unsigned_tx.vin] found = [0] * len(ourins) receiver_input_indices = [] @@ -196,22 +209,24 @@ def check_receiver_proposal(self, in_pbst, signed_psbt_for_fees): if any([f != 1 for f in found]): return (False, "Receiver proposed PSBT does not contain our inputs.") + # 2 + if self.disable_output_substitution: + found_payment = 0 + for out in in_pbst.unsigned_tx.vout: + if btc.CCoinAddress.from_scriptPubKey(out.scriptPubKey) == \ + self.destination and out.nValue >= self.amount: + found_payment += 1 + if found_payment != 1: + return (False, "Our payment output not found exactly once or " + "with wrong amount.") # 3 - found = 0 - for out in in_pbst.unsigned_tx.vout: - if btc.CCoinAddress.from_scriptPubKey(out.scriptPubKey) == \ - self.destination and out.nValue >= self.amount: - found += 1 - if found != 1: - return (False, "Our payment output not found exactly once or " - "with wrong amount.") - # 4 for ind in receiver_input_indices: # check the input is finalized if not self.wallet_service.is_input_finalized(in_pbst.inputs[ind]): return (False, "receiver input is not finalized.") # check the utxo field of the input and see if the # scriptPubKey is of the right type. + # TODO this can be genericized to arbitrary wallets in future. spk = in_pbst.inputs[ind].utxo.scriptPubKey if isinstance(self.wallet_service.wallet, SegwitLegacyWallet): try: @@ -227,7 +242,7 @@ def check_receiver_proposal(self, in_pbst, signed_psbt_for_fees): "Receiver input type does not match ours.") else: assert False - # 5 + # 4, 5 # To get the feerate of the psbt proposed, we use the already-signed # version (so all witnesses filled in) to calculate its size, # then compare that with the fee, and do the same for the @@ -237,49 +252,52 @@ def check_receiver_proposal(self, in_pbst, signed_psbt_for_fees): except ValueError: return (False, "receiver proposed tx has negative fee.") nonpayjoin_tx_fee = self.initial_psbt.get_fee() + if proposed_tx_fee < nonpayjoin_tx_fee: + return (False, "receiver proposed transaction has lower fee.") proposed_tx_size = signed_psbt_for_fees.extract_transaction( ).get_virtual_size() - nonpayjoin_tx_size = self.initial_psbt.extract_transaction( - ).get_virtual_size() proposed_fee_rate = proposed_tx_fee / float(proposed_tx_size) log.debug("proposed fee rate: " + str(proposed_fee_rate)) - nonpayjoin_fee_rate = nonpayjoin_tx_fee / float(nonpayjoin_tx_size) - log.debug("nonpayjoin fee rate: " + str(nonpayjoin_fee_rate)) - diff_rate = abs(proposed_fee_rate - nonpayjoin_fee_rate)/nonpayjoin_fee_rate - if diff_rate > 0.2: - log.error("Bad fee rate differential: " + str(diff_rate)) - return (False, "fee rate of payjoin tx is more than 20% different " - "from inital fee rate, rejecting.") + if proposed_fee_rate < float( + jm_single().config.get("PAYJOIN", "min_fee_rate")): + return (False, "receiver proposed transaction has too low " + "feerate: " + str(proposed_fee_rate)) # 6 + if self.change_out: + found_change = 0 + for out in in_pbst.unsigned_tx.vout: + if out.scriptPubKey == self.change_out.scriptPubKey: + found_change += 1 + actual_contribution = self.change_out.nValue - out.nValue + if actual_contribution > in_pbst.get_fee( + ) - self.initial_psbt.get_fee(): + return (False, "Our change output is reduced more" + " than the fee is bumped.") + mafc = get_max_additional_fee_contribution(self) + if actual_contribution > mafc: + return (False, "Proposed transactions requires " + "us to pay more additional fee that we " + "agreed to: " + str(mafc) + " sats.") + # note this check is only if the initial tx had change: + if found_change != 1: + return (False, "Our change output was not found " + "exactly once.") + # 7 if in_pbst.xpubs: return (False, "Receiver proposal contains xpub information.") - # 7 - # we created all inputs with one sequence number, make sure everything - # agrees - # TODO - discussion with btcpayserver devs, docs will be updated, - # server will agree with client in future. For now disabling check - # (it's a very complicated issue, surprisingly!) - #seqno = self.initial_psbt.unsigned_tx.vin[0].nSequence - #for inp in in_pbst.unsigned_tx.vin: - # if inp.nSequence != seqno: - # return (False, "all sequence numbers are not the same.") # 8 + seqno = self.initial_psbt.unsigned_tx.vin[0].nSequence + for inp in in_pbst.unsigned_tx.vin: + if inp.nSequence != seqno: + return (False, "all sequence numbers are not the same.") + # 9 if in_pbst.unsigned_tx.nLockTime != \ self.initial_psbt.unsigned_tx.nLockTime: return (False, "receiver proposal has altered nLockTime.") if in_pbst.unsigned_tx.nVersion != \ self.initial_psbt.unsigned_tx.nVersion: return (False, "receiver proposal has altered nVersion.") - # 9 - if proposed_tx_fee >= 2 * nonpayjoin_tx_fee: - return (False, "receiver's tx fee is too large (possibly " - "too many extra inputs.") - # as well as the overall fee, check our pay-out specifically: - for out in in_pbst.unsigned_tx.vout: - if out.scriptPubKey == self.change_out.scriptPubKey: - found += 1 - if self.change_out.nValue - out.nValue > nonpayjoin_tx_fee: - return (False, "Our change output was reduced too much.") + # all checks passed return (True, None) def set_payjoin_psbt(self, in_psbt, signed_psbt_for_fees): @@ -367,11 +385,32 @@ def parse_payjoin_setup(bip21_uri, wallet_service, mixdepth): # this will throw for any invalid address: destaddr = btc.CCoinAddress(destaddr) server = decoded["pj"] - os_allowed = True + disable_output_substitution = False if "pjos" in decoded and decoded["pjos"] == "0": - os_allowed = False + disable_output_substitution = True return JMPayjoinManager(wallet_service, mixdepth, destaddr, amount, server, - output_sub_allowed=os_allowed) + disable_output_substitution=disable_output_substitution) + +def get_max_additional_fee_contribution(manager): + """ See definition of maxadditionalfeecontribution in BIP 78. + """ + max_additional_fee_contribution = jm_single( + ).config.get("PAYJOIN", "max_additional_fee_contribution") + if max_additional_fee_contribution == "default": + # calculate the fee bumping allowed according to policy: + if isinstance(manager.wallet_service.wallet, SegwitLegacyWallet): + vsize = 91 + elif isinstance(manager.wallet_service.wallet, SegwitWallet): + vsize = 68 + else: + raise Exception("Payjoin only supported for segwit wallets") + original_fee_rate = manager.initial_psbt.get_fee()/float( + manager.initial_psbt.extract_transaction().get_virtual_size()) + log.debug("Initial nonpayjoin transaction feerate is: " + str(original_fee_rate)) + max_additional_fee_contribution = int(original_fee_rate * 1.2 * vsize) + log.debug("From which we calculated a max additional fee " + "contribution of: " + str(max_additional_fee_contribution)) + return max_additional_fee_contribution def send_payjoin(manager, accept_callback=None, info_callback=None, tls_whitelist=None): @@ -416,13 +455,47 @@ def send_payjoin(manager, accept_callback=None, contextFactory=WhitelistContextFactory(tls_whitelist)) body = BytesProducer(payment_psbt.to_base64().encode("utf-8")) + + #Set the query parameters for the request: + + # construct the URI from the given parameters + pj_version = jm_single().config.getint("PAYJOIN", + "payjoin_version") + params = {"v": pj_version} + + disable_output_substitution = "false" + if manager.disable_output_substitution: + disable_output_substitution = "true" + else: + if jm_single().config.getint("PAYJOIN", + "disable_output_substitution") == 1: + disable_output_substitution = "true" + params["disableoutputsubstitution"] = disable_output_substitution + + # to determine the additionalfeeoutputindex in cases where we have + # change and we are allowing fee bump, we examine the initial tx: + if manager.change_out: + params["additionalfeeoutputindex"] = manager.change_out_index + params["maxadditionalfeecontribution"] = \ + get_max_additional_fee_contribution(manager) + + min_fee_rate = float(jm_single().config.get("PAYJOIN", "min_fee_rate")) + params["minfeerate"] = min_fee_rate + + destination_url = manager.server.encode("utf-8") + url_parts = list(urlparse.urlparse(destination_url)) + print("From destination url: ", destination_url, " got urlparts: ", url_parts) + url_parts[4] = urlencode(params).encode("utf-8") + print("after insertion, url_parts is: ", url_parts) + destination_url = urlparse.urlunparse(url_parts) # TODO what to use as user agent? - d = agent.request(b"POST", manager.server.encode("utf-8"), - Headers({"Content-Type": ['text/plain']}), + d = agent.request(b"POST", destination_url, + Headers({"User-Agent": ["Twisted Web Client Example"], + "Content-Type": ["text/plain"]}), bodyProducer=body) d.addCallback(receive_payjoin_proposal_from_server, manager) - # note that the errback (here "noresponse") is *not* triggered + # note that the errback (here "noResponse") is *not* triggered # by a server rejection (which is accompanied by a non-200 # status code returned), but by failure to communicate. def noResponse(failure): diff --git a/scripts/sendpayment.py b/scripts/sendpayment.py index 79dd5217f..19b580865 100755 --- a/scripts/sendpayment.py +++ b/scripts/sendpayment.py @@ -66,7 +66,7 @@ def main(): #without schedule file option, use the arguments to create a schedule #of a single transaction sweeping = False - payjoinurl = None + bip78url = None if options.schedule == '': if btc.is_bip21_uri(args[1]): parsed = btc.decode_bip21_uri(args[1]) @@ -78,19 +78,19 @@ def main(): destaddr = parsed['address'] if 'jmnick' in parsed: if "pj" in parsed: - parser.error("Cannot specify both BIP79++ and Joinmarket " + parser.error("Cannot specify both BIP78 and Joinmarket " "peer-to-peer payjoin at the same time!") sys.exit(EXIT_ARGERROR) options.p2ep = parsed['jmnick'] elif "pj" in parsed: # note that this is a URL; its validity # checking is deferred to twisted.web.client.Agent - payjoinurl = parsed["pj"] + bip78url = parsed["pj"] # setting makercount only for fee sanity check. # note we ignore any user setting and enforce N=0, # as this is a flag in the code for a non-JM coinjoin; - # for the fee sanity check, note that BIP79++ currently - # will only allow very small fee changes, so N=0 won't + # for the fee sanity check, note that BIP78 currently + # will only allow small fee changes, so N=0 won't # be very inaccurate. jmprint("Attempting to pay via payjoin.", "info") options.makercount = 0 @@ -211,7 +211,7 @@ def main(): log.info("Estimated miner/tx fees for this coinjoin amount: {:.1%}" .format(exp_tx_fees_ratio)) - if options.makercount == 0 and not options.p2ep and not payjoinurl: + if options.makercount == 0 and not options.p2ep and not bip78url: tx = direct_send(wallet_service, amount, mixdepth, destaddr, options.answeryes, with_final_psbt=options.with_psbt) if options.with_psbt: @@ -322,7 +322,7 @@ def p2ep_on_finished_callback(res, fromtx=False, waittime=0.0, taker = P2EPTaker(options.p2ep, wallet_service, schedule, callbacks=(None, None, p2ep_on_finished_callback)) - elif payjoinurl: + elif bip78url: # TODO sanity check wallet type is segwit manager = parse_payjoin_setup(args[1], wallet_service, options.mixdepth) reactor.callWhenRunning(send_payjoin, manager, tls_whitelist=["127.0.0.1"]) diff --git a/test/payjoinclient.py b/test/payjoinclient.py index 764b8b601..729698507 100644 --- a/test/payjoinclient.py +++ b/test/payjoinclient.py @@ -22,7 +22,7 @@ pjurl = "http://127.0.0.1:8080" else: pjurl = "https://127.0.0.1:8080" - bip21uri = "bitcoin:2N7CAdEUjJW9tUHiPhDkmL9ukPtcukJMoxK?amount=0.3&pj=" + pjurl + bip21uri = "bitcoin:2N7CAdEUjJW9tUHiPhDkmL9ukPtcukJMoxK?amount=0.3&pj=" + pjurl wallet_path = get_wallet_path(wallet_name, None) if jm_single().config.get("POLICY", "native") == "true": walletclass = SegwitWallet diff --git a/test/regtest_joinmarket.cfg b/test/regtest_joinmarket.cfg index 4d2233164..785ea2cc9 100644 --- a/test/regtest_joinmarket.cfg +++ b/test/regtest_joinmarket.cfg @@ -62,3 +62,32 @@ minimum_makers = 1 listunspent_args = [0] max_sats_freeze_reuse = -1 +[PAYJOIN] +# for the majority of situations, the defaults +# need not be altered - they will ensure you don't pay +# a significantly higher fee. +# MODIFICATION OF THESE SETTINGS IS DISADVISED. + +# Payjoin protocol version; currently only '1' is supported. +payjoin_version = 1 + +# servers can change their destination address by default (0). +# if '1', they cannot. Note that servers can explicitly request +# that this is activated, in which case we respect that choice. +disable_output_substitution = 0 + +# "default" here indicates that we will allow the receiver to +# increase the fee we pay by: +# 1.2 * (our_fee_rate_per_vbyte * vsize_of_our_input_type) +# (see https://github.com/bitcoin/bips/blob/master/bip-0078.mediawiki#span_idfeeoutputspanFee_output) +# (and 1.2 to give breathing room) +# which indicates we are allowing roughly one extra input's fee. +# If it is instead set to an integer, then that many satoshis are allowed. +# Additionally, note that we will also set the parameter additionafeeoutputindex +# to that of our change output, unless there is none in which case this is disabled. +max_additional_fee_contribution = default + +# this is the minimum satoshis per vbyte we allow in the payjoin +# transaction; note it is decimal, not integer. +min_fee_rate = 1.1 +