From dc8cfd06aa7b0441ba5c903d814c28e5fde2ed34 Mon Sep 17 00:00:00 2001 From: "Glenn H Tarbox, PhD" Date: Mon, 7 Dec 2015 18:09:43 -0800 Subject: [PATCH] Create joinmarket module. Rename lib/common.py joinmarket/__init__.py --- create-unsigned-tx.py | 83 ++-- {joinmarket => external}/bitcoin/__init__.py | 0 {joinmarket => external}/bitcoin/bci.py | 0 {joinmarket => external}/bitcoin/blocks.py | 0 {joinmarket => external}/bitcoin/composite.py | 0 .../bitcoin/deterministic.py | 0 {joinmarket => external}/bitcoin/main.py | 0 .../bitcoin/py2specials.py | 0 .../bitcoin/py3specials.py | 0 {joinmarket => external}/bitcoin/ripemd.py | 0 {joinmarket => external}/bitcoin/stealth.py | 0 .../bitcoin/transaction.py | 0 {joinmarket => external}/libnacl/__init__.py | 0 {joinmarket => external}/libnacl/base.py | 0 {joinmarket => external}/libnacl/blake.py | 0 {joinmarket => external}/libnacl/dual.py | 0 {joinmarket => external}/libnacl/encode.py | 0 {joinmarket => external}/libnacl/public.py | 0 {joinmarket => external}/libnacl/secret.py | 0 {joinmarket => external}/libnacl/sign.py | 0 {joinmarket => external}/libnacl/utils.py | 0 {joinmarket => external}/libnacl/version.py | 0 joinmarket/{common.py => __init__.py} | 356 +++++++++------ joinmarket/blockchaininterface.py | 289 ++++++------ joinmarket/enc_wrapper.py | 78 ++-- joinmarket/irc.py | 202 +++++---- joinmarket/jsonrpc.py | 3 +- joinmarket/maker.py | 189 ++++---- joinmarket/message_channel.py | 20 +- joinmarket/old_mnemonic.py | 5 +- joinmarket/slowaes.py | 74 ++-- joinmarket/socks.py | 46 +- joinmarket/taker.py | 303 +++++++------ ob-watcher.py | 146 ++++--- patientsendpayment.py | 152 +++---- sendpayment.py | 151 ++++--- tumbler.py | 413 +++++++++--------- wallet-tool.py | 98 +++-- yield-generator.py | 109 ++--- 39 files changed, 1455 insertions(+), 1262 deletions(-) rename {joinmarket => external}/bitcoin/__init__.py (100%) rename {joinmarket => external}/bitcoin/bci.py (100%) rename {joinmarket => external}/bitcoin/blocks.py (100%) rename {joinmarket => external}/bitcoin/composite.py (100%) rename {joinmarket => external}/bitcoin/deterministic.py (100%) rename {joinmarket => external}/bitcoin/main.py (100%) rename {joinmarket => external}/bitcoin/py2specials.py (100%) rename {joinmarket => external}/bitcoin/py3specials.py (100%) rename {joinmarket => external}/bitcoin/ripemd.py (100%) rename {joinmarket => external}/bitcoin/stealth.py (100%) rename {joinmarket => external}/bitcoin/transaction.py (100%) rename {joinmarket => external}/libnacl/__init__.py (100%) rename {joinmarket => external}/libnacl/base.py (100%) rename {joinmarket => external}/libnacl/blake.py (100%) rename {joinmarket => external}/libnacl/dual.py (100%) rename {joinmarket => external}/libnacl/encode.py (100%) rename {joinmarket => external}/libnacl/public.py (100%) rename {joinmarket => external}/libnacl/secret.py (100%) rename {joinmarket => external}/libnacl/sign.py (100%) rename {joinmarket => external}/libnacl/utils.py (100%) rename {joinmarket => external}/libnacl/version.py (100%) rename joinmarket/{common.py => __init__.py} (76%) diff --git a/create-unsigned-tx.py b/create-unsigned-tx.py index 2f961fd1..37c63c68 100644 --- a/create-unsigned-tx.py +++ b/create-unsigned-tx.py @@ -1,19 +1,26 @@ #! /usr/bin/env python +from __future__ import absolute_import -import os import sys +import threading +import time from optparse import OptionParser -data_dir = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(data_dir, 'joinmarket')) +# data_dir = os.path.dirname(os.path.realpath(__file__)) +# sys.path.insert(0, os.path.join(data_dir, 'joinmarket')) -from common import * -import common -import taker as takermodule -from irc import IRCMessageChannel, random_nick +# from common import * +# import common +from joinmarket import taker as takermodule, choose_sweep_orders, get_log, \ + choose_orders, load_program_config, validate_address, bc_interface, \ + get_p2pk_vbyte, pick_order, cheapest_order_choose, weighted_order_choose, \ + set_nickname, AbstractWallet, debug_dump_object +from joinmarket import IRCMessageChannel, random_nick import bitcoin as btc import sendpayment +log = get_log() + # thread which does the buy-side algorithm # chooses which coinjoins to initiate and when @@ -47,9 +54,9 @@ def create_tx(self): self.ignored_makers) if not self.taker.options.answeryes: total_cj_fee = total_value - cjamount - self.taker.options.txfee - debug('total cj fee = ' + str(total_cj_fee)) + log.debug('total cj fee = ' + str(total_cj_fee)) total_fee_pc = 1.0 * total_cj_fee / cjamount - debug('total coinjoin fee = ' + str(float('%.3g' % ( + log.debug('total coinjoin fee = ' + str(float('%.3g' % ( 100.0 * total_fee_pc))) + '%') sendpayment.check_high_fee(total_fee_pc) if raw_input('send with these orders? (y/n):')[0] != 'y': @@ -60,7 +67,8 @@ def create_tx(self): orders, total_cj_fee = self.sendpayment_choose_orders( self.taker.cjamount, self.taker.options.makercount) if not orders: - debug('ERROR not enough liquidity in the orderbook, exiting') + log.debug( + 'ERROR not enough liquidity in the orderbook, exiting') return total_amount = self.taker.cjamount + total_cj_fee + self.taker.options.txfee print 'total amount spent = ' + str(total_amount) @@ -87,11 +95,12 @@ def finishcallback(self, coinjointx): tx = btc.sign(tx, index, coinjointx.wallet.get_key_from_addr(addr)) print 'unsigned tx = \n\n' + tx + '\n' - debug('created unsigned tx, ending') + log.debug('created unsigned tx, ending') self.taker.msgchan.shutdown() return self.ignored_makers += coinjointx.nonrespondants - debug('recreating the tx, ignored_makers=' + str(self.ignored_makers)) + log.debug( + 'recreating the tx, ignored_makers=' + str(self.ignored_makers)) self.create_tx() def sendpayment_choose_orders(self, @@ -118,11 +127,11 @@ def sendpayment_choose_orders(self, else: noun = 'additional' total_fee_pc = 1.0 * total_cj_fee / cj_amount - debug(noun + ' coinjoin fee = ' + str(float('%.3g' % ( + log.debug(noun + ' coinjoin fee = ' + str(float('%.3g' % ( 100.0 * total_fee_pc))) + '%') sendpayment.check_high_fee(total_fee_pc) if raw_input('send with these orders? (y/n):')[0] != 'y': - debug('ending') + log.debug('ending') self.taker.msgchan.shutdown() return None, -1 return orders, total_cj_fee @@ -153,16 +162,17 @@ def on_welcome(self): def main(): parser = OptionParser( - usage= - 'usage: %prog [options] [auth utxo] [cjamount] [cjaddr] [changeaddr] [utxos..]', - description= - 'Creates an unsigned coinjoin transaction. Outputs a partially signed transaction ' - + - 'hex string. The user must sign their inputs independently and broadcast them. The JoinMarket' - + - ' protocol requires the taker to have a single p2pk UTXO input to use to authenticate the ' - + - ' encrypted messages. For this reason you must pass auth utxo and the corresponding private key') + usage='usage: %prog [options] [auth utxo] [cjamount] [cjaddr] [' + 'changeaddr] [utxos..]', + description=('Creates an unsigned coinjoin transaction. Outputs ' + 'a partially signed transaction hex string. The user ' + 'must sign their inputs independently and broadcast ' + 'them. The JoinMarket protocol requires the taker to ' + 'have a single p2pk UTXO input to use to ' + 'authenticate the encrypted messages. For this ' + 'reason you must pass auth utxo and the ' + 'corresponding private key')) + # for cjamount=0 do a sweep, and ignore change address parser.add_option('-f', '--txfee', @@ -221,7 +231,7 @@ def main(): changeaddr = args[3] cold_utxos = args[4:] - common.load_program_config() + load_program_config() addr_valid1, errormsg1 = validate_address(destaddr) errormsg2 = None # if amount = 0 dont bother checking changeaddr so user can write any junk @@ -237,7 +247,7 @@ def main(): return all_utxos = [auth_utxo] + cold_utxos - query_result = common.bc_interface.query_utxo_set(all_utxos) + query_result = bc_interface.query_utxo_set(all_utxos) if None in query_result: print query_result utxo_data = {} @@ -246,7 +256,7 @@ def main(): auth_privkey = raw_input('input private key for ' + utxo_data[auth_utxo][ 'address'] + ' :') if utxo_data[auth_utxo]['address'] != btc.privtoaddr( - auth_privkey, common.get_p2pk_vbyte()): + auth_privkey, get_p2pk_vbyte()): print 'ERROR: privkey does not match auth utxo' return @@ -259,30 +269,31 @@ def main(): else: # choose randomly (weighted) chooseOrdersFunc = weighted_order_choose - common.nickname = random_nick() - debug('starting sendpayment') + nickname = random_nick() + set_nickname(nickname) + log.debug('starting sendpayment') - class UnsignedTXWallet(common.AbstractWallet): + class UnsignedTXWallet(AbstractWallet): def get_key_from_addr(self, addr): - debug('getting privkey of ' + addr) - if btc.privtoaddr(auth_privkey, common.get_p2pk_vbyte()) != addr: + log.debug('getting privkey of ' + addr) + if btc.privtoaddr(auth_privkey, get_p2pk_vbyte()) != addr: raise RuntimeError('privkey doesnt match given address') return auth_privkey wallet = UnsignedTXWallet() - irc = IRCMessageChannel(common.nickname) + irc = IRCMessageChannel(nickname) taker = CreateUnsignedTx(irc, wallet, auth_utxo, cjamount, destaddr, changeaddr, utxo_data, options, chooseOrdersFunc) try: - debug('starting irc') + log.debug('starting irc') irc.run() except: - debug('CRASHING, DUMPING EVERYTHING') + log.debug('CRASHING, DUMPING EVERYTHING') debug_dump_object(wallet, ['addr_cache', 'keys', 'wallet_name', 'seed']) debug_dump_object(taker) import traceback - debug(traceback.format_exc()) + log.debug(traceback.format_exc()) if __name__ == "__main__": diff --git a/joinmarket/bitcoin/__init__.py b/external/bitcoin/__init__.py similarity index 100% rename from joinmarket/bitcoin/__init__.py rename to external/bitcoin/__init__.py diff --git a/joinmarket/bitcoin/bci.py b/external/bitcoin/bci.py similarity index 100% rename from joinmarket/bitcoin/bci.py rename to external/bitcoin/bci.py diff --git a/joinmarket/bitcoin/blocks.py b/external/bitcoin/blocks.py similarity index 100% rename from joinmarket/bitcoin/blocks.py rename to external/bitcoin/blocks.py diff --git a/joinmarket/bitcoin/composite.py b/external/bitcoin/composite.py similarity index 100% rename from joinmarket/bitcoin/composite.py rename to external/bitcoin/composite.py diff --git a/joinmarket/bitcoin/deterministic.py b/external/bitcoin/deterministic.py similarity index 100% rename from joinmarket/bitcoin/deterministic.py rename to external/bitcoin/deterministic.py diff --git a/joinmarket/bitcoin/main.py b/external/bitcoin/main.py similarity index 100% rename from joinmarket/bitcoin/main.py rename to external/bitcoin/main.py diff --git a/joinmarket/bitcoin/py2specials.py b/external/bitcoin/py2specials.py similarity index 100% rename from joinmarket/bitcoin/py2specials.py rename to external/bitcoin/py2specials.py diff --git a/joinmarket/bitcoin/py3specials.py b/external/bitcoin/py3specials.py similarity index 100% rename from joinmarket/bitcoin/py3specials.py rename to external/bitcoin/py3specials.py diff --git a/joinmarket/bitcoin/ripemd.py b/external/bitcoin/ripemd.py similarity index 100% rename from joinmarket/bitcoin/ripemd.py rename to external/bitcoin/ripemd.py diff --git a/joinmarket/bitcoin/stealth.py b/external/bitcoin/stealth.py similarity index 100% rename from joinmarket/bitcoin/stealth.py rename to external/bitcoin/stealth.py diff --git a/joinmarket/bitcoin/transaction.py b/external/bitcoin/transaction.py similarity index 100% rename from joinmarket/bitcoin/transaction.py rename to external/bitcoin/transaction.py diff --git a/joinmarket/libnacl/__init__.py b/external/libnacl/__init__.py similarity index 100% rename from joinmarket/libnacl/__init__.py rename to external/libnacl/__init__.py diff --git a/joinmarket/libnacl/base.py b/external/libnacl/base.py similarity index 100% rename from joinmarket/libnacl/base.py rename to external/libnacl/base.py diff --git a/joinmarket/libnacl/blake.py b/external/libnacl/blake.py similarity index 100% rename from joinmarket/libnacl/blake.py rename to external/libnacl/blake.py diff --git a/joinmarket/libnacl/dual.py b/external/libnacl/dual.py similarity index 100% rename from joinmarket/libnacl/dual.py rename to external/libnacl/dual.py diff --git a/joinmarket/libnacl/encode.py b/external/libnacl/encode.py similarity index 100% rename from joinmarket/libnacl/encode.py rename to external/libnacl/encode.py diff --git a/joinmarket/libnacl/public.py b/external/libnacl/public.py similarity index 100% rename from joinmarket/libnacl/public.py rename to external/libnacl/public.py diff --git a/joinmarket/libnacl/secret.py b/external/libnacl/secret.py similarity index 100% rename from joinmarket/libnacl/secret.py rename to external/libnacl/secret.py diff --git a/joinmarket/libnacl/sign.py b/external/libnacl/sign.py similarity index 100% rename from joinmarket/libnacl/sign.py rename to external/libnacl/sign.py diff --git a/joinmarket/libnacl/utils.py b/external/libnacl/utils.py similarity index 100% rename from joinmarket/libnacl/utils.py rename to external/libnacl/utils.py diff --git a/joinmarket/libnacl/version.py b/external/libnacl/version.py similarity index 100% rename from joinmarket/libnacl/version.py rename to external/libnacl/version.py diff --git a/joinmarket/common.py b/joinmarket/__init__.py similarity index 76% rename from joinmarket/common.py rename to joinmarket/__init__.py index 67c75fbb..b805b681 100644 --- a/joinmarket/common.py +++ b/joinmarket/__init__.py @@ -1,11 +1,32 @@ -import bitcoin as btc -from decimal import Decimal, InvalidOperation -from math import factorial, exp -import sys, datetime, json, time, pprint, threading, getpass -import random -import blockchaininterface, jsonrpc, slowaes -from ConfigParser import SafeConfigParser, NoSectionError, NoOptionError -import os, io, itertools +from __future__ import absolute_import + +import datetime +import io +import logging +import threading +from getpass import getpass +from math import exp + +from ConfigParser import SafeConfigParser, NoSectionError +from configparser import NoOptionError + +# Set default logging handler to avoid "No handler found" warnings. +try: + from logging import NullHandler +except ImportError: + class NullHandler(logging.Handler): + def emit(self, record): + pass + +logging.getLogger(__name__).addHandler(NullHandler()) + +log = logging.getLogger('gitreformat') +log.addHandler(logging.NullHandler()) + + +def get_log(): + return log + JM_VERSION = 2 nickname = None @@ -22,75 +43,58 @@ config = SafeConfigParser() config_location = 'joinmarket.cfg' -# FIXME: Add rpc_* options here in the future! -required_options = {'BLOCKCHAIN': ['blockchain_source', 'network'], - 'MESSAGING': ['host', 'channel', 'port']} -defaultconfig =\ -""" -[BLOCKCHAIN] -blockchain_source = blockr -#options: blockr, bitcoin-rpc, json-rpc, regtest -#for instructions on bitcoin-rpc read https://github.com/chris-belcher/joinmarket/wiki/Running-JoinMarket-with-Bitcoin-Core-full-node -network = mainnet -rpc_host = localhost -rpc_port = 8332 -rpc_user = bitcoin -rpc_password = password - -[MESSAGING] -host = irc.cyberguerrilla.org -channel = joinmarket-pit -port = 6697 -usessl = true -socks5 = false -socks5_host = localhost -socks5_port = 9050 -#for tor -#host = 6dvj6v5imhny3anf.onion -#port = 6697 -#usessl = true -#socks5 = true -maker_timeout_sec = 30 -[POLICY] -# for dust sweeping, try merge_algorithm = gradual -# for more rapid dust sweeping, try merge_algorithm = greedy -# for most rapid dust sweeping, try merge_algorithm = greediest -# but don't forget to bump your miner fees! -merge_algorithm = default -""" +# todo: we need a class or something to manage this global state +def set_nickname(nick): + global nickname + nickname = nick -def load_program_config(): - loadedFiles = config.read([config_location]) - #Create default config file if not found - if len(loadedFiles) != 1: - config.readfp(io.BytesIO(defaultconfig)) - with open(config_location, "w") as configfile: - configfile.write(defaultconfig) - #check for sections - for s in required_options: - if s not in config.sections(): - raise Exception( - "Config file does not contain the required section: " + s) - #then check for specific options - for k, v in required_options.iteritems(): - for o in v: - if o not in config.options(k): - raise Exception( - "Config file does not contain the required option: " + o) +def set_debug_silence(state): + global debug_silence + debug_silence = state - try: - global maker_timeout_sec - maker_timeout_sec = config.getint('MESSAGING', 'maker_timeout_sec') - except NoOptionError: - debug('maker_timeout_sec not found in .cfg file, using default value') - #configure the interface to the blockchain on startup - global bc_interface - bc_interface = blockchaininterface.get_blockchain_interface_instance(config) +# FIXME: Add rpc_* options here in the future! +required_options = {'BLOCKCHAIN': ['blockchain_source', 'network'], + 'MESSAGING': ['host', 'channel', 'port']} + +defaultconfig = \ + """ + [BLOCKCHAIN] + blockchain_source = blockr + #options: blockr, bitcoin-rpc, json-rpc, regtest + #for instructions on bitcoin-rpc read https://github.com/chris-belcher/joinmarket/wiki/Running-JoinMarket-with-Bitcoin-Core-full-node + network = mainnet + rpc_host = localhost + rpc_port = 8332 + rpc_user = bitcoin + rpc_password = password + + [MESSAGING] + host = irc.cyberguerrilla.org + channel = joinmarket-pit + port = 6697 + usessl = true + socks5 = false + socks5_host = localhost + socks5_port = 9050 + #for tor + #host = 6dvj6v5imhny3anf.onion + #port = 6697 + #usessl = true + #socks5 = true + maker_timeout_sec = 30 + + [POLICY] + # for dust sweeping, try merge_algorithm = gradual + # for more rapid dust sweeping, try merge_algorithm = greedy + # for most rapid dust sweeping, try merge_algorithm = greediest + # but don't forget to bump your miner fees! + merge_algorithm = default + """ def get_config_irc_channel(): @@ -105,7 +109,7 @@ def debug(msg): with debug_file_lock: if nickname and not debug_file_handle: debug_file_handle = open( - os.path.join('logs', nickname + '.log'), 'ab', 1) + os.path.join('logs', nickname + '.log'), 'ab', 1) outmsg = datetime.datetime.now().strftime("[%Y/%m/%d %H:%M:%S] ") + msg if not debug_silence: if core_alert: @@ -113,28 +117,30 @@ def debug(msg): if joinmarket_alert: print 'JoinMarket Alert Message: ' + joinmarket_alert print outmsg - if nickname: #debugs before creating bot nick won't be handled like this + if nickname: # debugs before creating bot nick won't be handled like this debug_file_handle.write(outmsg + '\r\n') - #Random functions - replacing some NumPy features - #NOTE THESE ARE NEITHER CRYPTOGRAPHICALLY SECURE - #NOR PERFORMANT NOR HIGH PRECISION! - #Only for sampling purposes + # Random functions - replacing some NumPy features + # NOTE THESE ARE NEITHER CRYPTOGRAPHICALLY SECURE + # NOR PERFORMANT NOR HIGH PRECISION! + # Only for sampling purposes + + def rand_norm_array(mu, sigma, n): - #use normalvariate instead of gauss for thread safety + # use normalvariate instead of gauss for thread safety return [random.normalvariate(mu, sigma) for i in range(n)] def rand_exp_array(lamda, n): - #'lambda' is reserved (in case you are triggered by spelling errors) + # 'lambda' is reserved (in case you are triggered by spelling errors) return [random.expovariate(1.0 / lamda) for i in range(n)] def rand_pow_array(power, n): - #rather crude in that uses a uniform sample which is a multiple of 1e-4 - #for basis of formula, see: http://mathworld.wolfram.com/RandomNumber.html - return [y**(1.0 / power) + # rather crude in that uses a uniform sample which is a multiple of 1e-4 + # for basis of formula, see: http://mathworld.wolfram.com/RandomNumber.html + return [y ** (1.0 / power) for y in [x * 0.0001 for x in random.sample( xrange(10000), n)]] @@ -152,7 +158,9 @@ def rand_weighted_choice(n, p_arr): cum_pr = [sum(p_arr[:i + 1]) for i in xrange(len(p_arr))] r = random.random() return sorted(cum_pr + [r]).index(r) -#End random functions + + +# End random functions def chunks(d, n): @@ -188,7 +196,9 @@ def validate_address(addr): return True, 'address validated' -def debug_dump_object(obj, skip_fields=[]): +def debug_dump_object(obj, skip_fields=None): + if skip_fields is None: + skip_fields = [] debug('Class debug dump, name:' + obj.__class__.__name__) for k, v in obj.__dict__.iteritems(): if k in skip_fields: @@ -288,6 +298,7 @@ class AbstractWallet(object): ''' def __init__(self): + self.max_mix_depth = 0 self.utxo_selector = btc.select # default fallback: upstream try: if config.get("POLICY", "merge_algorithm") == "gradual": @@ -343,7 +354,6 @@ def get_balance_by_mixdepth(self): class Wallet(AbstractWallet): - def __init__(self, seedarg, max_mix_depth=2, @@ -353,8 +363,8 @@ def __init__(self, super(Wallet, self).__init__() self.max_mix_depth = max_mix_depth self.storepassword = storepassword - #key is address, value is (mixdepth, forchange, index) - #if mixdepth = -1 it's an imported key and index refers to imported_privkeys + # key is address, value is (mixdepth, forchange, index) + # if mixdepth = -1 it's an imported key and index refers to imported_privkeys self.addr_cache = {} self.unspent = {} self.spent_utxos = [] @@ -370,7 +380,7 @@ def __init__(self, self.keys = [(btc.bip32_ckd(m, 0), btc.bip32_ckd(m, 1)) for m in mixing_depth_keys] - #self.index = [[0, 0]]*max_mix_depth + # self.index = [[0, 0]]*max_mix_depth self.index = [] for i in range(self.max_mix_depth): self.index.append([0, 0]) @@ -382,7 +392,7 @@ def read_wallet_file_data(self, filename): if not os.path.isfile(path): if get_network() == 'testnet': debug( - 'filename interpreted as seed, only available in testnet because this probably has lower entropy') + 'filename interpreted as seed, only available in testnet because this probably has lower entropy') return filename else: raise IOError('wallet file not found') @@ -399,14 +409,15 @@ def read_wallet_file_data(self, filename): self.index_cache = walletdata['index_cache'] decrypted = False while not decrypted: - password = getpass.getpass('Enter wallet decryption passphrase: ') + password = getpass('Enter wallet decryption passphrase: ') password_key = btc.bin_dbl_sha256(password) encrypted_seed = walletdata['encrypted_seed'] try: - decrypted_seed = slowaes.decryptData( - password_key, encrypted_seed.decode('hex')).encode('hex') - #there is a small probability of getting a valid PKCS7 padding - #by chance from a wrong password; sanity check the seed length + decrypted_seed = decryptData( + password_key, encrypted_seed.decode('hex')).encode( + 'hex') + # there is a small probability of getting a valid PKCS7 padding + # by chance from a wrong password; sanity check the seed length if len(decrypted_seed) == 32: decrypted = True else: @@ -419,9 +430,9 @@ def read_wallet_file_data(self, filename): self.walletdata = walletdata if 'imported_keys' in walletdata: for epk_m in walletdata['imported_keys']: - privkey = slowaes.decryptData(password_key, - epk_m['encrypted_privkey'] - .decode('hex')).encode('hex') + privkey = decryptData(password_key, + epk_m['encrypted_privkey'] + .decode('hex')).encode('hex') privkey = btc.encode_privkey(privkey, 'hex_compressed') if epk_m['mixdepth'] not in self.imported_privkeys: self.imported_privkeys[epk_m['mixdepth']] = [] @@ -448,20 +459,20 @@ def update_cache_index(self): def get_key(self, mixing_depth, forchange, i): return btc.bip32_extract_key(btc.bip32_ckd(self.keys[mixing_depth][ - forchange], i)) + forchange], i)) def get_addr(self, mixing_depth, forchange, i): return btc.privtoaddr( - self.get_key(mixing_depth, forchange, i), get_p2pk_vbyte()) + self.get_key(mixing_depth, forchange, i), get_p2pk_vbyte()) def get_new_addr(self, mixing_depth, forchange): index = self.index[mixing_depth] addr = self.get_addr(mixing_depth, forchange, index[forchange]) self.addr_cache[addr] = (mixing_depth, forchange, index[forchange]) index[forchange] += 1 - #self.update_cache_index() - if isinstance(bc_interface, blockchaininterface.BitcoinCoreInterface): - if bc_interface.wallet_synced: #do not import in the middle of sync_wallet() + # self.update_cache_index() + if isinstance(bc_interface, BitcoinCoreInterface): + if bc_interface.wallet_synced: # do not import in the middle of sync_wallet() if bc_interface.rpc('getaccount', [addr]) == '': debug('importing address ' + addr + ' to bitcoin core') bc_interface.rpc('importaddress', @@ -493,7 +504,7 @@ def remove_old_utxos(self, tx): removed_utxos[utxo] = self.unspent[utxo] del self.unspent[utxo] debug('removed utxos, wallet now is \n' + pprint.pformat( - self.get_utxos_by_mixdepth())) + self.get_utxos_by_mixdepth())) self.spent_utxos += removed_utxos.keys() return removed_utxos @@ -508,7 +519,7 @@ def add_new_utxos(self, tx, txid): added_utxos[utxo] = addrdict self.unspent[utxo] = addrdict debug('added utxos, wallet now is \n' + pprint.pformat( - self.get_utxos_by_mixdepth())) + self.get_utxos_by_mixdepth())) return added_utxos def get_utxos_by_mixdepth(self): @@ -528,13 +539,12 @@ def get_utxos_by_mixdepth(self): class BitcoinCoreWallet(AbstractWallet): - def __init__(self, fromaccount): super(BitcoinCoreWallet, self).__init__() if not isinstance(bc_interface, - blockchaininterface.BitcoinCoreInterface): + BitcoinCoreInterface): raise RuntimeError( - 'Bitcoin Core wallet can only be used when blockchain interface is BitcoinCoreInterface') + 'Bitcoin Core wallet can only be used when blockchain interface is BitcoinCoreInterface') self.fromaccount = fromaccount self.max_mix_depth = 1 @@ -549,12 +559,14 @@ def get_utxos_by_mixdepth(self): if not u['spendable']: continue if self.fromaccount and ( - ('account' not in u) or u['account'] != self.fromaccount): + ('account' not in u) or u[ + 'account'] != self.fromaccount): continue result[0][u['txid'] + ':' + str(u[ - 'vout'])] = {'address': u['address'], - 'value': - int(Decimal(str(u['amount'])) * Decimal('1e8'))} + 'vout'])] = { + 'address': u['address'], + 'value': + int(Decimal(str(u['amount'])) * Decimal('1e8'))} return result def get_change_addr(self, mixing_depth): @@ -563,20 +575,20 @@ def get_change_addr(self, mixing_depth): def ensure_wallet_unlocked(self): wallet_info = bc_interface.rpc('getwalletinfo', []) if 'unlocked_until' in wallet_info and wallet_info[ - 'unlocked_until'] <= 0: + 'unlocked_until'] <= 0: while True: - password = getpass.getpass( - 'Enter passphrase to unlock wallet: ') + password = getpass( + 'Enter passphrase to unlock wallet: ') if password == '': raise RuntimeError('Aborting wallet unlock') try: - #TODO cleanly unlock wallet after use, not with arbitrary timeout + # TODO cleanly unlock wallet after use, not with arbitrary timeout bc_interface.rpc('walletpassphrase', [password, 10]) break - except jsonrpc.JsonRpcError as exc: + except JsonRpcError as exc: if exc.code != -14: raise exc - # Wrong passphrase, try again. + # Wrong passphrase, try again. def calc_cj_fee(ordertype, cjfee, cj_amount): @@ -585,7 +597,7 @@ def calc_cj_fee(ordertype, cjfee, cj_amount): real_cjfee = int(cjfee) elif ordertype == 'relorder': real_cjfee = int((Decimal(cjfee) * Decimal(cj_amount)).quantize(Decimal( - 1))) + 1))) else: raise RuntimeError('unknown order type: ' + str(ordertype)) return real_cjfee @@ -647,37 +659,40 @@ def pick_order(orders, n, feekey): pickedOrderIndex = -1 continue - if pickedOrderIndex >= 0 and pickedOrderIndex < len(orders): + if 0 <= pickedOrderIndex < len(orders): return orders[pickedOrderIndex] pickedOrderIndex = -1 -def choose_orders(db, cj_amount, n, chooseOrdersBy, ignored_makers=[]): +def choose_orders(db, cj_amount, n, chooseOrdersBy, ignored_makers=None): + if ignored_makers is None: + ignored_makers = [] sqlorders = db.execute('SELECT * FROM orderbook;').fetchall() orders = [(o['counterparty'], o['oid'], calc_cj_fee( - o['ordertype'], o['cjfee'], cj_amount), o['txfee']) + o['ordertype'], o['cjfee'], cj_amount), o['txfee']) for o in sqlorders - if cj_amount >= o['minsize'] and cj_amount <= o['maxsize'] and o[ + if o['minsize'] <= cj_amount <= o['maxsize'] and o[ 'counterparty'] not in ignored_makers] - feekey = lambda o: o[2] - o[3] # function that returns the fee for a given order + feekey = lambda o: o[2] - o[ + 3] # function that returns the fee for a given order counterparties = set([o[0] for o in orders]) if n > len(counterparties): debug( - 'ERROR not enough liquidity in the orderbook n=%d suitable-counterparties=%d amount=%d totalorders=%d' - % (n, len(counterparties), cj_amount, len(orders))) - return None, 0 #TODO handle not enough liquidity better, maybe an Exception - #restrict to one order per counterparty, choose the one with the lowest cjfee - #this is done in advance of the order selection algo, so applies to all of them. - #however, if orders are picked manually, allow duplicates. + 'ERROR not enough liquidity in the orderbook n=%d suitable-counterparties=%d amount=%d totalorders=%d' + % (n, len(counterparties), cj_amount, len(orders))) + return None, 0 # TODO handle not enough liquidity better, maybe an Exception + # restrict to one order per counterparty, choose the one with the lowest cjfee + # this is done in advance of the order selection algo, so applies to all of them. + # however, if orders are picked manually, allow duplicates. if chooseOrdersBy != pick_order: orders = sorted( - dict((v[0], v) for v in sorted(orders, - key=feekey, - reverse=True)).values(), - key=feekey) + dict((v[0], v) for v in sorted(orders, + key=feekey, + reverse=True)).values(), + key=feekey) else: orders = sorted(orders, - key=feekey) #sort from smallest to biggest cj fee + key=feekey) # sort from smallest to biggest cj fee debug('considered orders = \n' + '\n'.join([str(o) for o in orders])) total_cj_fee = 0 @@ -685,7 +700,7 @@ def choose_orders(db, cj_amount, n, chooseOrdersBy, ignored_makers=[]): for i in range(n): chosen_order = chooseOrdersBy(orders, n, feekey) orders = [o for o in orders if o[0] != chosen_order[0] - ] #remove all orders from that same counterparty + ] # remove all orders from that same counterparty chosen_orders.append(chosen_order) total_cj_fee += chosen_order[2] debug('chosen orders = \n' + '\n'.join([str(o) for o in chosen_orders])) @@ -698,7 +713,7 @@ def choose_sweep_orders(db, total_txfee, n, chooseOrdersBy, - ignored_makers=[]): + ignored_makers=None): ''' choose an order given that we want to be left with no change i.e. sweep an entire group of utxos @@ -710,6 +725,9 @@ def choose_sweep_orders(db, => cjamount = (totalin - mytxfee - sum(absfee)) / (1 + sum(relfee)) ''' + if ignored_makers is None: + ignored_makers = [] + def calc_zero_change_cj_amount(ordercombo): sumabsfee = 0 sumrelfee = Decimal('0') @@ -722,46 +740,47 @@ def calc_zero_change_cj_amount(ordercombo): sumrelfee += Decimal(order[0]['cjfee']) else: raise RuntimeError('unknown order type: ' + str(order[0][ - 'ordertype'])) + 'ordertype'])) my_txfee = max(total_txfee - sumtxfee_contribution, 0) cjamount = (total_input_value - my_txfee - sumabsfee) / (1 + sumrelfee) cjamount = int(cjamount.quantize(Decimal(1))) return cjamount, int(sumabsfee + sumrelfee * cjamount) debug('choosing sweep orders for total_input_value = ' + str( - total_input_value)) + total_input_value)) sqlorders = db.execute('SELECT * FROM orderbook WHERE minsize <= ?;', (total_input_value,)).fetchall() orderkeys = ['counterparty', 'oid', 'ordertype', 'minsize', 'maxsize', 'txfee', 'cjfee'] orderlist = [dict([(k, o[k]) for k in orderkeys]) for o in sqlorders if o['counterparty'] not in ignored_makers] - #orderlist = sqlorders #uncomment this and comment previous two lines for faster runtime but less readable output + # orderlist = sqlorders #uncomment this and comment previous two lines for faster runtime but less readable output debug('orderlist = \n' + '\n'.join([str(o) for o in orderlist])) - #choose N amount of orders + # choose N amount of orders available_orders = [(o, calc_cj_fee(o['ordertype'], o['cjfee'], total_input_value), o['txfee']) for o in orderlist] - feekey = lambda o: o[1] - o[2] # function that returns the fee for a given order + feekey = lambda o: o[1] - o[ + 2] # function that returns the fee for a given order available_orders = sorted(available_orders, - key=feekey) #sort from smallest to biggest cj fee + key=feekey) # sort from smallest to biggest cj fee chosen_orders = [] while len(chosen_orders) < n: if len(available_orders) < n - len(chosen_orders): debug('ERROR not enough liquidity in the orderbook') - return None, 0 #TODO handle not enough liquidity better, maybe an Exception + return None, 0 # TODO handle not enough liquidity better, maybe an Exception for i in range(n - len(chosen_orders)): chosen_order = chooseOrdersBy(available_orders, n, feekey) debug('chosen = ' + str(chosen_order)) - #remove all orders from that same counterparty + # remove all orders from that same counterparty available_orders = [ o for o in available_orders if o[0]['counterparty'] != chosen_order[0]['counterparty'] - ] + ] chosen_orders.append(chosen_order) - #calc cj_amount and check its in range + # calc cj_amount and check its in range cj_amount, total_fee = calc_zero_change_cj_amount(chosen_orders) for c in list(chosen_orders): minsize = c[0]['minsize'] @@ -772,3 +791,50 @@ def calc_zero_change_cj_amount(ordercombo): result = dict([(o[0]['counterparty'], o[0]['oid']) for o in chosen_orders]) debug('cj amount = ' + str(cj_amount)) return result, cj_amount + + +from .slowaes import * +from .message_channel import * +from .socks import * +from .message_channel import * +from .old_mnemonic import * +from .jsonrpc import * +from .enc_wrapper import * + +from .irc import * +from .taker import * + +from joinmarket.blockchaininterface import * + +def load_program_config(): + loadedFiles = config.read([config_location]) + # Create default config file if not found + if len(loadedFiles) != 1: + config.readfp(io.BytesIO(defaultconfig)) + with open(config_location, "w") as configfile: + configfile.write(defaultconfig) + + # check for sections + for s in required_options: + if s not in config.sections(): + raise Exception( + "Config file does not contain the required section: " + s) + # then check for specific options + for k, v in required_options.iteritems(): + for o in v: + if o not in config.options(k): + raise Exception( + "Config file does not contain the required option: " + o) + + try: + global maker_timeout_sec + maker_timeout_sec = config.getint('MESSAGING', 'maker_timeout_sec') + except NoOptionError: + debug('maker_timeout_sec not found in .cfg file, using default value') + + # configure the interface to the blockchain on startup + global bc_interface + bc_interface = get_blockchain_interface_instance(config) + +from .maker import * + diff --git a/joinmarket/blockchaininterface.py b/joinmarket/blockchaininterface.py index d224065e..dc608a8c 100644 --- a/joinmarket/blockchaininterface.py +++ b/joinmarket/blockchaininterface.py @@ -1,16 +1,28 @@ -#from joinmarket import * -import unittest -import json, threading, abc, pprint, time, random, sys, os, re -import BaseHTTPServer, urllib +from __future__ import absolute_import + +import BaseHTTPServer +import abc +import json +import os +import pprint +import random +import re +import sys +import threading +import time +import urllib from decimal import Decimal -import bitcoin as btc -import common -import jsonrpc +import bitcoin as btc # This can be removed once CliJsonRpc is gone. import subprocess +from joinmarket import get_log, get_network, get_p2pk_vbyte, chunks, config, \ + BitcoinCoreWallet, JsonRpc, JsonRpcConnectionError + +log = get_log() + class CliJsonRpc(object): """ @@ -54,14 +66,14 @@ def is_index_ahead_of_cache(wallet, mix_depth, forchange): def get_blockchain_interface_instance(config): source = config.get("BLOCKCHAIN", "blockchain_source") - network = common.get_network() + network = get_network() testnet = network == 'testnet' if source == 'bitcoin-rpc': rpc_host = config.get("BLOCKCHAIN", "rpc_host") rpc_port = config.get("BLOCKCHAIN", "rpc_port") rpc_user = config.get("BLOCKCHAIN", "rpc_user") rpc_password = config.get("BLOCKCHAIN", "rpc_password") - rpc = jsonrpc.JsonRpc(rpc_host, rpc_port, rpc_user, rpc_password) + rpc = JsonRpc(rpc_host, rpc_port, rpc_user, rpc_password) bc_interface = BitcoinCoreInterface(rpc, network) elif source == 'json-rpc': bitcoin_cli_cmd = config.get("BLOCKCHAIN", "bitcoin_cli_cmd").split(' ') @@ -72,7 +84,7 @@ def get_blockchain_interface_instance(config): rpc_port = config.get("BLOCKCHAIN", "rpc_port") rpc_user = config.get("BLOCKCHAIN", "rpc_user") rpc_password = config.get("BLOCKCHAIN", "rpc_password") - rpc = jsonrpc.JsonRpc(rpc_host, rpc_port, rpc_user, rpc_password) + rpc = JsonRpc(rpc_host, rpc_port, rpc_user, rpc_password) bc_interface = RegtestBitcoinCoreInterface(rpc) elif source == 'blockr': bc_interface = BlockrInterface(testnet) @@ -118,7 +130,7 @@ def query_utxo_set(self, txouts): returns None if they are spend or unconfirmed otherwise returns value in satoshis, address and output script ''' - #address and output script contain the same information btw + # address and output script contain the same information btw class BlockrInterface(BlockchainInterface): @@ -126,13 +138,13 @@ class BlockrInterface(BlockchainInterface): def __init__(self, testnet=False): super(BlockrInterface, self).__init__() - self.network = 'testnet' if testnet else 'btc' #see bci.py in bitcoin module + self.network = 'testnet' if testnet else 'btc' # see bci.py in bitcoin module self.blockr_domain = 'tbtc' if testnet else 'btc' self.last_sync_unspent = 0 def sync_addresses(self, wallet): - common.debug('downloading wallet history') - #sets Wallet internal indexes to be at the next unused address + log.debug('downloading wallet history') + # sets Wallet internal indexes to be at the next unused address for mix_depth in range(wallet.max_mix_depth): for forchange in [0, 1]: unused_addr_count = 0 @@ -142,14 +154,14 @@ def sync_addresses(self, wallet): addrs = [wallet.get_new_addr(mix_depth, forchange) for i in range(self.BLOCKR_MAX_ADDR_REQ_COUNT)] - #TODO send a pull request to pybitcointools + # TODO send a pull request to pybitcointools # because this surely should be possible with a function from it blockr_url = 'https://' + self.blockr_domain + '.blockr.io/api/v1/address/txs/' - #print 'downloading, lastusedaddr = ' + last_used_addr + ' unusedaddrcount= ' + str(unused_addr_count) + # print 'downloading, lastusedaddr = ' + last_used_addr + ' unusedaddrcount= ' + str(unused_addr_count) res = btc.make_request(blockr_url + ','.join(addrs)) data = json.loads(res)['data'] for dat in data: - #if forchange == 0: + # if forchange == 0: # print ' nbtxs ' + str(dat['nb_txs']) + ' addr=' + dat['address'] + ' unused=' + str(unused_addr_count) if dat['nb_txs'] != 0: last_used_addr = dat['address'] @@ -160,22 +172,23 @@ def sync_addresses(self, wallet): wallet.index[mix_depth][forchange] = 0 else: wallet.index[mix_depth][forchange] = wallet.addr_cache[ - last_used_addr][2] + 1 + last_used_addr][ + 2] + 1 def sync_unspent(self, wallet): - #finds utxos in the wallet + # finds utxos in the wallet st = time.time() - rate_limit_time = 10 * 60 #dont refresh unspent dict more often than 10 minutes + rate_limit_time = 10 * 60 # dont refresh unspent dict more often than 10 minutes if st - self.last_sync_unspent < rate_limit_time: - common.debug( - 'blockr sync_unspent() happened too recently (%dsec), skipping' - % (st - self.last_sync_unspent)) + log.debug( + 'blockr sync_unspent() happened too recently (%dsec), skipping' + % (st - self.last_sync_unspent)) return wallet.unspent = {} addrs = wallet.addr_cache.keys() if len(addrs) == 0: - common.debug('no tx used') + log.debug('no tx used') return i = 0 while i < len(addrs): @@ -183,7 +196,7 @@ def sync_unspent(self, wallet): req = addrs[i:i + inc] i += inc - #TODO send a pull request to pybitcointools + # TODO send a pull request to pybitcointools # unspent() doesnt tell you which address, you get a bunch of utxos # but dont know which privkey to sign with @@ -195,17 +208,18 @@ def sync_unspent(self, wallet): for dat in data: for u in dat['unspent']: wallet.unspent[u['tx'] + ':' + str(u[ - 'n'])] = {'address': dat['address'], - 'value': int(u['amount'].replace('.', ''))} + 'n'])] = { + 'address': dat['address'], + 'value': int(u['amount'].replace('.', ''))} for u in wallet.spent_utxos: wallet.unspent.pop(u, None) self.last_sync_unspent = time.time() - common.debug('blockr sync_unspent took ' + str((self.last_sync_unspent - - st)) + 'sec') + log.debug('blockr sync_unspent took ' + str((self.last_sync_unspent - + st)) + 'sec') def add_tx_notify(self, txd, unconfirmfun, confirmfun, notifyaddr): - unconfirm_timeout = 10 * 60 #seconds + unconfirm_timeout = 10 * 60 # seconds unconfirm_poll_period = 5 confirm_timeout = 2 * 60 * 60 confirm_poll_period = 5 * 60 @@ -221,10 +235,10 @@ def __init__(self, blockr_domain, txd, unconfirmfun, confirmfun): self.tx_output_set = set([(sv['script'], sv['value']) for sv in txd['outs']]) self.output_addresses = [btc.script_to_address( - scrval[0], common.get_p2pk_vbyte()) + scrval[0], get_p2pk_vbyte()) for scrval in self.tx_output_set] - common.debug('txoutset=' + pprint.pformat(self.tx_output_set)) - common.debug('outaddrs=' + ','.join(self.output_addresses)) + log.debug('txoutset=' + pprint.pformat(self.tx_output_set)) + log.debug('outaddrs=' + ','.join(self.output_addresses)) def run(self): st = int(time.time()) @@ -233,13 +247,13 @@ def run(self): while not unconfirmed_txid: time.sleep(unconfirm_poll_period) if int(time.time()) - st > unconfirm_timeout: - common.debug('checking for unconfirmed tx timed out') + log.debug('checking for unconfirmed tx timed out') return blockr_url = 'https://' + self.blockr_domain + '.blockr.io/api/v1/address/unspent/' random.shuffle(self.output_addresses - ) #seriously weird bug with blockr.io + ) # seriously weird bug with blockr.io data = json.loads(btc.make_request(blockr_url + ','.join( - self.output_addresses) + '?unconfirmed=1'))['data'] + self.output_addresses) + '?unconfirmed=1'))['data'] shared_txid = None for unspent_list in data: txs = set([str(txdata['tx']) @@ -248,29 +262,29 @@ def run(self): shared_txid = txs else: shared_txid = shared_txid.intersection(txs) - common.debug('sharedtxid = ' + str(shared_txid)) + log.debug('sharedtxid = ' + str(shared_txid)) if len(shared_txid) == 0: continue time.sleep( - 2 - ) #here for some race condition bullshit with blockr.io + 2 + ) # here for some race condition bullshit with blockr.io blockr_url = 'https://' + self.blockr_domain + '.blockr.io/api/v1/tx/raw/' data = json.loads(btc.make_request(blockr_url + ','.join( - shared_txid)))['data'] + shared_txid)))['data'] if not isinstance(data, list): data = [data] for txinfo in data: txhex = str(txinfo['tx']['hex']) outs = set([(sv['script'], sv['value']) for sv in btc.deserialize(txhex)['outs']]) - common.debug('unconfirm query outs = ' + str(outs)) + log.debug('unconfirm query outs = ' + str(outs)) if outs == self.tx_output_set: unconfirmed_txid = txinfo['tx']['txid'] unconfirmed_txhex = str(txinfo['tx']['hex']) break self.unconfirmfun( - btc.deserialize(unconfirmed_txhex), unconfirmed_txid) + btc.deserialize(unconfirmed_txhex), unconfirmed_txid) st = int(time.time()) confirmed_txid = None @@ -278,11 +292,11 @@ def run(self): while not confirmed_txid: time.sleep(confirm_poll_period) if int(time.time()) - st > confirm_timeout: - common.debug('checking for confirmed tx timed out') + log.debug('checking for confirmed tx timed out') return blockr_url = 'https://' + self.blockr_domain + '.blockr.io/api/v1/address/txs/' data = json.loads(btc.make_request(blockr_url + ','.join( - self.output_addresses)))['data'] + self.output_addresses)))['data'] shared_txid = None for addrtxs in data: txs = set([str(txdata['tx']) for txdata in addrtxs[ @@ -291,25 +305,25 @@ def run(self): shared_txid = txs else: shared_txid = shared_txid.intersection(txs) - common.debug('sharedtxid = ' + str(shared_txid)) + log.debug('sharedtxid = ' + str(shared_txid)) if len(shared_txid) == 0: continue blockr_url = 'https://' + self.blockr_domain + '.blockr.io/api/v1/tx/raw/' data = json.loads(btc.make_request(blockr_url + ','.join( - shared_txid)))['data'] + shared_txid)))['data'] if not isinstance(data, list): data = [data] for txinfo in data: txhex = str(txinfo['tx']['hex']) outs = set([(sv['script'], sv['value']) for sv in btc.deserialize(txhex)['outs']]) - common.debug('confirm query outs = ' + str(outs)) + log.debug('confirm query outs = ' + str(outs)) if outs == self.tx_output_set: confirmed_txid = txinfo['tx']['txid'] confirmed_txhex = str(txinfo['tx']['hex']) break self.confirmfun( - btc.deserialize(confirmed_txhex), confirmed_txid, 1) + btc.deserialize(confirmed_txhex), confirmed_txid, 1) NotifyThread(self.blockr_domain, txd, unconfirmfun, confirmfun).start() @@ -317,11 +331,11 @@ def pushtx(self, txhex): try: json_str = btc.blockr_pushtx(txhex, self.network) except Exception: - common.debug('failed blockr.io pushtx') + log.debug('failed blockr.io pushtx') return None data = json.loads(json_str) if data['status'] != 'success': - common.debug(data) + log.debug(data) return None return data['data'] @@ -329,17 +343,17 @@ def query_utxo_set(self, txout): if not isinstance(txout, list): txout = [txout] txids = [h[:64] for h in txout] - txids = list(set(txids)) #remove duplicates - #self.BLOCKR_MAX_ADDR_REQ_COUNT = 2 + txids = list(set(txids)) # remove duplicates + # self.BLOCKR_MAX_ADDR_REQ_COUNT = 2 if len(txids) > self.BLOCKR_MAX_ADDR_REQ_COUNT: - txids = common.chunks(txids, self.BLOCKR_MAX_ADDR_REQ_COUNT) + txids = chunks(txids, self.BLOCKR_MAX_ADDR_REQ_COUNT) else: txids = [txids] data = [] for ids in txids: blockr_url = 'https://' + self.blockr_domain + '.blockr.io/api/v1/tx/info/' blockr_data = json.loads(btc.make_request(blockr_url + ','.join( - ids)))['data'] + ids)))['data'] if not isinstance(blockr_data, list): blockr_data = [blockr_data] data += blockr_data @@ -351,19 +365,18 @@ def query_utxo_set(self, txout): result.append(None) else: result.append({'value': int(Decimal(vout['amount']) * Decimal( - '1e8')), + '1e8')), 'address': vout['address'], 'script': vout['extras']['script']}) return result class NotifyRequestHeader(BaseHTTPServer.BaseHTTPRequestHandler): - def __init__(self, request, client_address, base_server): self.btcinterface = base_server.btcinterface self.base_server = base_server BaseHTTPServer.BaseHTTPRequestHandler.__init__( - self, request, client_address, base_server) + self, request, client_address, base_server) def do_HEAD(self): pages = ('/walletnotify?', '/alertnotify?') @@ -371,11 +384,11 @@ def do_HEAD(self): if self.path.startswith('/walletnotify?'): txid = self.path[len(pages[0]):] if not re.match('^[0-9a-fA-F]*$', txid): - common.debug('not a txid') + log.debug('not a txid') return tx = self.btcinterface.rpc('getrawtransaction', [txid]) if not re.match('^[0-9a-fA-F]*$', tx): - common.debug('not a txhex') + log.debug('not a txhex') return txd = btc.deserialize(tx) tx_output_set = set([(sv['script'], sv['value']) for sv in txd[ @@ -387,41 +400,40 @@ def do_HEAD(self): unconfirmfun = ucfun confirmfun = cfun break - if unconfirmfun == None: - common.debug('txid=' + txid + ' not being listened for') + if unconfirmfun is None: + log.debug('txid=' + txid + ' not being listened for') else: - txdata = None #on rare occasions people spend their output without waiting for a confirm + txdata = None # on rare occasions people spend their output without waiting for a confirm for n in range(len(txd['outs'])): txdata = self.btcinterface.rpc('gettxout', [txid, n, True]) if txdata is not None: break - assert txdata != None + assert txdata is not None if txdata['confirmations'] == 0: unconfirmfun(txd, txid) - #TODO pass the total transfered amount value here somehow - #wallet_name = self.get_wallet_name() - #amount = - #bitcoin-cli move wallet_name "" amount - common.debug('ran unconfirmfun') + # TODO pass the total transfered amount value here somehow + # wallet_name = self.get_wallet_name() + # amount = + # bitcoin-cli move wallet_name "" amount + log.debug('ran unconfirmfun') else: confirmfun(txd, txid, txdata['confirmations']) self.btcinterface.txnotify_fun.remove((tx_out, unconfirmfun, confirmfun)) - common.debug('ran confirmfun') + log.debug('ran confirmfun') elif self.path.startswith('/alertnotify?'): - common.core_alert = urllib.unquote(self.path[len(pages[1]):]) - common.debug('Got an alert!\nMessage=' + common.core_alert) + core_alert = urllib.unquote(self.path[len(pages[1]):]) + log.debug('Got an alert!\nMessage=' + core_alert) os.system('curl -sI --connect-timeout 1 http://localhost:' + str( - self.base_server.server_address[1] + 1) + self.path) + self.base_server.server_address[1] + 1) + self.path) self.send_response(200) - #self.send_header('Connection', 'close') + # self.send_header('Connection', 'close') self.end_headers() class BitcoinCoreNotifyThread(threading.Thread): - def __init__(self, btcinterface): threading.Thread.__init__(self) self.daemon = True @@ -429,11 +441,11 @@ def __init__(self, btcinterface): def run(self): notify_host = 'localhost' - notify_port = 62602 #defaults - if 'notify_host' in common.config.options("BLOCKCHAIN"): - notify_host = common.config.get("BLOCKCHAIN", "notify_host").strip() - if 'notify_port' in common.config.options("BLOCKCHAIN"): - notify_port = int(common.config.get("BLOCKCHAIN", "notify_port")) + notify_port = 62602 # defaults + if 'notify_host' in config.options("BLOCKCHAIN"): + notify_host = config.get("BLOCKCHAIN", "notify_host").strip() + if 'notify_port' in config.options("BLOCKCHAIN"): + notify_port = int(config.get("BLOCKCHAIN", "notify_port")) for inc in range(10): hostport = (notify_host, notify_port + inc) try: @@ -441,20 +453,20 @@ def run(self): except Exception: continue httpd.btcinterface = self.btcinterface - common.debug('started bitcoin core notify listening thread, host=' + - str(notify_host) + ' port=' + str(hostport[1])) + log.debug('started bitcoin core notify listening thread, host=' + + str(notify_host) + ' port=' + str(hostport[1])) httpd.serve_forever() - common.debug('failed to bind for bitcoin core notify listening') + log.debug('failed to bind for bitcoin core notify listening') + -#must run bitcoind with -server -#-walletnotify="curl -sI --connect-timeout 1 http://localhost:62602/walletnotify?%s" -#and make sure curl is installed (git uses it, odds are you've already got it) +# must run bitcoind with -server +# -walletnotify="curl -sI --connect-timeout 1 http://localhost:62602/walletnotify?%s" +# and make sure curl is installed (git uses it, odds are you've already got it) -#TODO must add the tx addresses as watchonly if case we ever broadcast a tx +# TODO must add the tx addresses as watchonly if case we ever broadcast a tx # with addresses not belonging to us class BitcoinCoreInterface(BlockchainInterface): - def __init__(self, jsonRpc, network): super(BitcoinCoreInterface, self).__init__() self.jsonRpc = jsonRpc @@ -475,26 +487,26 @@ def get_wallet_name(self, wallet): def rpc(self, method, args): if method not in ['importaddress', 'walletpassphrase']: - common.debug('rpc: ' + method + " " + str(args)) + log.debug('rpc: ' + method + " " + str(args)) res = self.jsonRpc.call(method, args) if isinstance(res, unicode): res = str(res) return res def add_watchonly_addresses(self, addr_list, wallet_name): - common.debug('importing ' + str(len(addr_list)) + - ' addresses into account ' + wallet_name) + log.debug('importing ' + str(len(addr_list)) + + ' addresses into account ' + wallet_name) for addr in addr_list: self.rpc('importaddress', [addr, wallet_name, False]) - if common.config.get("BLOCKCHAIN", "blockchain_source") != 'regtest': + if config.get("BLOCKCHAIN", "blockchain_source") != 'regtest': print 'restart Bitcoin Core with -rescan if you\'re recovering an existing wallet from backup seed' print ' otherwise just restart this joinmarket script' sys.exit(0) def sync_addresses(self, wallet): - if isinstance(wallet, common.BitcoinCoreWallet): + if isinstance(wallet, BitcoinCoreWallet): return - common.debug('requesting wallet history') + log.debug('requesting wallet history') wallet_name = self.get_wallet_name(wallet) addr_req_count = 20 wallet_addr_list = [] @@ -503,11 +515,11 @@ def sync_addresses(self, wallet): wallet_addr_list += [wallet.get_new_addr(mix_depth, forchange) for i in range(addr_req_count)] wallet.index[mix_depth][forchange] = 0 - #makes more sense to add these in an account called "joinmarket-imported" but its much + # makes more sense to add these in an account called "joinmarket-imported" but its much # simpler to add to the same account here for privkey_list in wallet.imported_privkeys.values(): for privkey in privkey_list: - imported_addr = btc.privtoaddr(privkey, common.get_p2pk_vbyte()) + imported_addr = btc.privtoaddr(privkey, get_p2pk_vbyte()) wallet_addr_list.append(imported_addr) imported_addr_list = self.rpc('getaddressesbyaccount', [wallet_name]) if not set(wallet_addr_list).issubset(set(imported_addr_list)): @@ -521,9 +533,9 @@ def sync_addresses(self, wallet): buf = self.rpc('listtransactions', [wallet_name, 1000, len(txs), True]) txs += buf - #TODO check whether used_addr_list can be a set, may be faster (if its a hashset) and allows + # TODO check whether used_addr_list can be a set, may be faster (if its a hashset) and allows # using issubset() here and setdiff() for finding which addresses need importing - #TODO also check the fastest way to build up python lists, i suspect using += is slow + # TODO also check the fastest way to build up python lists, i suspect using += is slow used_addr_list = [tx['address'] for tx in txs if tx['category'] == 'receive'] too_few_addr_mix_change = [] @@ -533,15 +545,17 @@ def sync_addresses(self, wallet): last_used_addr = '' breakloop = False while not breakloop: - if unused_addr_count >= wallet.gaplimit and\ - is_index_ahead_of_cache(wallet, mix_depth, forchange): + if unused_addr_count >= wallet.gaplimit and \ + is_index_ahead_of_cache(wallet, mix_depth, + forchange): break mix_change_addrs = [wallet.get_new_addr( - mix_depth, forchange) for i in range(addr_req_count)] + mix_depth, forchange) for i in + range(addr_req_count)] for mc_addr in mix_change_addrs: if mc_addr not in imported_addr_list: too_few_addr_mix_change.append((mix_depth, forchange - )) + )) breakloop = True break if mc_addr in used_addr_list: @@ -554,11 +568,12 @@ def sync_addresses(self, wallet): wallet.index[mix_depth][forchange] = 0 else: wallet.index[mix_depth][forchange] = wallet.addr_cache[ - last_used_addr][2] + 1 + last_used_addr][ + 2] + 1 wallet_addr_list = [] if len(too_few_addr_mix_change) > 0: - common.debug('too few addresses in ' + str(too_few_addr_mix_change)) + log.debug('too few addresses in ' + str(too_few_addr_mix_change)) for mix_depth, forchange in too_few_addr_mix_change: wallet_addr_list += [wallet.get_new_addr(mix_depth, forchange) for i in range(addr_req_count * 3)] @@ -568,7 +583,7 @@ def sync_addresses(self, wallet): self.wallet_synced = True def sync_unspent(self, wallet): - if isinstance(wallet, common.BitcoinCoreWallet): + if isinstance(wallet, BitcoinCoreWallet): return st = time.time() wallet_name = self.get_wallet_name(wallet) @@ -582,11 +597,12 @@ def sync_unspent(self, wallet): if u['address'] not in wallet.addr_cache: continue wallet.unspent[u['txid'] + ':' + str(u[ - 'vout'])] = {'address': u['address'], - 'value': - int(Decimal(str(u['amount'])) * Decimal('1e8'))} + 'vout'])] = { + 'address': u['address'], + 'value': + int(Decimal(str(u['amount'])) * Decimal('1e8'))} et = time.time() - common.debug('bitcoind sync_unspent took ' + str((et - st)) + 'sec') + log.debug('bitcoind sync_unspent took ' + str((et - st)) + 'sec') def add_tx_notify(self, txd, unconfirmfun, confirmfun, notifyaddr): if not self.notifythread: @@ -595,7 +611,7 @@ def add_tx_notify(self, txd, unconfirmfun, confirmfun, notifyaddr): one_addr_imported = False for outs in txd['outs']: addr = btc.script_to_address(outs['script'], - common.get_p2pk_vbyte()) + get_p2pk_vbyte()) if self.rpc('getaccount', [addr]) != '': one_addr_imported = True break @@ -607,7 +623,7 @@ def add_tx_notify(self, txd, unconfirmfun, confirmfun, notifyaddr): def pushtx(self, txhex): try: return self.rpc('sendrawtransaction', [txhex]) - except jsonrpc.JsonRpcConnectionError: + except JsonRpcConnectionError: return None def query_utxo_set(self, txout): @@ -626,12 +642,11 @@ def query_utxo_set(self, txout): return result -#class for regtest chain access -#running on local daemon. Only -#to be instantiated after network is up -#with > 100 blocks. +# class for regtest chain access +# running on local daemon. Only +# to be instantiated after network is up +# with > 100 blocks. class RegtestBitcoinCoreInterface(BitcoinCoreInterface): - def __init__(self, jsonRpc): super(RegtestBitcoinCoreInterface, self).__init__(jsonRpc, 'regtest') @@ -639,7 +654,6 @@ def pushtx(self, txhex): ret = super(RegtestBitcoinCoreInterface, self).pushtx(txhex) class TickChainThread(threading.Thread): - def __init__(self, bcinterface): threading.Thread.__init__(self) self.bcinterface = bcinterface @@ -674,36 +688,37 @@ def grab_coins(self, receiving_addr, amt=50): if self.rpc('setgenerate', [True, reqd_blocks]): raise Exception("Something went wrong") ''' - #now we do a custom create transaction and push to the receiver + # now we do a custom create transaction and push to the receiver txid = self.rpc('sendtoaddress', [receiving_addr, amt]) if not txid: raise Exception("Failed to broadcast transaction") - #confirm + # confirm self.tick_forward_chain(1) return txid def get_received_by_addr(self, addresses, query_params): - #NB This will NOT return coinbase coins (but wont matter in our use case). - #allow importaddress to fail in case the address is already in the wallet + # NB This will NOT return coinbase coins (but wont matter in our use case). + # allow importaddress to fail in case the address is already in the wallet res = [] for address in addresses: self.rpc('importaddress', [address, 'watchonly']) - res.append({'address':address,'balance':\ - int(Decimal(1e8) * Decimal(self.rpc('getreceivedbyaddress', [address])))}) + res.append({'address': address, 'balance': \ + int(Decimal(1e8) * Decimal( + self.rpc('getreceivedbyaddress', [address])))}) return {'data': res} - -def main(): - #TODO some useful quick testing here, so people know if they've set it up right - myBCI = RegtestBitcoinCoreInterface() - #myBCI.send_tx('stuff') - print myBCI.get_utxos_from_addr(["n4EjHhGVS4Rod8ociyviR3FH442XYMWweD"]) - print myBCI.get_balance_at_addr(["n4EjHhGVS4Rod8ociyviR3FH442XYMWweD"]) - txid = myBCI.grab_coins('mygp9fsgEJ5U7jkPpDjX9nxRj8b5nC3Hnd', 23) - print txid - print myBCI.get_balance_at_addr(['mygp9fsgEJ5U7jkPpDjX9nxRj8b5nC3Hnd']) - print myBCI.get_utxos_from_addr(['mygp9fsgEJ5U7jkPpDjX9nxRj8b5nC3Hnd']) - - -if __name__ == '__main__': - main() +# todo: won't run anyways +# def main(): +# #TODO some useful quick testing here, so people know if they've set it up right +# myBCI = RegtestBitcoinCoreInterface() +# #myBCI.send_tx('stuff') +# print myBCI.get_utxos_from_addr(["n4EjHhGVS4Rod8ociyviR3FH442XYMWweD"]) +# print myBCI.get_balance_at_addr(["n4EjHhGVS4Rod8ociyviR3FH442XYMWweD"]) +# txid = myBCI.grab_coins('mygp9fsgEJ5U7jkPpDjX9nxRj8b5nC3Hnd', 23) +# print txid +# print myBCI.get_balance_at_addr(['mygp9fsgEJ5U7jkPpDjX9nxRj8b5nC3Hnd']) +# print myBCI.get_utxos_from_addr(['mygp9fsgEJ5U7jkPpDjX9nxRj8b5nC3Hnd']) +# +# +# if __name__ == '__main__': +# main() diff --git a/joinmarket/enc_wrapper.py b/joinmarket/enc_wrapper.py index 8c90c839..f01fc497 100644 --- a/joinmarket/enc_wrapper.py +++ b/joinmarket/enc_wrapper.py @@ -1,31 +1,34 @@ -#A wrapper for public key -#authenticated encryption -#using Diffie Hellman key -#exchange to set up a -#symmetric encryption. +from __future__ import absolute_import -import libnacl.public -import binascii, base64 +# A wrapper for public key +# authenticated encryption +# using Diffie Hellman key +# exchange to set up a +# symmetric encryption. + +import binascii + +from libnacl import public def init_keypair(fname=None): - '''Create a new encryption + '''Create a new encryption keypair; stored in file fname if provided. The keypair object is returned. ''' - kp = libnacl.public.SecretKey() + kp = public.SecretKey() if fname: - #Note: handles correct file permissions + # Note: handles correct file permissions kp.save(fname) return kp -#the next two functions are useful -#for exchaging pubkeys with counterparty +# the next two functions are useful +# for exchaging pubkeys with counterparty def get_pubkey(kp, as_hex=False): '''Given a keypair object, - return its public key, + return its public key, optionally in hex.''' return kp.hex_pk() if as_hex else kp.pk @@ -35,7 +38,7 @@ def init_pubkey(hexpk, fname=None): hex formatted string. Save to file fname if specified. ''' - pk = libnacl.public.PublicKey(binascii.unhexlify(hexpk)) + pk = public.PublicKey(binascii.unhexlify(hexpk)) if fname: pk.save(fname) return pk @@ -44,10 +47,10 @@ def init_pubkey(hexpk, fname=None): def as_init_encryption(kp, c_pk): '''Given an initialised keypair kp and a counterparty - pubkey c_pk, create a Box + pubkey c_pk, create a Box ready for encryption/decryption. ''' - return libnacl.public.Box(kp.sk, c_pk) + return public.Box(kp.sk, c_pk) ''' @@ -62,11 +65,12 @@ def as_init_encryption(kp, c_pk): 2. Nonce is handled at the implementation layer. ''' -#TODO: Sign, verify. At the moment we are using -#bitcoin signatures so it isn't necessary. + +# TODO: Sign, verify. At the moment we are using +# bitcoin signatures so it isn't necessary. -#encoding for passing over the wire +# encoding for passing over the wire def encrypt_encode(msg, box): encrypted = box.encrypt(msg) return base64.b64encode(encrypted) @@ -85,20 +89,20 @@ def test_case(case_name, num_iterations=1): for i in range(num_iterations): ab_message = ''.join( - random.choice(string.ascii_letters) - for x in range(100)) if ab_message == 'rand' else ab_message + random.choice(string.ascii_letters) + for x in range(100)) if ab_message == 'rand' else ab_message ba_message = ''.join( - random.choice(string.ascii_letters) - for x in range(100)) if ba_message == 'rand' else ba_message + random.choice(string.ascii_letters) + for x in range(100)) if ba_message == 'rand' else ba_message otw_amsg = alice_box.encrypt(ab_message) bob_ptext = bob_box.decrypt(otw_amsg) - assert bob_ptext == ab_message, "Encryption test: FAILED. Alice sent: "\ - +ab_message+" , Bob received: " + bob_ptext + assert bob_ptext == ab_message, "Encryption test: FAILED. Alice sent: " \ + + ab_message + " , Bob received: " + bob_ptext otw_bmsg = bob_box.encrypt(ba_message) alice_ptext = alice_box.decrypt(otw_bmsg) - assert alice_ptext == ba_message, "Encryption test: FAILED. Bob sent: "\ - +ba_message+" , Alice received: " + alice_ptext + assert alice_ptext == ba_message, "Encryption test: FAILED. Bob sent: " \ + + ba_message + " , Alice received: " + alice_ptext print "Encryption test PASSED for case: " + case_name @@ -107,7 +111,7 @@ def test_keypair_setup(): alice_kp = init_keypair() bob_kp = init_keypair() - #this is the DH key exchange part + # this is the DH key exchange part bob_otwpk = get_pubkey(bob_kp, True) alice_otwpk = get_pubkey(alice_kp, True) @@ -116,31 +120,31 @@ def test_keypair_setup(): alice_pk = init_pubkey(alice_otwpk) bob_box = as_init_encryption(bob_kp, alice_pk) - #now Alice and Bob can use their 'box' - #constructs (both of which utilise the same - #shared secret) to perform encryption/decryption - #to test the encryption functionality - return (alice_box, bob_box) + # now Alice and Bob can use their 'box' + # constructs (both of which utilise the same + # shared secret) to perform encryption/decryption + # to test the encryption functionality + return alice_box, bob_box if __name__ == "__main__": - alice_box, bob_box = test_keypair_setup() test_case("short ascii", alice_box, bob_box, "Attack at dawn", "Not tonight Josephine!", 5) import base64, string, random + alice_box, bob_box = test_keypair_setup() longb641 = base64.b64encode(''.join(random.choice(string.ascii_letters) for x in range(5000))) longb642 = base64.b64encode(''.join(random.choice(string.ascii_letters) for x in range(5000))) test_case("long b64", alice_box, bob_box, longb641, longb642, 5) - #test a large number of messages on the same connection + # test a large number of messages on the same connection alice_box, bob_box = test_keypair_setup() test_case("endless_wittering", alice_box, bob_box, 'rand', 'rand', 40000) - #edge cases - #1 character + # edge cases + # 1 character alice_box, bob_box = test_keypair_setup() test_case("1 char", alice_box, bob_box, '\x00', '\x00', 5) print "All test cases passed - encryption and decryption should work correctly." diff --git a/joinmarket/irc.py b/joinmarket/irc.py index c2e0064c..0c50bc62 100644 --- a/joinmarket/irc.py +++ b/joinmarket/irc.py @@ -1,12 +1,16 @@ -# -from common import * -from message_channel import MessageChannel -from message_channel import CJPeerError +from __future__ import absolute_import -import string, random -import socket, threading, time, ssl, socks -import base64, os, re -import enc_wrapper +import base64 +import random +import socket +import ssl +import threading +import time + +import socks + +from joinmarket import get_log, MessageChannel, CJPeerError, encrypt_encode, \ + chunks, ordername_list, decode_decrypt, config, get_config_irc_channel MAX_PRIVMSG_LEN = 400 COMMAND_PREFIX = '!' @@ -16,11 +20,13 @@ plaintext_commands = ["fill", "error", "pubkey", "orderbook", "relorder", "absorder", "push"] +log = get_log() + def random_nick(nick_len=9): vowels = "aeiou" consonants = ''.join([chr( - c) for c in range( + c) for c in range( ord('a'), ord('z') + 1) if vowels.find(chr(c)) == -1]) assert nick_len % 2 == 1 N = (nick_len - 1) / 2 @@ -30,15 +36,15 @@ def random_nick(nick_len=9): for v in range(N)] + [''] ircnick = ''.join([i for sl in zip(rnd_consonants, rnd_vowels) for i in sl]) ircnick = ircnick.capitalize() - print 'Generated random nickname: ' + ircnick #not using debug because it might not know the logfile name at this point + print 'Generated random nickname: ' + ircnick # not using debug because it might not know the logfile name at this point return ircnick - #Other ideas for random nickname generation: + # Other ideas for random nickname generation: # - weight randomness by frequency of letter appearance # - u always follows q # - generate different length nicks # - append two or more of these words together # - randomly combine phonetic sounds instead consonants, which may be two consecutive consonants - # - e.g. th, dj, g, p, gr, ch, sh, kr, + # - e.g. th, dj, g, p, gr, ch, sh, kr, # - neutral network that generates nicks @@ -51,25 +57,24 @@ def get_irc_nick(source): class PingThread(threading.Thread): - def __init__(self, irc): threading.Thread.__init__(self) self.daemon = True self.irc = irc def run(self): - debug('starting ping thread') + log.debug('starting ping thread') while not self.irc.give_up: time.sleep(PING_INTERVAL) try: self.irc.ping_reply = False - #maybe use this to calculate the lag one day + # maybe use this to calculate the lag one day self.irc.lockcond.acquire() self.irc.send_raw('PING LAG' + str(int(time.time() * 1000))) self.irc.lockcond.wait(PING_TIMEOUT) self.irc.lockcond.release() if not self.irc.ping_reply: - debug('irc ping timed out') + log.debug('irc ping timed out') try: self.irc.close() except: @@ -84,34 +89,33 @@ def run(self): except: pass except IOError as e: - debug('ping thread: ' + repr(e)) - debug('ended ping thread') + log.debug('ping thread: ' + repr(e)) + log.debug('ended ping thread') -#handle one channel at a time +# handle one channel at a time class IRCMessageChannel(MessageChannel): - - #close implies it will attempt to reconnect + # close implies it will attempt to reconnect def close(self): try: self.send_raw("QUIT") except IOError as e: - debug('errored while trying to quit: ' + repr(e)) + log.debug('errored while trying to quit: ' + repr(e)) def shutdown(self): self.close() self.give_up = True def send_error(self, nick, errormsg): - debug('error<%s> : %s' % (nick, errormsg)) + log.debug('error<%s> : %s' % (nick, errormsg)) self.__privmsg(nick, 'error', errormsg) raise CJPeerError() - #OrderbookWatch callback + # OrderbookWatch callback def request_orderbook(self): self.__pubmsg(COMMAND_PREFIX + 'orderbook') - #Taker callbacks + # Taker callbacks def fill_orders(self, nickoid_dict, cj_amount, taker_pubkey): for c, oid in nickoid_dict.iteritems(): msg = str(oid) + ' ' + str(cj_amount) + ' ' + taker_pubkey @@ -126,21 +130,21 @@ def send_tx(self, nick_list, txhex): for nick in nick_list: self.__privmsg(nick, 'tx', txb64) time.sleep( - 1) #HACK! really there should be rate limiting, see issue#31 + 1) # HACK! really there should be rate limiting, see issue#31 def push_tx(self, nick, txhex): txb64 = base64.b64encode(txhex.decode('hex')) self.__privmsg(nick, 'push', txb64) - #Maker callbacks + # Maker callbacks def announce_orders(self, orderlist, nick=None): - #nick=None means announce publicly + # nick=None means announce publicly order_keys = ['oid', 'minsize', 'maxsize', 'txfee', 'cjfee'] header = 'PRIVMSG ' + (nick if nick else self.channel) + ' :' orderlines = [] for i, order in enumerate(orderlist): - orderparams = COMMAND_PREFIX + order['ordertype'] +\ - ' ' + ' '.join([str(order[k]) for k in order_keys]) + orderparams = COMMAND_PREFIX + order['ordertype'] + \ + ' ' + ' '.join([str(order[k]) for k in order_keys]) orderlines.append(orderparams) line = header + ''.join(orderlines) + ' ~' if len(line) > MAX_PRIVMSG_LEN or i == len(orderlist) - 1: @@ -162,31 +166,32 @@ def send_ioauth(self, nick, utxo_list, cj_pubkey, change_addr, sig): self.__privmsg(nick, 'ioauth', authmsg) def send_sigs(self, nick, sig_list): - #TODO make it send the sigs on one line if there's space + # TODO make it send the sigs on one line if there's space for s in sig_list: self.__privmsg(nick, 'sig', s) time.sleep( - 0.5) #HACK! really there should be rate limiting, see issue#31 + 0.5) # HACK! really there should be rate limiting, see issue#31 def __pubmsg(self, message): - debug('>>pubmsg ' + message) + log.debug('>>pubmsg ' + message) self.send_raw("PRIVMSG " + self.channel + " :" + message) def __privmsg(self, nick, cmd, message): - debug('>>privmsg ' + 'nick=' + nick + ' cmd=' + cmd + ' msg=' + message) - #should we encrypt? + log.debug( + '>>privmsg ' + 'nick=' + nick + ' cmd=' + cmd + ' msg=' + message) + # should we encrypt? box, encrypt = self.__get_encryption_box(cmd, nick) - #encrypt before chunking + # encrypt before chunking if encrypt: if not box: - debug('error, dont have encryption box object for ' + nick + - ', dropping message') + log.debug('error, dont have encryption box object for ' + nick + + ', dropping message') return - message = enc_wrapper.encrypt_encode(message, box) + message = encrypt_encode(message, box) header = "PRIVMSG " + nick + " :" max_chunk_len = MAX_PRIVMSG_LEN - len(header) - len(cmd) - 4 - #1 for command prefix 1 for space 2 for trailer + # 1 for command prefix 1 for space 2 for trailer if len(message) > max_chunk_len: message_chunks = chunks(message, max_chunk_len) else: @@ -198,8 +203,8 @@ def __privmsg(self, nick, cmd, message): self.send_raw(header + m + trailer) def send_raw(self, line): - #if not line.startswith('PING LAG'): - # debug('sendraw ' + line) + # if not line.startswith('PING LAG'): + # log.debug('sendraw ' + line) self.sock.sendall(line + '\r\n') def check_for_orders(self, nick, chunks): @@ -216,8 +221,8 @@ def check_for_orders(self, nick, chunks): self.on_order_seen(counterparty, oid, ordertype, minsize, maxsize, txfee, cjfee) except IndexError as e: - debug('index error parsing chunks') - #TODO what now? just ignore iirc + log.debug('index error parsing chunks') + # TODO what now? just ignore iirc finally: return True return False @@ -228,16 +233,16 @@ def __on_privmsg(self, nick, message): return for command in message[1:].split(COMMAND_PREFIX): chunks = command.split(" ") - #looks like a very similar pattern for all of these + # looks like a very similar pattern for all of these # check for a command name, parse arguments, call a function # maybe we need some eval() trickery to do it better try: - #orderbook watch commands + # orderbook watch commands if self.check_for_orders(nick, chunks): pass - #taker commands + # taker commands elif chunks[0] == 'pubkey': maker_pk = chunks[1] if self.on_pubkey: @@ -255,7 +260,7 @@ def __on_privmsg(self, nick, message): if self.on_sig: self.on_sig(nick, sig) - #maker commands + # maker commands if chunks[0] == 'fill': try: oid = int(chunks[1]) @@ -290,8 +295,8 @@ def __on_privmsg(self, nick, message): if self.on_push_tx: self.on_push_tx(nick, txhex) except CJPeerError: - #TODO proper error handling - debug('cj peer error TODO handle') + # TODO proper error handling + log.debug('cj peer error TODO handle') continue def __on_pubmsg(self, nick, message): @@ -302,19 +307,19 @@ def __on_pubmsg(self, nick, message): if self.check_for_orders(nick, chunks): pass elif chunks[0] == 'cancel': - #!cancel [oid] + # !cancel [oid] try: oid = int(chunks[1]) if self.on_order_cancel: self.on_order_cancel(nick, oid) except ValueError as e: - debug("!cancel " + repr(e)) + log.debug("!cancel " + repr(e)) return elif chunks[0] == 'orderbook': if self.on_orderbook_requested: self.on_orderbook_requested(nick) else: - #TODO this is for testing/debugging, should be removed, see taker.py + # TODO this is for testing/debugging, should be removed, see taker.py if hasattr(self, 'debug_on_pubmsg_cmd'): self.debug_on_pubmsg_cmd(nick, chunks) @@ -324,7 +329,7 @@ def __get_encryption_box(self, cmd, nick): If so, retrieve the appropriate crypto_box object and return. Sending/receiving flag enables us to check which command strings correspond to which - type of object (maker/taker).''' #old doc, dont trust + type of object (maker/taker).''' # old doc, dont trust if cmd in plaintext_commands: return None, False else: @@ -345,57 +350,59 @@ def __handle_privmsg(self, source, target, message): if nick not in self.built_privmsg: if message[0] != COMMAND_PREFIX: - debug('message not a cmd') + log.debug('message not a cmd') return - #new message starting + # new message starting cmd_string = message[1:].split(' ')[0] if cmd_string not in plaintext_commands + encrypted_commands: - debug('cmd not in cmd_list, line="' + message + '"') + log.debug('cmd not in cmd_list, line="' + message + '"') return self.built_privmsg[nick] = [cmd_string, message[:-2]] else: self.built_privmsg[nick][1] += message[:-2] box, encrypt = self.__get_encryption_box( - self.built_privmsg[nick][0], nick) + self.built_privmsg[nick][0], nick) if message[-1] == ';': self.waiting[nick] = True elif message[-1] == '~': self.waiting[nick] = False if encrypt: if not box: - debug('error, dont have encryption box object for ' + - nick + ', dropping message') + log.debug( + 'error, dont have encryption box object for ' + + nick + ', dropping message') return - #need to decrypt everything after the command string + # need to decrypt everything after the command string to_decrypt = ''.join(self.built_privmsg[nick][1].split(' ')[ - 1]) + 1]) try: - decrypted = enc_wrapper.decode_decrypt(to_decrypt, box) + decrypted = decode_decrypt(to_decrypt, box) except ValueError as e: - debug('valueerror when decrypting, skipping: ' + repr( - e)) + log.debug( + 'valueerror when decrypting, skipping: ' + repr( + e)) return parsed = self.built_privmsg[nick][1].split(' ')[ - 0] + ' ' + decrypted + 0] + ' ' + decrypted else: parsed = self.built_privmsg[nick][1] - #wipe the message buffer waiting for the next one + # wipe the message buffer waiting for the next one del self.built_privmsg[nick] - debug("< # Copyright (C) 2014 by phelix / blockchained.com # @@ -81,7 +82,7 @@ def queryHTTP(self, obj): if response.status == 401: conn.close() raise JsonRpcConnectionError( - "authentication for JSON-RPC failed") + "authentication for JSON-RPC failed") # All of the codes below are 'fine' from a JSON-RPC point of view. if response.status not in [200, 404, 500]: diff --git a/joinmarket/maker.py b/joinmarket/maker.py index 78036fbd..cd402318 100644 --- a/joinmarket/maker.py +++ b/joinmarket/maker.py @@ -1,28 +1,34 @@ #! /usr/bin/env python +from __future__ import absolute_import + +import base64 +import pprint +import sys +import threading -from common import * -import common -from taker import CoinJoinerPeer import bitcoin as btc -import base64, pprint, threading -import enc_wrapper +from joinmarket import DUST_THRESHOLD, init_keypair, as_init_encryption, \ + init_pubkey, get_log, bc_interface, get_p2pk_vbyte, calc_cj_fee, \ + CoinJoinerPeer, load_program_config, Wallet, IRCMessageChannel, \ + debug_dump_object +log = get_log() -class CoinJoinOrder(object): +class CoinJoinOrder(object): def __init__(self, maker, nick, oid, amount, taker_pk): self.maker = maker self.oid = oid self.cj_amount = amount - if self.cj_amount <= common.DUST_THRESHOLD: + if self.cj_amount <= DUST_THRESHOLD: self.maker.msgchan.send_error(nick, 'amount below dust threshold') - #the btc pubkey of the utxo that the taker plans to use as input + # the btc pubkey of the utxo that the taker plans to use as input self.taker_pk = taker_pk - #create DH keypair on the fly for this Order object - self.kp = enc_wrapper.init_keypair() - #the encryption channel crypto box for this Order object - self.crypto_box = enc_wrapper.as_init_encryption(self.kp, \ - enc_wrapper.init_pubkey(taker_pk)) + # create DH keypair on the fly for this Order object + self.kp = init_keypair() + # the encryption channel crypto box for this Order object + self.crypto_box = as_init_encryption(self.kp, + init_pubkey(taker_pk)) order_s = [o for o in maker.orderlist if o['oid'] == oid] if len(order_s) == 0: @@ -33,30 +39,33 @@ def __init__(self, maker, nick, oid, amount, taker_pk): self.ordertype = order['ordertype'] self.txfee = order['txfee'] self.cjfee = order['cjfee'] - debug('new cjorder nick=%s oid=%d amount=%d' % (nick, oid, amount)) + log.debug('new cjorder nick=%s oid=%d amount=%d' % (nick, oid, amount)) self.utxos, self.cj_addr, self.change_addr = maker.oid_to_order( - self, oid, amount) + self, oid, amount) self.maker.wallet.update_cache_index() if not self.utxos: self.maker.msgchan.send_error( - nick, 'unable to fill order constrained by dust avoidance') - #TODO make up orders offers in a way that this error cant appear - #check nothing has messed up with the wallet code, remove this code after a while + nick, 'unable to fill order constrained by dust avoidance') + # TODO make up orders offers in a way that this error cant appear + # check nothing has messed up with the wallet code, remove this code after a while import pprint - debug('maker utxos = ' + pprint.pformat(self.utxos)) + log.debug('maker utxos = ' + pprint.pformat(self.utxos)) utxo_list = self.utxos.keys() - utxo_data = common.bc_interface.query_utxo_set(utxo_list) + utxo_data = bc_interface.query_utxo_set(utxo_list) if None in utxo_data: - debug('wrongly using an already spent utxo. utxo_data = ' + - pprint.pformat(utxo_data)) + log.debug('wrongly using an already spent utxo. utxo_data = ' + + pprint.pformat(utxo_data)) sys.exit(0) for utxo, data in zip(utxo_list, utxo_data): if self.utxos[utxo]['value'] != data['value']: - debug('wrongly labeled utxo, expected value ' + str(self.utxos[ - utxo]['value']) + ' got ' + str(data['value'])) + log.debug( + 'wrongly labeled utxo, expected value ' + str(self.utxos[ + utxo][ + 'value']) + ' got ' + str( + data['value'])) sys.exit(0) - #always a new address even if the order ends up never being + # always a new address even if the order ends up never being # furfilled, you dont want someone pretending to fill all your # orders to find out which addresses you use self.maker.msgchan.send_pubkey(nick, self.kp.hex_pk()) @@ -67,10 +76,10 @@ def auth_counterparty(self, nick, i_utxo_pubkey, btc_sig): if not btc.ecdsa_verify(self.taker_pk, btc_sig, self.i_utxo_pubkey): print 'signature didnt match pubkey and message' return False - #authorisation of taker passed - #(but input utxo pubkey is checked in verify_unsigned_tx). - #Send auth request to taker - #TODO the next 2 lines are a little inefficient. + # authorisation of taker passed + # (but input utxo pubkey is checked in verify_unsigned_tx). + # Send auth request to taker + # TODO the next 2 lines are a little inefficient. btc_key = self.maker.wallet.get_key_from_addr(self.cj_addr) btc_pub = btc.privtopub(btc_key) btc_sig = btc.ecdsa_sign(self.kp.hex_pk(), btc_key) @@ -83,13 +92,13 @@ def recv_tx(self, nick, txhex): self.tx = btc.deserialize(txhex) except IndexError as e: self.maker.msgchan.send_error(nick, 'malformed txhex. ' + repr(e)) - debug('obtained tx\n' + pprint.pformat(self.tx)) + log.debug('obtained tx\n' + pprint.pformat(self.tx)) goodtx, errmsg = self.verify_unsigned_tx(self.tx) if not goodtx: - debug('not a good tx, reason=' + errmsg) + log.debug('not a good tx, reason=' + errmsg) self.maker.msgchan.send_error(nick, errmsg) - #TODO: the above 3 errors should be encrypted, but it's a bit messy. - debug('goodtx') + # TODO: the above 3 errors should be encrypted, but it's a bit messy. + log.debug('goodtx') sigs = [] for index, ins in enumerate(self.tx['ins']): utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) @@ -99,12 +108,12 @@ def recv_tx(self, nick, txhex): txs = btc.sign(txhex, index, self.maker.wallet.get_key_from_addr(addr)) sigs.append(base64.b64encode(btc.deserialize(txs)['ins'][index][ - 'script'].decode('hex'))) - #len(sigs) > 0 guarenteed since i did verify_unsigned_tx() + 'script'].decode('hex'))) + # len(sigs) > 0 guarenteed since i did verify_unsigned_tx() - common.bc_interface.add_tx_notify(self.tx, self.unconfirm_callback, - self.confirm_callback, self.cj_addr) - debug('sending sigs ' + str(sigs)) + bc_interface.add_tx_notify(self.tx, self.unconfirm_callback, + self.confirm_callback, self.cj_addr) + log.debug('sending sigs ' + str(sigs)) self.maker.msgchan.send_sigs(nick, sigs) self.maker.active_orders[nick] = None @@ -114,8 +123,8 @@ def unconfirm_callback(self, txd, txid): removed_utxos = self.maker.wallet.remove_old_utxos(self.tx) finally: self.maker.wallet_unspent_lock.release() - debug('saw tx on network, removed_utxos=\n' + pprint.pformat( - removed_utxos)) + log.debug('saw tx on network, removed_utxos=\n' + pprint.pformat( + removed_utxos)) to_cancel, to_announce = self.maker.on_tx_unconfirmed(self, txid, removed_utxos) self.maker.modify_orders(to_cancel, to_announce) @@ -123,25 +132,26 @@ def unconfirm_callback(self, txd, txid): def confirm_callback(self, txd, txid, confirmations): self.maker.wallet_unspent_lock.acquire() try: - common.bc_interface.sync_unspent(self.maker.wallet) + bc_interface.sync_unspent(self.maker.wallet) finally: self.maker.wallet_unspent_lock.release() - debug('tx in a block') - debug('earned = ' + str(self.real_cjfee - self.txfee)) + log.debug('tx in a block') + log.debug('earned = ' + str(self.real_cjfee - self.txfee)) to_cancel, to_announce = self.maker.on_tx_confirmed(self, confirmations, txid) self.maker.modify_orders(to_cancel, to_announce) def verify_unsigned_tx(self, txd): tx_utxo_set = set([ins['outpoint']['hash'] + ':' \ - + str(ins['outpoint']['index']) for ins in txd['ins']]) - #complete authentication: check the tx input uses the authing pubkey - input_utxo_data = common.bc_interface.query_utxo_set(list(tx_utxo_set)) + + str(ins['outpoint']['index']) for ins in + txd['ins']]) + # complete authentication: check the tx input uses the authing pubkey + input_utxo_data = bc_interface.query_utxo_set(list(tx_utxo_set)) if None in input_utxo_data: return False, 'some utxos already spent or not confirmed yet' input_addresses = [u['address'] for u in input_utxo_data] - if btc.pubtoaddr(self.i_utxo_pubkey, get_p2pk_vbyte())\ - not in input_addresses: + if btc.pubtoaddr(self.i_utxo_pubkey, get_p2pk_vbyte()) \ + not in input_addresses: return False, "authenticating bitcoin address is not contained" my_utxo_set = set(self.utxos.keys()) if not tx_utxo_set.issuperset(my_utxo_set): @@ -152,8 +162,9 @@ def verify_unsigned_tx(self, txd): self.cj_amount) expected_change_value = ( my_total_in - self.cj_amount - self.txfee + self.real_cjfee) - debug('potentially earned = ' + str(self.real_cjfee - self.txfee)) - debug('mycjaddr, mychange = ' + self.cj_addr + ', ' + self.change_addr) + log.debug('potentially earned = ' + str(self.real_cjfee - self.txfee)) + log.debug( + 'mycjaddr, mychange = ' + self.cj_addr + ', ' + self.change_addr) times_seen_cj_addr = 0 times_seen_change_addr = 0 @@ -163,12 +174,12 @@ def verify_unsigned_tx(self, txd): times_seen_cj_addr += 1 if outs['value'] != self.cj_amount: return False, 'Wrong cj_amount. I expect ' + str( - self.cj_amount) + self.cj_amount) if addr == self.change_addr: times_seen_change_addr += 1 if outs['value'] != expected_change_value: return False, 'wrong change, i expect ' + str( - expected_change_value) + expected_change_value) if times_seen_cj_addr != 1 or times_seen_change_addr != 1: return False, ('cj or change addr not in tx outputs once, #cjaddr=' + str(times_seen_cj_addr) + ', #chaddr=' + @@ -181,7 +192,6 @@ class CJMakerOrderError(StandardError): class Maker(CoinJoinerPeer): - def __init__(self, msgchan, wallet): CoinJoinerPeer.__init__(self, msgchan) self.msgchan.register_channel_callbacks(self.on_welcome, @@ -200,8 +210,9 @@ def __init__(self, msgchan, wallet): def get_crypto_box_from_nick(self, nick): if nick not in self.active_orders: - debug('wrong ordering of protocol events, no crypto object, nick=' + - nick) + log.debug( + 'wrong ordering of protocol events, no crypto object, nick=' + + nick) return None else: return self.active_orders[nick].crypto_box @@ -210,9 +221,9 @@ def on_orderbook_requested(self, nick): self.msgchan.announce_orders(self.orderlist, nick) def on_order_fill(self, nick, oid, amount, taker_pubkey): - if nick in self.active_orders and self.active_orders[nick] != None: + if nick in self.active_orders and self.active_orders[nick] is not None: self.active_orders[nick] = None - debug('had a partially filled order but starting over now') + log.debug('had a partially filled order but starting over now') self.wallet_unspent_lock.acquire() try: self.active_orders[nick] = CoinJoinOrder(self, nick, oid, amount, @@ -221,14 +232,14 @@ def on_order_fill(self, nick, oid, amount, taker_pubkey): self.wallet_unspent_lock.release() def on_seen_auth(self, nick, pubkey, sig): - if nick not in self.active_orders or self.active_orders[nick] == None: + if nick not in self.active_orders or self.active_orders[nick] is None: self.msgchan.send_error(nick, 'No open order from this nick') self.active_orders[nick].auth_counterparty(nick, pubkey, sig) - #TODO if auth_counterparty returns false, remove this order from active_orders + # TODO if auth_counterparty returns false, remove this order from active_orders # and send an error def on_seen_tx(self, nick, txhex): - if nick not in self.active_orders or self.active_orders[nick] == None: + if nick not in self.active_orders or self.active_orders[nick] is None: self.msgchan.send_error(nick, 'No open order from this nick') self.wallet_unspent_lock.acquire() try: @@ -237,11 +248,11 @@ def on_seen_tx(self, nick, txhex): self.wallet_unspent_lock.release() def on_push_tx(self, nick, txhex): - debug('received txhex from ' + nick + ' to push\n' + txhex) - txid = common.bc_interface.pushtx(txhex) - debug('pushed tx ' + str(txid)) - if txid == None: - self.send_error(nick, 'Unable to push tx') + log.debug('received txhex from ' + nick + ' to push\n' + txhex) + txid = bc_interface.pushtx(txhex) + log.debug('pushed tx ' + str(txid)) + if txid is None: + self.msgchan.send_error(nick, 'Unable to push tx') def on_welcome(self): self.msgchan.announce_orders(self.orderlist) @@ -249,16 +260,18 @@ def on_welcome(self): def on_nick_leave(self, nick): if nick in self.active_orders: - debug('nick ' + nick + ' has left') + log.debug('nick ' + nick + ' has left') del self.active_orders[nick] def modify_orders(self, to_cancel, to_announce): - debug('modifying orders. to_cancel=' + str(to_cancel) + '\nto_announce=' - + str(to_announce)) + log.debug( + 'modifying orders. to_cancel=' + str(to_cancel) + '\nto_announce=' + + str(to_announce)) for oid in to_cancel: order = [o for o in self.orderlist if o['oid'] == oid] if len(order) == 0: - debug('didnt cancel order which doesnt exist, oid=' + str(oid)) + log.debug( + 'didnt cancel order which doesnt exist, oid=' + str(oid)) self.orderlist.remove(order[0]) if len(to_cancel) > 0: self.msgchan.cancel_orders(to_cancel) @@ -272,13 +285,13 @@ def modify_orders(self, to_cancel, to_announce): self.orderlist.remove(oldorder_s[0]) self.orderlist += to_announce - #these functions + # these functions # create_my_orders() # oid_to_uxto() # on_tx_unconfirmed() # on_tx_confirmed() - #define the sell-side pricing algorithm of this bot - #still might be a bad way of doing things, we'll see + # define the sell-side pricing algorithm of this bot + # still might be a bad way of doing things, we'll see def create_my_orders(self): ''' #tells the highest value possible made by combining all utxos @@ -292,7 +305,7 @@ def create_my_orders(self): return [order] ''' - #each utxo is a single absolute-fee order + # each utxo is a single absolute-fee order orderlist = [] for utxo, addrvalue in self.wallet.unspent.iteritems(): order = {'oid': self.get_next_oid(), @@ -303,14 +316,15 @@ def create_my_orders(self): 'cjfee': 100000, 'utxo': utxo, 'mixdepth': - self.wallet.addr_cache[addrvalue['address']][0]} + self.wallet.addr_cache[addrvalue['address']][0]} orderlist.append(order) - #yes you can add keys there that are never used by the rest of the Maker code + # yes you can add keys there that are never used by the rest of the Maker code # so im adding utxo and mixdepth here return orderlist - #has to return a list of utxos and mixing depth the cj address will be in + # has to return a list of utxos and mixing depth the cj address will be in # the change address will be in mixing_depth-1 + def oid_to_order(self, cjorder, oid, amount): ''' unspent = [] @@ -331,13 +345,13 @@ def get_next_oid(self): self.nextoid += 1 return self.nextoid - #gets called when the tx is seen on the network - #must return which orders to cancel or recreate + # gets called when the tx is seen on the network + # must return which orders to cancel or recreate def on_tx_unconfirmed(self, cjorder, txid, removed_utxos): - return ([cjorder.oid], []) + return [cjorder.oid], [] - #gets called when the tx is included in a block - #must return which orders to cancel or recreate + # gets called when the tx is included in a block + # must return which orders to cancel or recreate # and i have to think about how that will work for both # the blockchain explorer api method and the bitcoid walletnotify def on_tx_confirmed(self, cjorder, confirmations, txid): @@ -362,7 +376,7 @@ def on_tx_confirmed(self, cjorder, confirmations, txid): 'cjfee': 100000, 'utxo': txid + ':' + str(i)} to_announce.append(neworder) - return ([], to_announce) + return [], to_announce def main(): @@ -371,21 +385,20 @@ def main(): import sys seed = sys.argv[ 1 - ] #btc.sha256('dont use brainwallets except for holding testnet coins') + ] # btc.sha256('dont use brainwallets except for holding testnet coins') - common.load_program_config() + load_program_config() wallet = Wallet(seed, max_mix_depth=5) - common.bc_interface.sync_wallet(wallet) + bc_interface.sync_wallet(wallet) - from irc import IRCMessageChannel irc = IRCMessageChannel(nickname) maker = Maker(irc, wallet) try: print 'connecting to irc' irc.run() except: - debug('CRASHING, DUMPING EVERYTHING') - debug('wallet seed = ' + seed) + log.debug('CRASHING, DUMPING EVERYTHING') + log.debug('wallet seed = ' + seed) debug_dump_object(wallet, ['addr_cache']) debug_dump_object(maker) import traceback diff --git a/joinmarket/message_channel.py b/joinmarket/message_channel.py index 2ae2fb7b..4e460206 100644 --- a/joinmarket/message_channel.py +++ b/joinmarket/message_channel.py @@ -8,22 +8,22 @@ class MessageChannel(object): ''' def __init__(self): - #all + # all self.on_welcome = None self.on_set_topic = None self.on_connect = None self.on_disconnect = None self.on_nick_leave = None self.on_nick_change = None - #orderbook watch functions + # orderbook watch functions self.on_order_seen = None self.on_order_cancel = None - #taker functions + # taker functions self.on_error = None self.on_pubkey = None self.on_ioauth = None self.on_sig = None - #maker functions + # maker functions self.on_orderbook_requested = None self.on_order_fill = None self.on_seen_auth = None @@ -39,8 +39,8 @@ def shutdown(self): def send_error(self, nick, errormsg): pass - #callbacks for everyone - #some of these many not have meaning in a future channel, like bitmessage + # callbacks for everyone + # some of these many not have meaning in a future channel, like bitmessage def register_channel_callbacks(self, on_welcome=None, on_set_topic=None, @@ -55,7 +55,7 @@ def register_channel_callbacks(self, self.on_nick_leave = on_nick_leave self.on_nick_change = on_nick_change - #orderbook watcher commands + # orderbook watcher commands def register_orderbookwatch_callbacks(self, on_order_seen=None, on_order_cancel=None): @@ -65,7 +65,7 @@ def register_orderbookwatch_callbacks(self, def request_orderbook(self): pass - #taker commands + # taker commands def register_taker_callbacks(self, on_error=None, on_pubkey=None, @@ -88,7 +88,7 @@ def send_tx(self, nick_list, txhex): def push_tx(self, nick, txhex): pass - #maker commands + # maker commands def register_maker_callbacks(self, on_orderbook_requested=None, on_order_fill=None, @@ -102,7 +102,7 @@ def register_maker_callbacks(self, self.on_push_tx = on_push_tx def announce_orders(self, orderlist, nick=None): - pass #nick=None means announce publicly + pass # nick=None means announce publicly def cancel_orders(self, oid_list): pass diff --git a/joinmarket/old_mnemonic.py b/joinmarket/old_mnemonic.py index a75a4c57..f82562cc 100644 --- a/joinmarket/old_mnemonic.py +++ b/joinmarket/old_mnemonic.py @@ -226,7 +226,8 @@ "company", "coward", "creak", "danger", "deadly", "defense", "define", "depend", "desperate", "destination", "dew", "duck", "dusty", "embarrass", "engine", "example", "explore", "foe", "freely", "frustrate", "generation", - "glove", "guilty", "health", "hurry", "idiot", "impossible", "inhale", "jaw", + "glove", "guilty", "health", "hurry", "idiot", "impossible", "inhale", + "jaw", "kingdom", "mention", "mist", "moan", "mumble", "mutter", "observe", "ode", "pathetic", "pattern", "pie", "prefer", "puff", "rape", "rare", "revenge", "rude", "scrape", "spiral", "squeeze", "strain", "sunset", "suspend", @@ -235,6 +236,7 @@ n = 1626 + # Note about US patent no 5892470: Here each word does not represent a given digit. # Instead, the digit represented by a word is variable, it depends on the previous word. @@ -266,6 +268,7 @@ def mn_decode(wlist): if __name__ == '__main__': import sys + if len(sys.argv) == 1: print 'I need arguments: a hex string to encode, or a list of words to decode' elif len(sys.argv) == 2: diff --git a/joinmarket/slowaes.py b/joinmarket/slowaes.py index bbc81d71..f204e7af 100644 --- a/joinmarket/slowaes.py +++ b/joinmarket/slowaes.py @@ -11,9 +11,8 @@ # Licensed under the Apache License, Version 2.0 # http://www.apache.org/licenses/ # -import os -import sys import math +import os def append_PKCS7_padding(s): @@ -177,7 +176,7 @@ def expandKey(self, key, size, expandedKeySize): rconIteration += 1 # For 256-bit keys, we add an extra sbox to the calculation if size == self.keySize["SIZE_256"] and ( - (currentSize % size) == 16): + (currentSize % size) == 16): for l in range(4): t[l] = self.getSBoxValue(t[l]) @@ -186,7 +185,7 @@ def expandKey(self, key, size, expandedKeySize): # key. for m in range(4): expandedKey[currentSize] = expandedKey[currentSize - size] ^ \ - t[m] + t[m] currentSize += 1 return expandedKey @@ -227,8 +226,10 @@ def galois_multiplication(self, a, b): # using the state value as index for the SBox # def subBytes(self, state, isInv): - if isInv: getter = self.getSBoxInvert - else: getter = self.getSBoxValue + if isInv: + getter = self.getSBoxInvert + else: + getter = self.getSBoxValue for i in range(16): state[i] = getter(state[i]) return state @@ -243,13 +244,13 @@ def shiftRows(self, state, isInv): def shiftRow(self, state, statePointer, nbr, isInv): for i in range(nbr): if isInv: - state[statePointer:statePointer+4] = \ - state[statePointer+3:statePointer+4] + \ - state[statePointer:statePointer+3] + state[statePointer:statePointer + 4] = \ + state[statePointer + 3:statePointer + 4] + \ + state[statePointer:statePointer + 3] else: - state[statePointer:statePointer+4] = \ - state[statePointer+1:statePointer+4] + \ - state[statePointer:statePointer+1] + state[statePointer:statePointer + 4] = \ + state[statePointer + 1:statePointer + 4] + \ + state[statePointer:statePointer + 1] return state # galois multiplication of the 4x4 matrix @@ -267,8 +268,10 @@ def mixColumns(self, state, isInv): # galois multiplication of 1 column of the 4x4 matrix def mixColumn(self, column, isInv): - if isInv: mult = [14, 9, 13, 11] - else: mult = [2, 1, 1, 3] + if isInv: + mult = [14, 9, 13, 11] + else: + mult = [2, 1, 1, 3] cpy = list(column) g = self.galois_multiplication @@ -310,14 +313,14 @@ def aes_main(self, state, expandedKey, nbrRounds): state = self.subBytes(state, False) state = self.shiftRows(state, False) state = self.addRoundKey( - state, self.createRoundKey(expandedKey, 16 * nbrRounds)) + state, self.createRoundKey(expandedKey, 16 * nbrRounds)) return state # Perform the initial operations, the standard round, and the final # operations of the inverse aes, creating a round key for each round def aes_invMain(self, state, expandedKey, nbrRounds): state = self.addRoundKey( - state, self.createRoundKey(expandedKey, 16 * nbrRounds)) + state, self.createRoundKey(expandedKey, 16 * nbrRounds)) i = nbrRounds - 1 while i > 0: state = self.aes_invRound(state, @@ -336,10 +339,14 @@ def encrypt(self, iput, key, size): # the 128 bit block to encode block = [0] * 16 # set the number of rounds - if size == self.keySize["SIZE_128"]: nbrRounds = 10 - elif size == self.keySize["SIZE_192"]: nbrRounds = 12 - elif size == self.keySize["SIZE_256"]: nbrRounds = 14 - else: return None + if size == self.keySize["SIZE_128"]: + nbrRounds = 10 + elif size == self.keySize["SIZE_192"]: + nbrRounds = 12 + elif size == self.keySize["SIZE_256"]: + nbrRounds = 14 + else: + return None # the expanded keySize expandedKeySize = 16 * (nbrRounds + 1) @@ -379,10 +386,14 @@ def decrypt(self, iput, key, size): # the 128 bit block to decode block = [0] * 16 # set the number of rounds - if size == self.keySize["SIZE_128"]: nbrRounds = 10 - elif size == self.keySize["SIZE_192"]: nbrRounds = 12 - elif size == self.keySize["SIZE_256"]: nbrRounds = 14 - else: return None + if size == self.keySize["SIZE_128"]: + nbrRounds = 10 + elif size == self.keySize["SIZE_192"]: + nbrRounds = 12 + elif size == self.keySize["SIZE_256"]: + nbrRounds = 14 + else: + return None # the expanded keySize expandedKeySize = 16 * (nbrRounds + 1) @@ -412,7 +423,6 @@ def decrypt(self, iput, key, size): class AESModeOfOperation(object): - aes = AES() # structure of supported modes of operation @@ -421,8 +431,10 @@ class AESModeOfOperation(object): # converts a 16 character string into a number array def convertString(self, string, start, end, mode): if end - start > 16: end = start + 16 - if mode == self.modeOfOperation["CBC"]: ar = [0] * 16 - else: ar = [] + if mode == self.modeOfOperation["CBC"]: + ar = [0] * 16 + else: + ar = [] i = start j = 0 @@ -454,7 +466,7 @@ def encrypt(self, stringIn, mode, key, size, IV): cipherOut = [] # char firstRound firstRound = True - if stringIn != None: + if stringIn is not None: for j in range(int(math.ceil(float(len(stringIn)) / 16))): start = j * 16 end = j * 16 + 16 @@ -534,7 +546,7 @@ def decrypt(self, cipherIn, originalsize, mode, key, size, IV): stringOut = '' # char firstRound firstRound = True - if cipherIn != None: + if cipherIn is not None: for j in range(int(math.ceil(float(len(cipherIn)) / 16))): start = j * 16 end = j * 16 + 16 @@ -644,9 +656,9 @@ def decryptData(key, data, mode=AESModeOfOperation.modeOfOperation["CBC"]): def generateRandomKey(keysize): """Generates a key from random data of length `keysize`. - + The returned key is a string of bytes. - + """ if keysize not in (16, 24, 32): emsg = 'Invalid keysize, %s. Should be one of (16, 24, 32).' diff --git a/joinmarket/socks.py b/joinmarket/socks.py index 219b36c6..f1314803 100644 --- a/joinmarket/socks.py +++ b/joinmarket/socks.py @@ -13,7 +13,7 @@ 3. Neither the name of Dan Haim nor the names of his contributors may be used to endorse or promote products derived from this software without specific prior written permission. - + THIS SOFTWARE IS PROVIDED BY DAN HAIM "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO @@ -42,7 +42,6 @@ class ProxyError(Exception): - def __init__(self, value): self.value = value @@ -51,7 +50,6 @@ def __str__(self): class GeneralProxyError(ProxyError): - def __init__(self, value): self.value = value @@ -60,7 +58,6 @@ def __str__(self): class Socks5AuthError(ProxyError): - def __init__(self, value): self.value = value @@ -69,7 +66,6 @@ def __str__(self): class Socks5Error(ProxyError): - def __init__(self, value): self.value = value @@ -78,7 +74,6 @@ def __str__(self): class Socks4Error(ProxyError): - def __init__(self, value): self.value = value @@ -87,7 +82,6 @@ def __str__(self): class HTTPError(ProxyError): - def __init__(self, value): self.value = value @@ -131,7 +125,7 @@ def setdefaultproxy(proxytype=None, class socksocket(socket.socket): """socksocket([family[, type[, proto]]]) -> socket object - + Open a SOCKS enabled socket. The parameters are the same as those of the standard socket init. In order for SOCKS to work, you must specify family=AF_INET, type=SOCK_STREAM and proto=0. @@ -143,7 +137,7 @@ def __init__(self, proto=0, _sock=None): _orgsocket.__init__(self, family, type, proto, _sock) - if _defaultproxy != None: + if _defaultproxy is not None: self.__proxy = _defaultproxy else: self.__proxy = (None, None, None, None, None, None) @@ -190,7 +184,7 @@ def __negotiatesocks5(self, destaddr, destport): Negotiates a connection through a SOCKS5 server. """ # First we'll send the authentication packages we support. - if (self.__proxy[4] != None) and (self.__proxy[5] != None): + if (self.__proxy[4] is not None) and (self.__proxy[5] is not None): # The username/password details were supplied to the # setproxy method so we support the USERNAME/PASSWORD # authentication (in addition to the standard none). @@ -222,8 +216,8 @@ def __negotiatesocks5(self, destaddr, destport): if authstat[1] != "\x00": # Authentication failed self.close() - raise Socks5AuthError, ((3, _socks5autherrors[3])) - # Authentication succeeded + raise Socks5AuthError, (3, _socks5autherrors[3]) + # Authentication succeeded else: # Reaching here is always bad self.close() @@ -240,7 +234,7 @@ def __negotiatesocks5(self, destaddr, destport): req = req + "\x01" + ipaddr except socket.error: # Well it's not an IP number, so it's probably a DNS name. - if self.__proxy[3] == True: + if self.__proxy[3]: # Resolve remotely ipaddr = None req = req + "\x03" + chr(len(destaddr)) + destaddr @@ -270,7 +264,7 @@ def __negotiatesocks5(self, destaddr, destport): raise GeneralProxyError((1, _generalerrors[1])) boundport = struct.unpack(">H", self.__recvall(2))[0] self.__proxysockname = (boundaddr, boundport) - if ipaddr != None: + if ipaddr is not None: self.__proxypeername = (socket.inet_ntoa(ipaddr), destport) else: self.__proxypeername = (destaddr, destport) @@ -304,7 +298,7 @@ def __negotiatesocks4(self, destaddr, destport): ipaddr = socket.inet_aton(destaddr) except socket.error: # It's a DNS name. Check where it should be resolved. - if self.__proxy[3] == True: + if self.__proxy[3]: ipaddr = "\x00\x00\x00\x01" rmtrslv = True else: @@ -312,13 +306,13 @@ def __negotiatesocks4(self, destaddr, destport): # Construct the request packet req = "\x04\x01" + struct.pack(">H", destport) + ipaddr # The username parameter is considered userid for SOCKS4 - if self.__proxy[4] != None: + if self.__proxy[4] is not None: req = req + self.__proxy[4] req = req + "\x00" # DNS name if remote resolving is required # NOTE: This is actually an extension to the SOCKS4 protocol # called SOCKS4A and may not be supported in all cases. - if rmtrslv == True: + if rmtrslv: req = req + destaddr + "\x00" self.sendall(req) # Get the response from the server @@ -338,8 +332,8 @@ def __negotiatesocks4(self, destaddr, destport): raise Socks4Error((94, _socks4errors[4])) # Get the bound address/port self.__proxysockname = (socket.inet_ntoa(resp[4:]), struct.unpack( - ">H", resp[2:4])[0]) - if rmtrslv != None: + ">H", resp[2:4])[0]) + if rmtrslv is not None: self.__proxypeername = (socket.inet_ntoa(ipaddr), destport) else: self.__proxypeername = (destaddr, destport) @@ -349,7 +343,7 @@ def __negotiatehttp(self, destaddr, destport): Negotiates a connection through an HTTP server. """ # If we need to resolve locally, we do this now - if self.__proxy[3] == False: + if not self.__proxy[3]: addr = socket.gethostbyname(destaddr) else: addr = destaddr @@ -385,31 +379,31 @@ def connect(self, destpair): """ # Do a minimal input check first if (type(destpair) in - (list, tuple) == False) or (len(destpair) < 2) or ( - type(destpair[0]) != str) or (type(destpair[1]) != int): + (list, tuple) == False) or (len(destpair) < 2) or ( + type(destpair[0]) != str) or (type(destpair[1]) != int): raise GeneralProxyError((5, _generalerrors[5])) if self.__proxy[0] == PROXY_TYPE_SOCKS5: - if self.__proxy[2] != None: + if self.__proxy[2] is not None: portnum = self.__proxy[2] else: portnum = 1080 _orgsocket.connect(self, (self.__proxy[1], portnum)) self.__negotiatesocks5(destpair[0], destpair[1]) elif self.__proxy[0] == PROXY_TYPE_SOCKS4: - if self.__proxy[2] != None: + if self.__proxy[2] is not None: portnum = self.__proxy[2] else: portnum = 1080 _orgsocket.connect(self, (self.__proxy[1], portnum)) self.__negotiatesocks4(destpair[0], destpair[1]) elif self.__proxy[0] == PROXY_TYPE_HTTP: - if self.__proxy[2] != None: + if self.__proxy[2] is not None: portnum = self.__proxy[2] else: portnum = 8080 _orgsocket.connect(self, (self.__proxy[1], portnum)) self.__negotiatehttp(destpair[0], destpair[1]) - elif self.__proxy[0] == None: + elif self.__proxy[0] is None: _orgsocket.connect(self, (destpair[0], destpair[1])) else: raise GeneralProxyError((4, _generalerrors[4])) diff --git a/joinmarket/taker.py b/joinmarket/taker.py index 376104e4..7270dc4f 100644 --- a/joinmarket/taker.py +++ b/joinmarket/taker.py @@ -1,15 +1,23 @@ #! /usr/bin/env python +from __future__ import absolute_import + +import base64 +import pprint +import random +import sqlite3 +import sys +import threading +from decimal import InvalidOperation, Decimal -from common import * -import common -import enc_wrapper import bitcoin as btc +from joinmarket import get_log, init_keypair, as_init_encryption, init_pubkey, \ + bc_interface, calc_cj_fee, get_p2pk_vbyte, maker_timeout_sec, JM_VERSION -import sqlite3, base64, threading, time, random, pprint +log = get_log() class CoinJoinTX(object): - #soon the taker argument will be removed and just be replaced by wallet or some other interface + # soon the taker argument will be removed and just be replaced by wallet or some other interface def __init__(self, msgchan, wallet, @@ -28,9 +36,10 @@ def __init__(self, thats used if you want to entirely coinjoin one utxo with no change left over orders is the orders you want to fill {'counterpartynick': oid, 'cp2': oid2} ''' - debug('starting cj to ' + str(my_cj_addr) + ' with change at ' + str( - my_change_addr)) - #parameters + log.debug( + 'starting cj to ' + str(my_cj_addr) + ' with change at ' + str( + my_change_addr)) + # parameters self.msgchan = msgchan self.wallet = wallet self.db = db @@ -43,12 +52,12 @@ def __init__(self, self.my_change_addr = my_change_addr self.choose_orders_recover = choose_orders_recover self.auth_addr = auth_addr - self.timeout_lock = threading.Condition() #used to wait() and notify() - #used to restrict access to certain variables across threads + self.timeout_lock = threading.Condition() # used to wait() and notify() + # used to restrict access to certain variables across threads self.timeout_thread_lock = threading.Condition() self.end_timeout_thread = False CoinJoinTX.TimeoutThread(self).start() - #state variables + # state variables self.txid = None self.cjfee_total = 0 self.maker_txfee_contributions = 0 @@ -56,21 +65,21 @@ def __init__(self, self.all_responded = False self.latest_tx = None self.utxos = {None: - self.input_utxos.keys()} #None means they belong to me + self.input_utxos.keys()} # None means they belong to me self.outputs = [] - #create DH keypair on the fly for this Tx object - self.kp = enc_wrapper.init_keypair() + # create DH keypair on the fly for this Tx object + self.kp = init_keypair() self.crypto_boxes = {} self.msgchan.fill_orders(self.active_orders, self.cj_amount, self.kp.hex_pk()) def start_encryption(self, nick, maker_pk): if nick not in self.active_orders.keys(): - debug("Counterparty not part of this transaction. Ignoring") + log.debug("Counterparty not part of this transaction. Ignoring") return - self.crypto_boxes[nick] = [maker_pk, enc_wrapper.as_init_encryption(\ - self.kp, enc_wrapper.init_pubkey(maker_pk))] - #send authorisation request + self.crypto_boxes[nick] = [maker_pk, as_init_encryption( + self.kp, init_pubkey(maker_pk))] + # send authorisation request if self.auth_addr: my_btc_addr = self.auth_addr else: @@ -82,68 +91,70 @@ def start_encryption(self, nick, maker_pk): def auth_counterparty(self, nick, btc_sig, cj_pub): '''Validate the counterpartys claim to own the btc - address/pubkey that will be used for coinjoining + address/pubkey that will be used for coinjoining with an ecdsa verification.''' - #crypto_boxes[nick][0] = maker_pubkey + # crypto_boxes[nick][0] = maker_pubkey if not btc.ecdsa_verify(self.crypto_boxes[nick][0], btc_sig, cj_pub): - debug('signature didnt match pubkey and message') + log.debug('signature didnt match pubkey and message') return False return True def recv_txio(self, nick, utxo_list, cj_pub, change_addr): if nick not in self.nonrespondants: - debug('recv_txio => nick=' + nick + ' not in nonrespondants ' + str( - self.nonrespondants)) + log.debug( + 'recv_txio => nick=' + nick + ' not in nonrespondants ' + str( + self.nonrespondants)) return self.utxos[nick] = utxo_list order = self.db.execute('SELECT ordertype, txfee, cjfee FROM ' 'orderbook WHERE oid=? AND counterparty=?', (self.active_orders[nick], nick)).fetchone() - utxo_data = common.bc_interface.query_utxo_set(self.utxos[nick]) + utxo_data = bc_interface.query_utxo_set(self.utxos[nick]) if None in utxo_data: - common.debug( - 'ERROR outputs unconfirmed or already spent. utxo_data=' + - pprint.pformat(utxo_data)) - #when internal reviewing of makers is created, add it here to immediately quit - return #ignore this message, eventually the timeout thread will recover + log.debug( + 'ERROR outputs unconfirmed or already spent. utxo_data=' + + pprint.pformat(utxo_data)) + # when internal reviewing of makers is created, add it here to immediately quit + return # ignore this message, eventually the timeout thread will recover total_input = sum([d['value'] for d in utxo_data]) real_cjfee = calc_cj_fee(order['ordertype'], order['cjfee'], self.cj_amount) self.outputs.append({'address': change_addr, 'value': total_input - self.cj_amount - order[ 'txfee'] + real_cjfee}) - debug( - 'fee breakdown for %s totalin=%d cjamount=%d txfee=%d realcjfee=%d' - % (nick, total_input, self.cj_amount, order['txfee'], real_cjfee)) + log.debug( + 'fee breakdown for %s totalin=%d cjamount=%d txfee=%d realcjfee=%d' + % ( + nick, total_input, self.cj_amount, order['txfee'], real_cjfee)) cj_addr = btc.pubtoaddr(cj_pub, get_p2pk_vbyte()) self.outputs.append({'address': cj_addr, 'value': self.cj_amount}) self.cjfee_total += real_cjfee self.maker_txfee_contributions += order['txfee'] self.nonrespondants.remove(nick) if len(self.nonrespondants) > 0: - debug('nonrespondants = ' + str(self.nonrespondants)) + log.debug('nonrespondants = ' + str(self.nonrespondants)) return self.all_responded = True with self.timeout_lock: self.timeout_lock.notify() - debug('got all parts, enough to build a tx') + log.debug('got all parts, enough to build a tx') self.nonrespondants = list(self.active_orders.keys()) my_total_in = sum([va['value'] for u, va in self.input_utxos.iteritems() - ]) + ]) my_txfee = max(self.total_txfee - self.maker_txfee_contributions, 0) my_change_value = ( my_total_in - self.cj_amount - self.cjfee_total - my_txfee) - debug( - 'fee breakdown for me totalin=%d my_txfee=%d makers_txfee=%d cjfee_total=%d => changevalue=%d' - % (my_total_in, my_txfee, self.maker_txfee_contributions, - self.cjfee_total, my_change_value)) - if self.my_change_addr == None: + log.debug( + 'fee breakdown for me totalin=%d my_txfee=%d makers_txfee=%d cjfee_total=%d => changevalue=%d' + % (my_total_in, my_txfee, self.maker_txfee_contributions, + self.cjfee_total, my_change_value)) + if self.my_change_addr is None: if my_change_value != 0 and abs(my_change_value) != 1: - #seems you wont always get exactly zero because of integer rounding + # seems you wont always get exactly zero because of integer rounding # so 1 satoshi extra or fewer being spent as miner fees is acceptable - debug('WARNING CHANGE NOT BEING USED\nCHANGEVALUE = ' + str( - my_change_value)) + log.debug('WARNING CHANGE NOT BEING USED\nCHANGEVALUE = ' + str( + my_change_value)) else: self.outputs.append({'address': self.my_change_addr, 'value': my_change_value}) @@ -154,7 +165,7 @@ def recv_txio(self, nick, utxo_list, cj_pub, change_addr): random.shuffle(self.utxo_tx) random.shuffle(self.outputs) tx = btc.mktx(self.utxo_tx, self.outputs) - debug('obtained tx\n' + pprint.pformat(btc.deserialize(tx))) + log.debug('obtained tx\n' + pprint.pformat(btc.deserialize(tx))) self.msgchan.send_tx(self.active_orders.keys(), tx) self.latest_tx = btc.deserialize(tx) @@ -162,51 +173,54 @@ def recv_txio(self, nick, utxo_list, cj_pub, change_addr): utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) if utxo not in self.input_utxos.keys(): continue - #placeholders required + # placeholders required ins['script'] = 'deadbeef' def add_signature(self, nick, sigb64): if nick not in self.nonrespondants: - debug('add_signature => nick=' + nick + ' not in nonrespondants ' + - str(self.nonrespondants)) + log.debug( + 'add_signature => nick=' + nick + ' not in nonrespondants ' + + str(self.nonrespondants)) return sig = base64.b64decode(sigb64).encode('hex') inserted_sig = False txhex = btc.serialize(self.latest_tx) - #batch retrieval of utxo data + # batch retrieval of utxo data utxo = {} ctr = 0 for index, ins in enumerate(self.latest_tx['ins']): utxo_for_checking = ins['outpoint']['hash'] + ':' + str(ins[ - 'outpoint']['index']) - if ins['script'] != '' or utxo_for_checking in self.input_utxos.keys( + 'outpoint'][ + 'index']) + if ins[ + 'script'] != '' or utxo_for_checking in self.input_utxos.keys( ): continue utxo[ctr] = [index, utxo_for_checking] ctr += 1 - utxo_data = common.bc_interface.query_utxo_set([x[1] - for x in utxo.values()]) - #insert signatures + utxo_data = bc_interface.query_utxo_set([x[1] + for x in utxo.values()]) + # insert signatures for i, u in utxo.iteritems(): - if utxo_data[i] == None: + if utxo_data[i] is None: continue sig_good = btc.verify_tx_input(txhex, u[0], utxo_data[i]['script'], *btc.deserialize_script(sig)) if sig_good: - debug('found good sig at index=%d' % (u[0])) + log.debug('found good sig at index=%d' % (u[0])) self.latest_tx['ins'][u[0]]['script'] = sig inserted_sig = True - #check if maker has sent everything possible + # check if maker has sent everything possible self.utxos[nick].remove(u[1]) if len(self.utxos[nick]) == 0: - debug('nick = ' + nick + - ' sent all sigs, removing from nonrespondant list') + log.debug('nick = ' + nick + + ' sent all sigs, removing from nonrespondant list') self.nonrespondants.remove(nick) break if not inserted_sig: - debug('signature did not match anything in the tx') - #TODO what if the signature doesnt match anything + log.debug('signature did not match anything in the tx') + # TODO what if the signature doesnt match anything # nothing really to do except drop it, carry on and wonder why the # other guy sent a failed signature @@ -220,12 +234,12 @@ def add_signature(self, nick, sigb64): self.all_responded = True with self.timeout_lock: self.timeout_lock.notify() - debug('all makers have sent their signatures') + log.debug('all makers have sent their signatures') for index, ins in enumerate(self.latest_tx['ins']): - #remove placeholders + # remove placeholders if ins['script'] == 'deadbeef': ins['script'] = '' - if self.finishcallback != None: + if self.finishcallback is not None: self.finishcallback(self) def coinjoin_address(self): @@ -241,7 +255,7 @@ def sign_tx(self, tx, i, priv): return sign_donation_tx(tx, i, priv) def self_sign(self): - #now sign it ourselves + # now sign it ourselves tx = btc.serialize(self.latest_tx) for index, ins in enumerate(self.latest_tx['ins']): utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) @@ -253,53 +267,55 @@ def self_sign(self): def push(self, txd): tx = btc.serialize(txd) - debug('\n' + tx) - debug('txid = ' + btc.txhash(tx)) - #TODO send to a random maker or push myself - #TODO need to check whether the other party sent it - #self.msgchan.push_tx(self.active_orders.keys()[0], txhex) - self.txid = common.bc_interface.pushtx(tx) - if self.txid == None: - debug('unable to pushtx') + log.debug('\n' + tx) + log.debug('txid = ' + btc.txhash(tx)) + # TODO send to a random maker or push myself + # TODO need to check whether the other party sent it + # self.msgchan.push_tx(self.active_orders.keys()[0], txhex) + self.txid = bc_interface.pushtx(tx) + if self.txid is None: + log.debug('unable to pushtx') def self_sign_and_push(self): self.self_sign() self.push(self.latest_tx) def recover_from_nonrespondants(self): - debug('nonresponding makers = ' + str(self.nonrespondants)) - #if there is no choose_orders_recover then end and call finishcallback + log.debug('nonresponding makers = ' + str(self.nonrespondants)) + # if there is no choose_orders_recover then end and call finishcallback # so the caller can handle it in their own way, notable for sweeping # where simply replacing the makers wont work if not self.choose_orders_recover: self.end_timeout_thread = True - if self.finishcallback != None: + if self.finishcallback is not None: self.finishcallback(self) return - if self.latest_tx == None: - #nonresponding to !fill, recover by finding another maker - debug('nonresponse to !fill') + if self.latest_tx is None: + # nonresponding to !fill, recover by finding another maker + log.debug('nonresponse to !fill') for nr in self.nonrespondants: del self.active_orders[nr] new_orders, new_makers_fee = self.choose_orders_recover( - self.cj_amount, len(self.nonrespondants), self.nonrespondants, - self.active_orders.keys()) + self.cj_amount, len(self.nonrespondants), + self.nonrespondants, + self.active_orders.keys()) for nick, order in new_orders.iteritems(): self.active_orders[nick] = order self.nonrespondants = list(new_orders.keys()) - debug('new active_orders = ' + pprint.pformat(self.active_orders) + - '\nnew nonrespondants = ' + pprint.pformat( - self.nonrespondants)) + log.debug( + 'new active_orders = ' + pprint.pformat(self.active_orders) + + '\nnew nonrespondants = ' + pprint.pformat( + self.nonrespondants)) self.msgchan.fill_orders(new_orders, self.cj_amount, self.kp.hex_pk()) else: - debug('nonresponse to !sig') - #nonresponding to !sig, have to restart tx from the beginning + log.debug('nonresponse to !sig') + # nonresponding to !sig, have to restart tx from the beginning self.end_timeout_thread = True - if self.finishcallback != None: + if self.finishcallback is not None: self.finishcallback(self) - #finishcallback will check if self.all_responded is True and will know it came from here + # finishcallback will check if self.all_responded is True and will know it came from here class TimeoutThread(threading.Thread): @@ -308,32 +324,32 @@ def __init__(self, cjtx): self.cjtx = cjtx def run(self): - debug('started timeout thread for coinjoin of amount ' + str( - self.cjtx.cj_amount) + ' to addr ' + str(self.cjtx.my_cj_addr)) + log.debug('started timeout thread for coinjoin of amount ' + str( + self.cjtx.cj_amount) + ' to addr ' + str( + self.cjtx.my_cj_addr)) - #how the threading to check for nonresponding makers works like this - #there is a Condition object - #in a loop, call cond.wait(timeout) + # how the threading to check for nonresponding makers works like this + # there is a Condition object + # in a loop, call cond.wait(timeout) # after it returns, check a boolean # to see if if the messages have arrived while not self.cjtx.end_timeout_thread: - debug('waiting for all replies.. timeout=' + str( - common.maker_timeout_sec)) + log.debug('waiting for all replies.. timeout=' + str( + maker_timeout_sec)) with self.cjtx.timeout_lock: - self.cjtx.timeout_lock.wait(common.maker_timeout_sec) + self.cjtx.timeout_lock.wait(maker_timeout_sec) if self.cjtx.all_responded: - debug( - 'timeout thread woken by notify(), makers responded in time') + log.debug( + 'timeout thread woken by notify(), makers responded in time') self.cjtx.all_responded = False else: - debug( - 'timeout thread woken by timeout, makers didnt respond') + log.debug( + 'timeout thread woken by timeout, makers didnt respond') with self.cjtx.timeout_thread_lock: self.cjtx.recover_from_nonrespondants() class CoinJoinerPeer(object): - def __init__(self, msgchan): self.msgchan = msgchan @@ -351,66 +367,69 @@ def on_set_topic(self, newtopic): alert = msg[msg.index(params[1]) + len(params[1]):].strip() except ValueError, IndexError: continue - if min_version < common.JM_VERSION and max_version > common.JM_VERSION: + if min_version < JM_VERSION < max_version: print '=' * 60 print 'JOINMARKET ALERT' print alert print '=' * 60 - common.joinmarket_alert = alert + joinmarket_alert = alert class OrderbookWatch(CoinJoinerPeer): - def __init__(self, msgchan): CoinJoinerPeer.__init__(self, msgchan) self.msgchan.register_orderbookwatch_callbacks(self.on_order_seen, self.on_order_cancel) self.msgchan.register_channel_callbacks( - self.on_welcome, self.on_set_topic, None, self.on_disconnect, - self.on_nick_leave, None) + self.on_welcome, self.on_set_topic, None, self.on_disconnect, + self.on_nick_leave, None) con = sqlite3.connect(":memory:", check_same_thread=False) con.row_factory = sqlite3.Row self.db = con.cursor() self.db.execute( - "CREATE TABLE orderbook(counterparty TEXT, oid INTEGER, ordertype TEXT, " - + "minsize INTEGER, maxsize INTEGER, txfee INTEGER, cjfee TEXT);") + "CREATE TABLE orderbook(counterparty TEXT, oid INTEGER, ordertype TEXT, " + + "minsize INTEGER, maxsize INTEGER, txfee INTEGER, cjfee TEXT);") def on_order_seen(self, counterparty, oid, ordertype, minsize, maxsize, txfee, cjfee): try: if int(oid) < 0 or int(oid) > sys.maxint: - debug("Got invalid order ID: " + oid + " from " + counterparty) + log.debug( + "Got invalid order ID: " + oid + " from " + counterparty) return # delete orders eagerly, so in case a buggy maker sends an invalid offer, # we won't accidentally !fill based on the ghost of its previous message. self.db.execute( - "DELETE FROM orderbook WHERE counterparty=? AND oid=?;", - (counterparty, oid)) + "DELETE FROM orderbook WHERE counterparty=? AND oid=?;", + (counterparty, oid)) # now validate the remaining fields - if int(minsize) < 0 or int(minsize) > 21 * 10**14: - debug("Got invalid minsize: " + minsize + " from " + - counterparty) + if int(minsize) < 0 or int(minsize) > 21 * 10 ** 14: + log.debug("Got invalid minsize: " + minsize + " from " + + counterparty) return - if int(maxsize) < 0 or int(maxsize) > 21 * 10**14: - debug("Got invalid maxsize: " + maxsize + " from " + - counterparty) + if int(maxsize) < 0 or int(maxsize) > 21 * 10 ** 14: + log.debug("Got invalid maxsize: " + maxsize + " from " + + counterparty) return if int(txfee) < 0: - debug("Got invalid txfee: " + txfee + " from " + counterparty) + log.debug( + "Got invalid txfee: " + txfee + " from " + counterparty) return if int(minsize) > int(maxsize): - debug("Got minsize bigger than maxsize: " + minsize + " - " + - maxsize + " from " + counterparty) + log.debug( + "Got minsize bigger than maxsize: " + minsize + " - " + + maxsize + " from " + counterparty) return self.db.execute( - 'INSERT INTO orderbook VALUES(?, ?, ?, ?, ?, ?, ?);', - (counterparty, oid, ordertype, minsize, maxsize, txfee, - str(Decimal(cjfee)))) # any parseable Decimal is a valid cjfee + 'INSERT INTO orderbook VALUES(?, ?, ?, ?, ?, ?, ?);', + (counterparty, oid, ordertype, minsize, maxsize, txfee, + str(Decimal( + cjfee)))) # any parseable Decimal is a valid cjfee except InvalidOperation: - debug("Got invalid cjfee: " + cjfee + " from " + counterparty) + log.debug("Got invalid cjfee: " + cjfee + " from " + counterparty) except: - debug("Error parsing order " + oid + " from " + counterparty) + log.debug("Error parsing order " + oid + " from " + counterparty) def on_order_cancel(self, counterparty, oid): self.db.execute("DELETE FROM orderbook WHERE counterparty=? AND oid=?;", @@ -426,9 +445,8 @@ def on_disconnect(self): self.db.execute('DELETE FROM orderbook;') -#assume this only has one open cj tx at a time +# assume this only has one open cj tx at a time class Taker(OrderbookWatch): - def __init__(self, msgchan): OrderbookWatch.__init__(self, msgchan) msgchan.register_taker_callbacks(self.on_error, self.on_pubkey, @@ -436,15 +454,16 @@ def __init__(self, msgchan): msgchan.cjpeer = self self.cjtx = None self.maker_pks = {} - #TODO have a list of maker's nick we're coinjoining with, so + # TODO have a list of maker's nick we're coinjoining with, so # that some other guy doesnt send you confusing stuff def get_crypto_box_from_nick(self, nick): if nick in self.cjtx.crypto_boxes: - return self.cjtx.crypto_boxes[nick][1] #libsodium encryption object + return self.cjtx.crypto_boxes[nick][ + 1] # libsodium encryption object else: - debug('something wrong, no crypto object, nick=' + nick + - ', message will be dropped') + log.debug('something wrong, no crypto object, nick=' + nick + + ', message will be dropped') return None def start_cj(self, @@ -464,15 +483,15 @@ def start_cj(self, choose_orders_recover, auth_addr) def on_error(self): - pass #TODO implement + pass # TODO implement def on_pubkey(self, nick, maker_pubkey): self.cjtx.start_encryption(nick, maker_pubkey) def on_ioauth(self, nick, utxo_list, cj_pub, change_addr, btc_sig): if not self.cjtx.auth_counterparty(nick, btc_sig, cj_pub): - debug('Authenticated encryption with counterparty: ' + nick + \ - ' not established. TODO: send rejection message') + log.debug('Authenticated encryption with counterparty: ' + nick + \ + ' not established. TODO: send rejection message') return with self.cjtx.timeout_thread_lock: self.cjtx.recv_txio(nick, utxo_list, cj_pub, change_addr) @@ -482,7 +501,7 @@ def on_sig(self, nick, sig): self.cjtx.add_signature(nick, sig) -#this stuff copied and slightly modified from pybitcointools +# this stuff copied and slightly modified from pybitcointools def donation_address(cjtx): reusable_donation_pubkey = '02be838257fbfddabaea03afbb9f16e8529dfe2de921260a5c46036d97b5eacf2a' @@ -492,18 +511,18 @@ def donation_address(cjtx): privkey = cjtx.wallet.get_key_from_addr(donation_utxo_data[1]['address']) tx = btc.mktx(cjtx.utxo_tx, cjtx.outputs - ) #tx without our inputs and outputs - #address = privtoaddr(privkey) - #signing_tx = signature_form(tx, 0, mk_pubkey_script(address), SIGHASH_ALL) + ) # tx without our inputs and outputs + # address = privtoaddr(privkey) + # signing_tx = signature_form(tx, 0, mk_pubkey_script(address), SIGHASH_ALL) msghash = btc.bin_txhash(tx, btc.SIGHASH_ALL) - #generate unpredictable k + # generate unpredictable k global sign_k sign_k = btc.deterministic_generate_k(msghash, privkey) c = btc.sha256(btc.multiply(reusable_donation_pubkey, sign_k)) sender_pubkey = btc.add_pubkeys(reusable_donation_pubkey, btc.multiply( - btc.G, c)) + btc.G, c)) sender_address = btc.pubtoaddr(sender_pubkey, get_p2pk_vbyte()) - debug('sending coins to ' + sender_address) + log.debug('sending coins to ' + sender_address) return sender_address @@ -520,13 +539,13 @@ def sign_donation_tx(tx, i, priv): msghash = btc.bin_txhash(signing_tx, hashcode) z = btc.hash_to_int(msghash) - #k = deterministic_generate_k(msghash, priv) + # k = deterministic_generate_k(msghash, priv) r, y = btc.fast_multiply(btc.G, k) s = btc.inv(k, btc.N) * (z + r * btc.decode_privkey(priv)) % btc.N rawsig = 27 + (y % 2), r, s sig = btc.der_encode_sig(*rawsig) + btc.encode(hashcode, 16, 2) - #sig = ecdsa_tx_sign(signing_tx, priv, hashcode) + # sig = ecdsa_tx_sign(signing_tx, priv, hashcode) txobj = btc.deserialize(tx) txobj["ins"][i]["script"] = btc.serialize_script([sig, pub]) return btc.serialize(txobj) diff --git a/ob-watcher.py b/ob-watcher.py index fe960e2e..cf3df6ac 100644 --- a/ob-watcher.py +++ b/ob-watcher.py @@ -1,17 +1,28 @@ -import BaseHTTPServer, SimpleHTTPServer, threading +from __future__ import absolute_import + +import BaseHTTPServer +import SimpleHTTPServer +import base64 +import io +import json +import sys +import threading +import time import urllib2 -import io, base64, time, sys, os -data_dir = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(data_dir, 'joinmarket')) +from decimal import Decimal +from optparse import OptionParser -import taker -from irc import IRCMessageChannel, random_nick -from common import * -import common +from joinmarket import calc_cj_fee, ordername_list, joinmarket_alert, \ + OrderbookWatch, random_nick, set_nickname, load_program_config, \ + IRCMessageChannel -#https://stackoverflow.com/questions/2801882/generating-a-png-with-matplotlib-when-display-is-undefined +# data_dir = os.path.dirname(os.path.realpath(__file__)) +# sys.path.insert(0, os.path.join(data_dir, 'joinmarket')) + +# https://stackoverflow.com/questions/2801882/generating-a-png-with-matplotlib-when-display-is-undefined try: import matplotlib + matplotlib.use('Agg') import matplotlib.pyplot as plt except ImportError: @@ -32,10 +43,6 @@ def calc_depth_data(db, value): pass -def calc_order_size_data(db): - return ordersizes - - def create_depth_chart(db, cj_amount, args={}): sqlorders = db.execute('SELECT * FROM orderbook;').fetchall() orderfees = sorted([calc_cj_fee(o['ordertype'], o['cjfee'], cj_amount) / 1e8 @@ -51,12 +58,12 @@ def create_depth_chart(db, cj_amount, args={}): orderfees = [float(fee) for fee in orderfees] if orderfees[0] > 0: ratio = orderfees[-1] / orderfees[0] - step = ratio**0.0333 # 1/30 - bins = [orderfees[0] * (step**i) for i in range(30)] + step = ratio ** 0.0333 # 1/30 + bins = [orderfees[0] * (step ** i) for i in range(30)] else: ratio = orderfees[-1] / 1e-8 # single satoshi placeholder - step = ratio**0.0333 # 1/30 - bins = [1e-8 * (step**i) for i in range(30)] + step = ratio ** 0.0333 # 1/30 + bins = [1e-8 * (step ** i) for i in range(30)] bins[0] = orderfees[0] # replace placeholder plt.xscale('log') else: @@ -81,8 +88,8 @@ def create_size_histogram(db, args): scale = args.get("scale") if (scale is not None) and (scale[0] == "log"): ratio = ordersizes[-1] / ordersizes[0] - step = ratio**0.0333 # 1/30 - bins = [ordersizes[0] * (step**i) for i in range(30)] + step = ratio ** 0.0333 # 1/30 + bins = [ordersizes[0] * (step ** i) for i in range(30)] else: bins = 30 plt.hist(ordersizes, bins, histtype='bar', rwidth=0.8) @@ -102,7 +109,7 @@ def get_graph_html(fig): return '' -#callback functions for displaying order data +# callback functions for displaying order data def do_nothing(arg, order, btc_unit, rel_unit): return arg @@ -121,7 +128,8 @@ def cjfee_display(cjfee, order, btc_unit, rel_unit): def satoshi_to_unit(sat, order, btc_unit, rel_unit): power = unit_to_power[btc_unit] - return ("%." + str(power) + "f") % float(Decimal(sat) / Decimal(10**power)) + return ("%." + str(power) + "f") % float( + Decimal(sat) / Decimal(10 ** power)) def order_str(s, order, btc_unit, rel_unit): @@ -139,9 +147,14 @@ def create_orderbook_table(db, btc_unit, rel_unit): ('minsize', satoshi_to_unit), ('maxsize', satoshi_to_unit)) - #somewhat complex sorting to sort by cjfee but with absorders on top - orderby_cmp = lambda x, y: cmp(Decimal(x['cjfee']), Decimal(y['cjfee'])) if x['ordertype'] == y['ordertype'] \ - else cmp(common.ordername_list.index(x['ordertype']), common.ordername_list.index(y['ordertype'])) + # somewhat complex sorting to sort by cjfee but with absorders on top + + def orderby_cmp(x, y): + if x['ordertype'] == y['ordertype']: + return cmp(Decimal(x['cjfee']), Decimal(y['cjfee'])) + return cmp(ordername_list.index(x['ordertype']), + ordername_list.index(y['ordertype'])) + for o in sorted(rows, cmp=orderby_cmp): result += ' \n' for key, displayer in order_keys_display: @@ -152,17 +165,18 @@ def create_orderbook_table(db, btc_unit, rel_unit): def create_table_heading(btc_unit, rel_unit): - col = ' {1}\n' # .format(field,label) tableheading = '\n ' + ''.join( - [ - col.format('ordertype', 'Type'), col.format( - 'counterparty', 'Counterparty'), col.format('oid', 'Order ID'), - col.format('cjfee', 'Fee'), col.format( - 'txfee', 'Miner Fee Contribution / ' + btc_unit), col.format( - 'minsize', 'Minimum Size / ' + btc_unit), col.format( - 'maxsize', 'Maximum Size / ' + btc_unit) - ]) + ' ' + [ + col.format('ordertype', 'Type'), col.format( + 'counterparty', 'Counterparty'), + col.format('oid', 'Order ID'), + col.format('cjfee', 'Fee'), col.format( + 'txfee', 'Miner Fee Contribution / ' + btc_unit), + col.format( + 'minsize', 'Minimum Size / ' + btc_unit), col.format( + 'maxsize', 'Maximum Size / ' + btc_unit) + ]) + ' ' return tableheading @@ -175,21 +189,20 @@ def create_choose_units_form(selected_btc, selected_rel): ''.join(('' for u in sorted_rel_units)) + '') choose_units_form = choose_units_form.replace( - '
\n') @@ -256,13 +270,13 @@ def do_GET(self): 'MAINBODY': create_size_histogram(self.taker.db, args) } elif self.path.startswith('/depth'): - #if self.path[6] == '?': + # if self.path[6] == '?': # quantity = - cj_amounts = [10**cja for cja in range(4, 12, 1)] - mainbody = [create_depth_chart(self.taker.db, cja, args) \ - for cja in cj_amounts] + \ - ["
linear" if args.get("scale") \ - else "
log scale"] + cj_amounts = [10 ** cja for cja in range(4, 12, 1)] + mainbody = [create_depth_chart(self.taker.db, cja, args) \ + for cja in cj_amounts] + \ + ["
linear" if args.get("scale") \ + else "
log scale"] replacements = { 'PAGETITLE': 'JoinMarket Browser Interface', 'MAINHEADING': 'Depth Chart', @@ -304,7 +318,6 @@ def do_POST(self): class HTTPDThread(threading.Thread): - def __init__(self, taker, hostport): threading.Thread.__init__(self) self.daemon = True @@ -312,38 +325,33 @@ def __init__(self, taker, hostport): self.hostport = hostport def run(self): - #hostport = ('localhost', 62601) + # hostport = ('localhost', 62601) httpd = BaseHTTPServer.HTTPServer(self.hostport, OrderbookPageRequestHeader) httpd.taker = self.taker print('\nstarted http server, visit http://{0}:{1}/\n'.format( - *self.hostport)) + *self.hostport)) httpd.serve_forever() -class GUITaker(taker.OrderbookWatch): - +class GUITaker(OrderbookWatch): def __init__(self, msgchan, hostport): self.hostport = hostport super(GUITaker, self).__init__(msgchan) def on_welcome(self): - taker.OrderbookWatch.on_welcome(self) + OrderbookWatch.on_welcome(self) HTTPDThread(self, self.hostport).start() def main(): - import bitcoin as btc - import common - import binascii, os - from optparse import OptionParser - - common.nickname = random_nick() #watcher' +binascii.hexlify(os.urandom(4)) - common.load_program_config() + nickname = random_nick() # watcher' +binascii.hexlify(os.urandom(4)) + set_nickname(nickname) + load_program_config() parser = OptionParser( - usage='usage: %prog [options]', - description='Runs a webservice which shows the orderbook.') + usage='usage: %prog [options]', + description='Runs a webservice which shows the orderbook.') parser.add_option('-H', '--host', action='store', @@ -362,7 +370,7 @@ def main(): hostport = (options.host, options.port) - irc = IRCMessageChannel(common.nickname) + irc = IRCMessageChannel(nickname) taker = GUITaker(irc, hostport) print('starting irc') diff --git a/patientsendpayment.py b/patientsendpayment.py index 4dec9b2f..4d39f5a1 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -1,19 +1,22 @@ -from optparse import OptionParser +from __future__ import absolute_import + +import sys +import threading +import time from datetime import timedelta -import threading, time, binascii, os, sys -data_dir = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(data_dir, 'joinmarket')) +from optparse import OptionParser +# data_dir = os.path.dirname(os.path.realpath(__file__)) +# sys.path.insert(0, os.path.join(data_dir, 'joinmarket')) -from common import * -import common -import taker -import maker -from irc import IRCMessageChannel, random_nick -import bitcoin as btc +from joinmarket import choose_orders, weighted_order_choose, Maker, Taker, \ + get_log, load_program_config, validate_address, Wallet, BitcoinCoreWallet, \ + bc_interface, random_nick, set_nickname, IRCMessageChannel, \ + debug_dump_object +log = get_log() -class TakerThread(threading.Thread): +class TakerThread(threading.Thread): def __init__(self, tmaker): threading.Thread.__init__(self) self.daemon = True @@ -24,36 +27,35 @@ def finishcallback(self, coinjointx): self.tmaker.msgchan.shutdown() def run(self): - #TODO this thread doesnt wake up for what could be hours + # TODO this thread doesnt wake up for what could be hours # need a loop that periodically checks self.finished - #TODO another issue is, what if the bot has run out of utxos and + # TODO another issue is, what if the bot has run out of utxos and # needs to wait for some tx to confirm before it can trade # presumably it needs to wait here until the tx confirms time.sleep(self.tmaker.waittime) if self.finished: return print 'giving up waiting' - #cancel the remaining order + # cancel the remaining order self.tmaker.modify_orders([0], []) orders, total_cj_fee = choose_orders(self.tmaker.db, self.tmaker.amount, self.tmaker.makercount, weighted_order_choose) print 'chosen orders to fill ' + str(orders) + ' totalcjfee=' + str( - total_cj_fee) + total_cj_fee) total_amount = self.tmaker.amount + total_cj_fee + self.tmaker.txfee print 'total amount spent = ' + str(total_amount) utxos = self.tmaker.wallet.select_utxos(self.tmaker.mixdepth, total_amount) self.tmaker.start_cj( - self.tmaker.wallet, self.tmaker.amount, orders, utxos, - self.tmaker.destaddr, - self.tmaker.wallet.get_change_addr(self.tmaker.mixdepth), - self.tmaker.txfee, self.finishcallback) - + self.tmaker.wallet, self.tmaker.amount, orders, utxos, + self.tmaker.destaddr, + self.tmaker.wallet.get_change_addr(self.tmaker.mixdepth), + self.tmaker.txfee, self.finishcallback) -class PatientSendPayment(maker.Maker, taker.Taker): +class PatientSendPayment(Maker, Taker): def __init__(self, msgchan, wallet, destaddr, amount, makercount, txfee, cjfee, waittime, mixdepth): self.destaddr = destaddr @@ -63,23 +65,23 @@ def __init__(self, msgchan, wallet, destaddr, amount, makercount, txfee, self.cjfee = cjfee self.waittime = waittime self.mixdepth = mixdepth - maker.Maker.__init__(self, msgchan, wallet) - taker.Taker.__init__(self, msgchan) + Maker.__init__(self, msgchan, wallet) + Taker.__init__(self, msgchan) def get_crypto_box_from_nick(self, nick): if self.cjtx: - return taker.Taker.get_crypto_box_from_nick(self, nick) + return Taker.get_crypto_box_from_nick(self, nick) else: - return maker.Maker.get_crypto_box_from_nick(self, nick) + return Maker.get_crypto_box_from_nick(self, nick) def on_welcome(self): - maker.Maker.on_welcome(self) - taker.Taker.on_welcome(self) + Maker.on_welcome(self) + Taker.on_welcome(self) self.takerthread = TakerThread(self) self.takerthread.start() def create_my_orders(self): - #choose an absolute fee order to discourage people from + # choose an absolute fee order to discourage people from # mixing smaller amounts order = {'oid': 0, 'ordertype': 'absorder', @@ -90,8 +92,8 @@ def create_my_orders(self): return [order] def oid_to_order(self, cjorder, oid, amount): - #TODO race condition (kinda) - #if an order arrives and before it finishes another order arrives + # TODO race condition (kinda) + # if an order arrives and before it finishes another order arrives # its possible this bot will end up paying to the destaddr more than it # intended utxos = self.wallet.select_utxos(self.mixdepth, amount) @@ -114,7 +116,7 @@ def on_tx_unconfirmed(self, cjorder, balance, removed_utxos): 'cjfee': self.cjfee} return ([], [order]) else: - debug('not enough money left, have to wait until tx confirms') + log.debug('not enough money left, have to wait until tx confirms') return ([0], []) def on_tx_confirmed(self, cjorder, confirmations, txid, balance): @@ -132,12 +134,12 @@ def on_tx_confirmed(self, cjorder, confirmations, txid, balance): def main(): parser = OptionParser( - usage= - 'usage: %prog [options] [wallet file / fromaccount] [amount] [destaddr]', - description='Sends a payment from your wallet to an given address' + - ' using coinjoin. First acts as a maker, announcing an order and ' + - 'waiting for someone to fill it. After a set period of time, gives' + - ' up waiting and acts as a taker and coinjoins any remaining coins') + usage= + 'usage: %prog [options] [wallet file / fromaccount] [amount] [destaddr]', + description='Sends a payment from your wallet to an given address' + + ' using coinjoin. First acts as a maker, announcing an order and ' + + 'waiting for someone to fill it. After a set period of time, gives' + + ' up waiting and acts as a taker and coinjoins any remaining coins') parser.add_option('-f', '--txfee', action='store', @@ -146,31 +148,31 @@ def main(): default=10000, help='miner fee contribution, in satoshis, default=10000') parser.add_option( - '-N', - '--makercount', - action='store', - type='int', - dest='makercount', - help= - 'how many makers to coinjoin with when taking liquidity, default=2', - default=2) + '-N', + '--makercount', + action='store', + type='int', + dest='makercount', + help= + 'how many makers to coinjoin with when taking liquidity, default=2', + default=2) parser.add_option( - '-w', - '--wait-time', - action='store', - type='float', - dest='waittime', - help='wait time in hours as a maker before becoming a taker, default=8', - default=8) + '-w', + '--wait-time', + action='store', + type='float', + dest='waittime', + help='wait time in hours as a maker before becoming a taker, default=8', + default=8) parser.add_option( - '-c', - '--cjfee', - action='store', - type='int', - dest='cjfee', - help= - 'coinjoin fee asked for when being a maker, in satoshis per order filled, default=50000', - default=50000) + '-c', + '--cjfee', + action='store', + type='int', + dest='cjfee', + help= + 'coinjoin fee asked for when being a maker, in satoshis per order filled, default=50000', + default=50000) parser.add_option('-m', '--mixdepth', action='store', @@ -179,13 +181,13 @@ def main(): help='mixing depth to spend from, default=0', default=0) parser.add_option( - '--rpcwallet', - action='store_true', - dest='userpcwallet', - default=False, - help= - 'Use the Bitcoin Core wallet through json rpc, instead of the internal joinmarket ' - + 'wallet. Requires blockchain_source=json-rpc') + '--rpcwallet', + action='store_true', + dest='userpcwallet', + default=False, + help= + 'Use the Bitcoin Core wallet through json rpc, instead of the internal joinmarket ' + + 'wallet. Requires blockchain_source=json-rpc') (options, args) = parser.parse_args() if len(args) < 3: @@ -211,27 +213,29 @@ def main(): else: print 'not implemented yet' sys.exit(0) - wallet = BitcoinCoreWallet(fromaccount=wallet_name) - common.bc_interface.sync_wallet(wallet) + wallet = BitcoinCoreWallet(fromaccount=wallet_name) + bc_interface.sync_wallet(wallet) available_balance = wallet.get_balance_by_mixdepth()[options.mixdepth] if available_balance < amount: print 'not enough money at mixdepth=%d, exiting' % (options.mixdepth) return - common.nickname = random_nick() - debug('Running patient sender of a payment') + nickname = random_nick() + set_nickname(nickname) + log.debug('Running patient sender of a payment') - irc = IRCMessageChannel(common.nickname) + irc = IRCMessageChannel(nickname) bot = PatientSendPayment(irc, wallet, destaddr, amount, options.makercount, options.txfee, options.cjfee, waittime, options.mixdepth) try: irc.run() except: - debug('CRASHING, DUMPING EVERYTHING') + log.debug('CRASHING, DUMPING EVERYTHING') debug_dump_object(wallet, ['addr_cache', 'keys', 'seed']) - debug_dump_object(taker) + # todo: looks wrong. dump on the class object? + # debug_dump_object(taker) import traceback traceback.print_exc() diff --git a/sendpayment.py b/sendpayment.py index e67f6781..2ea54dcf 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -1,15 +1,20 @@ #! /usr/bin/env python +from __future__ import absolute_import +import sys +import threading from optparse import OptionParser -import threading, pprint, sys, os -data_dir = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(data_dir, 'joinmarket')) -from common import * -import common -import taker as takermodule -from irc import IRCMessageChannel, random_nick -import bitcoin as btc +# data_dir = os.path.dirname(os.path.realpath(__file__)) +# sys.path.insert(0, os.path.join(data_dir, 'joinmarket')) +import time + +from joinmarket import choose_sweep_orders, get_log, choose_orders, Taker, \ + load_program_config, validate_address, pick_order, cheapest_order_choose, \ + weighted_order_choose, random_nick, set_nickname, Wallet, BitcoinCoreWallet, \ + bc_interface, IRCMessageChannel, debug_dump_object + +log = get_log() def check_high_fee(total_fee_pc): @@ -24,10 +29,9 @@ def check_high_fee(total_fee_pc): print '\n'.join(['=' * 60] * 3) -#thread which does the buy-side algorithm +# thread which does the buy-side algorithm # chooses which coinjoins to initiate and when class PaymentThread(threading.Thread): - def __init__(self, taker): threading.Thread.__init__(self) self.daemon = True @@ -36,7 +40,7 @@ def __init__(self, taker): def create_tx(self): crow = self.taker.db.execute( - 'SELECT COUNT(DISTINCT counterparty) FROM orderbook;').fetchone() + 'SELECT COUNT(DISTINCT counterparty) FROM orderbook;').fetchone() counterparty_count = crow['COUNT(DISTINCT counterparty)'] counterparty_count -= len(self.ignored_makers) if counterparty_count < self.taker.makercount: @@ -54,14 +58,14 @@ def create_tx(self): self.taker.mixdepth] total_value = sum([va['value'] for va in utxos.values()]) orders, cjamount = choose_sweep_orders( - self.taker.db, total_value, self.taker.txfee, - self.taker.makercount, self.taker.chooseOrdersFunc, - self.ignored_makers) + self.taker.db, total_value, self.taker.txfee, + self.taker.makercount, self.taker.chooseOrdersFunc, + self.ignored_makers) if not self.taker.answeryes: total_cj_fee = total_value - cjamount - self.taker.txfee - debug('total cj fee = ' + str(total_cj_fee)) + log.debug('total cj fee = ' + str(total_cj_fee)) total_fee_pc = 1.0 * total_cj_fee / cjamount - debug('total coinjoin fee = ' + str(float('%.3g' % ( + log.debug('total coinjoin fee = ' + str(float('%.3g' % ( 100.0 * total_fee_pc))) + '%') check_high_fee(total_fee_pc) if raw_input('send with these orders? (y/n):')[0] != 'y': @@ -69,9 +73,10 @@ def create_tx(self): return else: orders, total_cj_fee = self.sendpayment_choose_orders( - self.taker.amount, self.taker.makercount) + self.taker.amount, self.taker.makercount) if not orders: - debug('ERROR not enough liquidity in the orderbook, exiting') + log.debug( + 'ERROR not enough liquidity in the orderbook, exiting') return total_amount = self.taker.amount + total_cj_fee + self.taker.txfee print 'total amount spent = ' + str(total_amount) @@ -88,11 +93,12 @@ def create_tx(self): def finishcallback(self, coinjointx): if coinjointx.all_responded: coinjointx.self_sign_and_push() - debug('created fully signed tx, ending') + log.debug('created fully signed tx, ending') self.taker.msgchan.shutdown() return self.ignored_makers += coinjointx.nonrespondants - debug('recreating the tx, ignored_makers=' + str(self.ignored_makers)) + log.debug( + 'recreating the tx, ignored_makers=' + str(self.ignored_makers)) self.create_tx() def sendpayment_choose_orders(self, @@ -102,23 +108,24 @@ def sendpayment_choose_orders(self, active_nicks=[]): self.ignored_makers += nonrespondants orders, total_cj_fee = choose_orders( - self.taker.db, cj_amount, makercount, self.taker.chooseOrdersFunc, - self.ignored_makers + active_nicks) + self.taker.db, cj_amount, makercount, + self.taker.chooseOrdersFunc, + self.ignored_makers + active_nicks) if not orders: return None, 0 print 'chosen orders to fill ' + str(orders) + ' totalcjfee=' + str( - total_cj_fee) + total_cj_fee) if not self.taker.answeryes: if len(self.ignored_makers) > 0: noun = 'total' else: noun = 'additional' total_fee_pc = 1.0 * total_cj_fee / cj_amount - debug(noun + ' coinjoin fee = ' + str(float('%.3g' % ( + log.debug(noun + ' coinjoin fee = ' + str(float('%.3g' % ( 100.0 * total_fee_pc))) + '%') check_high_fee(total_fee_pc) if raw_input('send with these orders? (y/n):')[0] != 'y': - debug('ending') + log.debug('ending') self.taker.msgchan.shutdown() return None, -1 return orders, total_cj_fee @@ -129,11 +136,10 @@ def run(self): self.create_tx() -class SendPayment(takermodule.Taker): - +class SendPayment(Taker): def __init__(self, msgchan, wallet, destaddr, amount, makercount, txfee, waittime, mixdepth, answeryes, chooseOrdersFunc): - takermodule.Taker.__init__(self, msgchan) + Taker.__init__(self, msgchan) self.wallet = wallet self.destaddr = destaddr self.amount = amount @@ -145,19 +151,19 @@ def __init__(self, msgchan, wallet, destaddr, amount, makercount, txfee, self.chooseOrdersFunc = chooseOrdersFunc def on_welcome(self): - takermodule.Taker.on_welcome(self) + Taker.on_welcome(self) PaymentThread(self).start() def main(): parser = OptionParser( - usage= - 'usage: %prog [options] [wallet file / fromaccount] [amount] [destaddr]', - description='Sends a single payment from a given mixing depth of your ' - + - 'wallet to an given address using coinjoin and then switches off. Also sends from bitcoinqt. ' - + - 'Setting amount to zero will do a sweep, where the entire mix depth is emptied') + usage= + 'usage: %prog [options] [wallet file / fromaccount] [amount] [destaddr]', + description='Sends a single payment from a given mixing depth of your ' + + + 'wallet to an given address using coinjoin and then switches off. Also sends from bitcoinqt. ' + + + 'Setting amount to zero will do a sweep, where the entire mix depth is emptied') parser.add_option('-f', '--txfee', action='store', @@ -166,13 +172,13 @@ def main(): default=10000, help='total miner fee in satoshis, default=10000') parser.add_option( - '-w', - '--wait-time', - action='store', - type='float', - dest='waittime', - help='wait time in seconds to allow orders to arrive, default=5', - default=5) + '-w', + '--wait-time', + action='store', + type='float', + dest='waittime', + help='wait time in seconds to allow orders to arrive, default=5', + default=5) parser.add_option('-N', '--makercount', action='store', @@ -181,20 +187,20 @@ def main(): help='how many makers to coinjoin with, default=2', default=2) parser.add_option( - '-C', - '--choose-cheapest', - action='store_true', - dest='choosecheapest', - default=False, - help='override weightened offers picking and choose cheapest') + '-C', + '--choose-cheapest', + action='store_true', + dest='choosecheapest', + default=False, + help='override weightened offers picking and choose cheapest') parser.add_option( - '-P', - '--pick-orders', - action='store_true', - dest='pickorders', - default=False, - help= - 'manually pick which orders to take. doesn\'t work while sweeping.') + '-P', + '--pick-orders', + action='store_true', + dest='pickorders', + default=False, + help= + 'manually pick which orders to take. doesn\'t work while sweeping.') parser.add_option('-m', '--mixdepth', action='store', @@ -208,13 +214,13 @@ def main(): default=False, help='answer yes to everything') parser.add_option( - '--rpcwallet', - action='store_true', - dest='userpcwallet', - default=False, - help= - 'Use the Bitcoin Core wallet through json rpc, instead of the internal joinmarket ' - + 'wallet. Requires blockchain_source=json-rpc') + '--rpcwallet', + action='store_true', + dest='userpcwallet', + default=False, + help= + 'Use the Bitcoin Core wallet through json rpc, instead of the internal joinmarket ' + + 'wallet. Requires blockchain_source=json-rpc') (options, args) = parser.parse_args() if len(args) < 3: @@ -231,35 +237,36 @@ def main(): return chooseOrdersFunc = None - if options.pickorders and amount != 0: #cant use for sweeping + if options.pickorders and amount != 0: # cant use for sweeping chooseOrdersFunc = pick_order elif options.choosecheapest: chooseOrdersFunc = cheapest_order_choose - else: #choose randomly (weighted) + else: # choose randomly (weighted) chooseOrdersFunc = weighted_order_choose - common.nickname = random_nick() - debug('starting sendpayment') + nickname = random_nick() + set_nickname(nickname) + log.debug('starting sendpayment') if not options.userpcwallet: wallet = Wallet(wallet_name, options.mixdepth + 1) else: wallet = BitcoinCoreWallet(fromaccount=wallet_name) - common.bc_interface.sync_wallet(wallet) + bc_interface.sync_wallet(wallet) - irc = IRCMessageChannel(common.nickname) + irc = IRCMessageChannel(nickname) taker = SendPayment(irc, wallet, destaddr, amount, options.makercount, options.txfee, options.waittime, options.mixdepth, options.answeryes, chooseOrdersFunc) try: - debug('starting irc') + log.debug('starting irc') irc.run() except: - debug('CRASHING, DUMPING EVERYTHING') + log.debug('CRASHING, DUMPING EVERYTHING') debug_dump_object(wallet, ['addr_cache', 'keys', 'wallet_name', 'seed']) debug_dump_object(taker) import traceback - debug(traceback.format_exc()) + log.debug(traceback.format_exc()) if __name__ == "__main__": diff --git a/tumbler.py b/tumbler.py index ff2b287e..84d2328b 100644 --- a/tumbler.py +++ b/tumbler.py @@ -1,42 +1,50 @@ -import datetime, threading, binascii, sys, os, copy -data_dir = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(data_dir, 'joinmarket')) +from __future__ import absolute_import -import taker as takermodule -import common -from common import * -from irc import IRCMessageChannel, random_nick +import copy +import sys +import threading +# data_dir = os.path.dirname(os.path.realpath(__file__)) +# sys.path.insert(0, os.path.join(data_dir, 'joinmarket')) + +import time from optparse import OptionParser from pprint import pprint +from joinmarket import rand_norm_array, rand_pow_array, rand_exp_array, get_log, \ + bc_interface, choose_orders, weighted_order_choose, choose_sweep_orders, \ + set_debug_silence, validate_address, Taker, load_program_config, Wallet, \ + random_nick, set_nickname, IRCMessageChannel, debug_dump_object + +log = get_log() + def lower_bounded_int(thelist, lowerbound): return [int(l) if int(l) >= lowerbound else lowerbound for l in thelist] def generate_tumbler_tx(destaddrs, options): - #sends the coins up through a few mixing depths - #send to the destination addresses from different mixing depths + # sends the coins up through a few mixing depths + # send to the destination addresses from different mixing depths - #simple algo, move coins completely from one mixing depth to the next + # simple algo, move coins completely from one mixing depth to the next # until you get to the end, then send to destaddrs - #txcounts for going completely from one mixdepth to the next + # txcounts for going completely from one mixdepth to the next # follows a normal distribution txcounts = rand_norm_array(options.txcountparams[0], options.txcountparams[1], options.mixdepthcount) txcounts = lower_bounded_int(txcounts, options.mintxcount) tx_list = [] for m, txcount in enumerate(txcounts): - #assume that the sizes of outputs will follow a power law + # assume that the sizes of outputs will follow a power law amount_fractions = rand_pow_array(options.amountpower, txcount) amount_fractions = [1.0 - x for x in amount_fractions] amount_fractions = [x / sum(amount_fractions) for x in amount_fractions] - #transaction times are uncorrelated - #time between events in a poisson process followed exp + # transaction times are uncorrelated + # time between events in a poisson process followed exp waits = rand_exp_array(options.timelambda, txcount) - #number of makers to use follows a normal distribution + # number of makers to use follows a normal distribution makercounts = rand_norm_array(options.makercountrange[0], options.makercountrange[1], txcount) makercounts = lower_bounded_int(makercounts, options.minmakercount) @@ -64,7 +72,7 @@ def generate_tumbler_tx(destaddrs, options): tx['destination'] = external_dest_addrs[mix_offset] break if mix_offset == 0: - #setting last mixdepth to send all to dest + # setting last mixdepth to send all to dest tx_list_remove = [] for tx in tx_list: if tx['srcmixdepth'] == srcmix: @@ -76,10 +84,9 @@ def generate_tumbler_tx(destaddrs, options): return tx_list -#thread which does the buy-side algorithm +# thread which does the buy-side algorithm # chooses which coinjoins to initiate and when class TumblerThread(threading.Thread): - def __init__(self, taker): threading.Thread.__init__(self) self.daemon = True @@ -88,8 +95,8 @@ def __init__(self, taker): self.sweeping = False def unconfirm_callback(self, txd, txid): - debug('that was %d tx out of %d' % - (self.current_tx + 1, len(self.taker.tx_list))) + log.debug('that was %d tx out of %d' % + (self.current_tx + 1, len(self.taker.tx_list))) def confirm_callback(self, txd, txid, confirmations): self.taker.wallet.add_new_utxos(txd, txid) @@ -99,15 +106,15 @@ def confirm_callback(self, txd, txid, confirmations): def finishcallback(self, coinjointx): if coinjointx.all_responded: - common.bc_interface.add_tx_notify( - coinjointx.latest_tx, self.unconfirm_callback, - self.confirm_callback, coinjointx.my_cj_addr) + bc_interface.add_tx_notify( + coinjointx.latest_tx, self.unconfirm_callback, + self.confirm_callback, coinjointx.my_cj_addr) self.taker.wallet.remove_old_utxos(coinjointx.latest_tx) coinjointx.self_sign_and_push() else: self.ignored_makers += coinjointx.nonrespondants - debug('recreating the tx, ignored_makers=' + str( - self.ignored_makers)) + log.debug('recreating the tx, ignored_makers=' + str( + self.ignored_makers)) self.create_tx() def tumbler_choose_orders(self, @@ -118,28 +125,28 @@ def tumbler_choose_orders(self, self.ignored_makers += nonrespondants while True: orders, total_cj_fee = choose_orders( - self.taker.db, cj_amount, makercount, weighted_order_choose, - self.ignored_makers + active_nicks) + self.taker.db, cj_amount, makercount, weighted_order_choose, + self.ignored_makers + active_nicks) abs_cj_fee = 1.0 * total_cj_fee / makercount rel_cj_fee = abs_cj_fee / cj_amount - debug('rel/abs average fee = ' + str(rel_cj_fee) + ' / ' + str( - abs_cj_fee)) + log.debug('rel/abs average fee = ' + str(rel_cj_fee) + ' / ' + str( + abs_cj_fee)) if rel_cj_fee > self.taker.options.maxcjfee[ - 0] and abs_cj_fee > self.taker.options.maxcjfee[1]: - debug('cj fee higher than maxcjfee, waiting ' + str( - self.taker.options.liquiditywait) + ' seconds') + 0] and abs_cj_fee > self.taker.options.maxcjfee[1]: + log.debug('cj fee higher than maxcjfee, waiting ' + str( + self.taker.options.liquiditywait) + ' seconds') time.sleep(self.taker.options.liquiditywait) continue if orders == None: - debug('waiting for liquidity ' + str( - self.taker.options.liquiditywait) + - 'secs, hopefully more orders should come in') + log.debug('waiting for liquidity ' + str( + self.taker.options.liquiditywait) + + 'secs, hopefully more orders should come in') time.sleep(self.taker.options.liquiditywait) continue break - debug('chosen orders to fill ' + str(orders) + ' totalcjfee=' + str( - total_cj_fee)) + log.debug('chosen orders to fill ' + str(orders) + ' totalcjfee=' + str( + total_cj_fee)) return orders, total_cj_fee def create_tx(self): @@ -149,30 +156,31 @@ def create_tx(self): change_addr = None choose_orders_recover = None if self.sweep: - debug('sweeping') + log.debug('sweeping') utxos = self.taker.wallet.get_utxos_by_mixdepth()[self.tx[ 'srcmixdepth']] total_value = sum([addrval['value'] for addrval in utxos.values()]) while True: orders, cj_amount = choose_sweep_orders( - self.taker.db, total_value, self.taker.options.txfee, - self.tx['makercount'], weighted_order_choose, - self.ignored_makers) + self.taker.db, total_value, self.taker.options.txfee, + self.tx['makercount'], weighted_order_choose, + self.ignored_makers) if orders == None: - debug('waiting for liquidity ' + str( - self.taker.options.liquiditywait) + - 'secs, hopefully more orders should come in') + log.debug('waiting for liquidity ' + str( + self.taker.options.liquiditywait) + + 'secs, hopefully more orders should come in') time.sleep(self.taker.options.liquiditywait) continue abs_cj_fee = 1.0 * ( total_value - cj_amount) / self.tx['makercount'] rel_cj_fee = abs_cj_fee / cj_amount - debug('rel/abs average fee = ' + str(rel_cj_fee) + ' / ' + str( - abs_cj_fee)) + log.debug( + 'rel/abs average fee = ' + str(rel_cj_fee) + ' / ' + str( + abs_cj_fee)) if rel_cj_fee > self.taker.options.maxcjfee[ - 0] and abs_cj_fee > self.taker.options.maxcjfee[1]: - debug('cj fee higher than maxcjfee, waiting ' + str( - self.taker.options.liquiditywait) + ' seconds') + 0] and abs_cj_fee > self.taker.options.maxcjfee[1]: + log.debug('cj fee higher than maxcjfee, waiting ' + str( + self.taker.options.liquiditywait) + ' seconds') time.sleep(self.taker.options.liquiditywait) continue break @@ -184,15 +192,15 @@ def create_tx(self): else: cj_amount = int(self.tx['amount_fraction'] * self.balance) if cj_amount < self.taker.options.mincjamount: - debug('cj amount too low, bringing up') + log.debug('cj amount too low, bringing up') cj_amount = self.taker.options.mincjamount change_addr = self.taker.wallet.get_change_addr(self.tx[ - 'srcmixdepth']) - debug('coinjoining ' + str(cj_amount) + ' satoshi') + 'srcmixdepth']) + log.debug('coinjoining ' + str(cj_amount) + ' satoshi') orders, total_cj_fee = self.tumbler_choose_orders( - cj_amount, self.tx['makercount']) + cj_amount, self.tx['makercount']) total_amount = cj_amount + total_cj_fee + self.taker.options.txfee - debug('total amount spent = ' + str(total_amount)) + log.debug('total amount spent = ' + str(total_amount)) utxos = self.taker.wallet.select_utxos(self.tx['srcmixdepth'], total_amount) choose_orders_recover = self.tumbler_choose_orders @@ -207,14 +215,14 @@ def init_tx(self, tx, balance, sweep): if tx['destination'] == 'internal': destaddr = self.taker.wallet.get_receive_addr(tx['srcmixdepth'] + 1) elif tx['destination'] == 'addrask': - common.debug_silence = True + set_debug_silence(True) while True: destaddr = raw_input('insert new address: ') addr_valid, errormsg = validate_address(destaddr) if addr_valid: break print 'Address ' + destaddr + ' invalid. ' + errormsg + ' try again' - common.debug_silence = False + set_debug_silence(False) else: destaddr = tx['destination'] self.sweep = sweep @@ -225,30 +233,31 @@ def init_tx(self, tx, balance, sweep): self.lockcond.acquire() self.lockcond.wait() self.lockcond.release() - debug('tx confirmed, waiting for ' + str(tx['wait']) + ' minutes') + log.debug('tx confirmed, waiting for ' + str(tx['wait']) + ' minutes') time.sleep(tx['wait'] * 60) - debug('woken') + log.debug('woken') def run(self): - debug('waiting for all orders to certainly arrive') + log.debug('waiting for all orders to certainly arrive') time.sleep(self.taker.options.waittime) sqlorders = self.taker.db.execute( - 'SELECT cjfee, ordertype FROM orderbook;').fetchall() + 'SELECT cjfee, ordertype FROM orderbook;').fetchall() orders = [o['cjfee'] for o in sqlorders if o['ordertype'] == 'relorder'] orders = sorted(orders) if len(orders) == 0: - debug( - 'There are no orders at all in the orderbook! Is the bot connecting to the right server?') + log.debug( + 'There are no orders at all in the orderbook! Is the bot connecting to the right server?') return relorder_fee = float(orders[0]) - debug('relorder fee = ' + str(relorder_fee)) + log.debug('relorder fee = ' + str(relorder_fee)) maker_count = sum([tx['makercount'] for tx in self.taker.tx_list]) - debug('uses ' + str(maker_count) + ' makers, at ' + str( - relorder_fee * 100) + '% per maker, estimated total cost ' + str( + log.debug('uses ' + str(maker_count) + ' makers, at ' + str( + relorder_fee * 100) + '% per maker, estimated total cost ' + str( round( - (1 - (1 - relorder_fee)**maker_count) * 100, 3)) + '%') - debug('starting') + (1 - (1 - relorder_fee) ** maker_count) * 100, + 3)) + '%') + log.debug('starting') self.lockcond = threading.Condition() self.balance_by_mixdepth = {} @@ -256,7 +265,7 @@ def run(self): if tx['srcmixdepth'] not in self.balance_by_mixdepth: self.balance_by_mixdepth[tx[ 'srcmixdepth']] = self.taker.wallet.get_balance_by_mixdepth( - )[tx['srcmixdepth']] + )[tx['srcmixdepth']] sweep = True for later_tx in self.taker.tx_list[i + 1:]: if later_tx['srcmixdepth'] == tx['srcmixdepth']: @@ -264,7 +273,7 @@ def run(self): self.current_tx = i self.init_tx(tx, self.balance_by_mixdepth[tx['srcmixdepth']], sweep) - debug('total finished') + log.debug('total finished') self.taker.msgchan.shutdown() ''' crow = self.taker.db.execute('SELECT COUNT(DISTINCT counterparty) FROM orderbook;').fetchone() @@ -276,17 +285,16 @@ def run(self): ''' -class Tumbler(takermodule.Taker): - +class Tumbler(Taker): def __init__(self, msgchan, wallet, tx_list, options): - takermodule.Taker.__init__(self, msgchan) + Taker.__init__(self, msgchan) self.wallet = wallet self.tx_list = tx_list self.options = options self.tumbler_thread = None def on_welcome(self): - takermodule.Taker.on_welcome(self) + Taker.on_welcome(self) if not self.tumbler_thread: self.tumbler_thread = TumblerThread(self) self.tumbler_thread.start() @@ -294,25 +302,25 @@ def on_welcome(self): def main(): parser = OptionParser( - usage='usage: %prog [options] [wallet file] [destaddr(s)...]', - description= - 'Sends bitcoins to many different addresses using coinjoin in' - ' an attempt to break the link between them. Sending to multiple ' - ' addresses is highly recommended for privacy. This tumbler can' - ' be configured to ask for more address mid-run, giving the user' - ' a chance to click `Generate New Deposit Address` on whatever service' - ' they are using.') + usage='usage: %prog [options] [wallet file] [destaddr(s)...]', + description= + 'Sends bitcoins to many different addresses using coinjoin in' + ' an attempt to break the link between them. Sending to multiple ' + ' addresses is highly recommended for privacy. This tumbler can' + ' be configured to ask for more address mid-run, giving the user' + ' a chance to click `Generate New Deposit Address` on whatever service' + ' they are using.') parser.add_option( - '-m', - '--mixdepthsource', - type='int', - dest='mixdepthsrc', - help= - 'Mixing depth to spend from. Useful if a previous tumbler run prematurely ended with ' - + - 'coins being left in higher mixing levels, this option can be used to resume without needing' - + ' to send to another address. default=0', - default=0) + '-m', + '--mixdepthsource', + type='int', + dest='mixdepthsrc', + help= + 'Mixing depth to spend from. Useful if a previous tumbler run prematurely ended with ' + + + 'coins being left in higher mixing levels, this option can be used to resume without needing' + + ' to send to another address. default=0', + default=0) parser.add_option('-f', '--txfee', type='int', @@ -320,43 +328,43 @@ def main(): default=10000, help='total miner fee in satoshis, default=10000') parser.add_option( - '-a', - '--addrcount', - type='int', - dest='addrcount', - default=3, - help= - 'How many destination addresses in total should be used. If not enough are given' - ' as command line arguments, the script will ask for more. This parameter is required' - ' to stop amount correlation. default=3') + '-a', + '--addrcount', + type='int', + dest='addrcount', + default=3, + help= + 'How many destination addresses in total should be used. If not enough are given' + ' as command line arguments, the script will ask for more. This parameter is required' + ' to stop amount correlation. default=3') parser.add_option( - '-x', - '--maxcjfee', - type='float', - dest='maxcjfee', - nargs=2, - default=(0.01, 10000), - help='maximum coinjoin fee and bitcoin value the tumbler is ' - 'willing to pay to a single market maker. Both values need to be exceeded, so if ' - 'the fee is 30% but only 500satoshi is paid the tx will go ahead. default=0.01, 10000 (1%, 10000satoshi)') + '-x', + '--maxcjfee', + type='float', + dest='maxcjfee', + nargs=2, + default=(0.01, 10000), + help='maximum coinjoin fee and bitcoin value the tumbler is ' + 'willing to pay to a single market maker. Both values need to be exceeded, so if ' + 'the fee is 30% but only 500satoshi is paid the tx will go ahead. default=0.01, 10000 (1%, 10000satoshi)') parser.add_option( - '-N', - '--makercountrange', - type='float', - nargs=2, - action='store', - dest='makercountrange', - help= - 'Input the mean and spread of number of makers to use. e.g. 3 1.5 will be a normal distribution ' - 'with mean 3 and standard deveation 1.5 inclusive, default=3 1.5', - default=(3, 1.5)) + '-N', + '--makercountrange', + type='float', + nargs=2, + action='store', + dest='makercountrange', + help= + 'Input the mean and spread of number of makers to use. e.g. 3 1.5 will be a normal distribution ' + 'with mean 3 and standard deveation 1.5 inclusive, default=3 1.5', + default=(3, 1.5)) parser.add_option( - '--minmakercount', - type='int', - dest='minmakercount', - default=2, - help= - 'The minimum maker count in a transaction, random values below this are clamped at this number. default=2') + '--minmakercount', + type='int', + dest='minmakercount', + default=2, + help= + 'The minimum maker count in a transaction, random values below this are clamped at this number. default=2') parser.add_option('-M', '--mixdepthcount', type='int', @@ -364,69 +372,69 @@ def main(): help='How many mixing depths to mix through', default=4) parser.add_option( - '-c', - '--txcountparams', - type='float', - nargs=2, - dest='txcountparams', - default=(4, 1), - help= - 'The number of transactions to take coins from one mixing depth to the next, it is' - ' randomly chosen following a normal distribution. Should be similar to --addrask. ' - 'This option controls the parameters of the normal distribution curve. (mean, standard deviation). default=(4, 1)') + '-c', + '--txcountparams', + type='float', + nargs=2, + dest='txcountparams', + default=(4, 1), + help= + 'The number of transactions to take coins from one mixing depth to the next, it is' + ' randomly chosen following a normal distribution. Should be similar to --addrask. ' + 'This option controls the parameters of the normal distribution curve. (mean, standard deviation). default=(4, 1)') parser.add_option( - '--mintxcount', - type='int', - dest='mintxcount', - default=1, - help='The minimum transaction count per mixing level, default=1') + '--mintxcount', + type='int', + dest='mintxcount', + default=1, + help='The minimum transaction count per mixing level, default=1') parser.add_option( - '--donateamount', - type='float', - dest='donateamount', - default=0, - help= - 'percent of funds to donate to joinmarket development, or zero to opt out (default=0%)') + '--donateamount', + type='float', + dest='donateamount', + default=0, + help= + 'percent of funds to donate to joinmarket development, or zero to opt out (default=0%)') parser.add_option( - '--amountpower', - type='float', - dest='amountpower', - default=100.0, - help= - 'The output amounts follow a power law distribution, this is the power, default=100.0') + '--amountpower', + type='float', + dest='amountpower', + default=100.0, + help= + 'The output amounts follow a power law distribution, this is the power, default=100.0') parser.add_option( - '-l', - '--timelambda', - type='float', - dest='timelambda', - default=30, - help= - 'Average the number of minutes to wait between transactions. Randomly chosen ' - ' following an exponential distribution, which describes the time between uncorrelated' - ' events. default=30') + '-l', + '--timelambda', + type='float', + dest='timelambda', + default=30, + help= + 'Average the number of minutes to wait between transactions. Randomly chosen ' + ' following an exponential distribution, which describes the time between uncorrelated' + ' events. default=30') parser.add_option( - '-w', - '--wait-time', - action='store', - type='float', - dest='waittime', - help='wait time in seconds to allow orders to arrive, default=20', - default=20) + '-w', + '--wait-time', + action='store', + type='float', + dest='waittime', + help='wait time in seconds to allow orders to arrive, default=20', + default=20) parser.add_option( - '-s', - '--mincjamount', - type='int', - dest='mincjamount', - default=100000, - help='minimum coinjoin amount in transaction in satoshi, default 100k') + '-s', + '--mincjamount', + type='int', + dest='mincjamount', + default=100000, + help='minimum coinjoin amount in transaction in satoshi, default 100k') parser.add_option( - '-q', - '--liquiditywait', - type='int', - dest='liquiditywait', - default=60, - help= - 'amount of seconds to wait after failing to choose suitable orders before trying again, default 60') + '-q', + '--liquiditywait', + type='int', + dest='liquiditywait', + default=60, + help= + 'amount of seconds to wait after failing to choose suitable orders before trying again, default 60') (options, args) = parser.parse_args() if len(args) < 1: @@ -436,7 +444,7 @@ def main(): destaddrs = args[1:] print destaddrs - common.load_program_config() + load_program_config() for addr in destaddrs: addr_valid, errormsg = validate_address(addr) if not addr_valid: @@ -449,7 +457,7 @@ def main(): print 'not enough mixing depths to pay to all destination addresses, increasing mixdepthcount' options.mixdepthcount = options.addrcount + 1 if options.donateamount > 10.0: - #fat finger probably, or misunderstanding + # fat finger probably, or misunderstanding options.donateamount = 0.9 print str(options) @@ -468,13 +476,13 @@ def main(): dbg_tx_list = [] for srcmixdepth, txlist in tx_dict.iteritems(): dbg_tx_list.append({'srcmixdepth': srcmixdepth, 'tx': txlist}) - debug('tumbler transaction list') + log.debug('tumbler transaction list') pprint(dbg_tx_list) total_wait = sum([tx['wait'] for tx in tx_list]) print 'creates ' + str(len(tx_list)) + ' transactions in total' print 'waits in total for ' + str(len(tx_list)) + ' blocks and ' + str( - total_wait) + ' minutes' + total_wait) + ' minutes' total_block_and_wait = len(tx_list) * 10 + total_wait print('estimated time taken ' + str(total_block_and_wait) + ' minutes or ' + str(round(total_block_and_wait / 60.0, 2)) + ' hours') @@ -488,37 +496,38 @@ def main(): if ret[0] != 'y': return - #NOTE: possibly out of date documentation - #a couple of modes - #im-running-from-the-nsa, takes about 80 hours, costs a lot - #python tumbler.py -a 10 -N 10 5 -c 10 5 -l 50 -M 10 wallet_file 1xxx + # NOTE: possibly out of date documentation + # a couple of modes + # im-running-from-the-nsa, takes about 80 hours, costs a lot + # python tumbler.py -a 10 -N 10 5 -c 10 5 -l 50 -M 10 wallet_file 1xxx # - #quick and cheap, takes about 90 minutes - #python tumbler.py -N 2 1 -c 3 0.001 -l 10 -M 3 -a 1 wallet_file 1xxx + # quick and cheap, takes about 90 minutes + # python tumbler.py -N 2 1 -c 3 0.001 -l 10 -M 3 -a 1 wallet_file 1xxx # - #default, good enough for most, takes about 5 hours - #python tumbler.py wallet_file 1xxx + # default, good enough for most, takes about 5 hours + # python tumbler.py wallet_file 1xxx # - #for quick testing - #python tumbler.py -N 2 1 -c 3 0.001 -l 0.1 -M 3 -a 0 wallet_file 1xxx 1yyy + # for quick testing + # python tumbler.py -N 2 1 -c 3 0.001 -l 0.1 -M 3 -a 0 wallet_file 1xxx 1yyy wallet = Wallet(wallet_file, max_mix_depth=options.mixdepthsrc + options.mixdepthcount) - common.bc_interface.sync_wallet(wallet) + bc_interface.sync_wallet(wallet) - common.nickname = random_nick() - debug('starting tumbler') - irc = IRCMessageChannel(common.nickname) + nickname = random_nick() + set_nickname(nickname) + log.debug('starting tumbler') + irc = IRCMessageChannel(nickname) tumbler = Tumbler(irc, wallet, tx_list, options) try: - debug('connecting to irc') + log.debug('connecting to irc') irc.run() except: - debug('CRASHING, DUMPING EVERYTHING') + log.debug('CRASHING, DUMPING EVERYTHING') debug_dump_object(wallet, ['addr_cache', 'keys', 'seed']) debug_dump_object(tumbler) debug_dump_object(tumbler.cjtx) import traceback - debug(traceback.format_exc()) + log.debug(traceback.format_exc()) if __name__ == "__main__": diff --git a/wallet-tool.py b/wallet-tool.py index bea8d54d..78a679e5 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -1,15 +1,21 @@ -import sys, os -import getpass, json, datetime +from __future__ import absolute_import + +import datetime +import getpass +import json +import os +import sys from optparse import OptionParser -data_dir = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(data_dir, 'joinmarket')) + +from joinmarket import load_program_config, Wallet, bc_interface, \ + get_p2pk_vbyte, \ + mn_encode, mn_decode, get_network, encryptData +# data_dir = os.path.dirname(os.path.realpath(__file__)) +# sys.path.insert(0, os.path.join(data_dir, 'joinmarket')) import bitcoin as btc -from common import Wallet, load_program_config, get_p2pk_vbyte -import common -import old_mnemonic, slowaes -#structure for cj market wallet +# structure for cj market wallet # m/0/ root key # m/0/n/ nth mixing depth, where n=0 is unmixed, n=1 is coinjoined once, etc # pay in coins to mix at n=0 addresses @@ -18,17 +24,20 @@ # m/0/n/0/k kth receive address, for mixing depth n # m/0/n/1/k kth change address, for mixing depth n +description=('Does useful little tasks involving your bip32 wallet. The ' + 'method is one of the following: display - shows addresses and ' + 'balances. displayall - shows ALL addresses and balances. ' + 'summary - shows a summary of mixing depth balances.generate - ' + 'generates a new wallet recover - recovers a wallet from the 12 ' + 'word recovery seed showseed - shows the wallet recovery seed ' + 'and hex seed. importprivkey - adds privkeys to this wallet (' + 'privkeys are spaces or commas separated) listwallets - lists ' + 'all wallets with creator and timestamp') + parser = OptionParser( - usage='usage: %prog [options] [wallet file] [method]', - description='Does useful little tasks involving your bip32 wallet. The' + - ' method is one of the following: display - shows addresses and balances.' + - ' displayall - shows ALL addresses and balances.' + - ' summary - shows a summary of mixing depth balances.' + - ' generate - generates a new wallet.' + - ' recover - recovers a wallet from the 12 word recovery seed.' + - ' showseed - shows the wallet recovery seed and hex seed.' + - ' importprivkey - adds privkeys to this wallet (privkeys are spaces or commas separated)' - + ' listwallets - lists all wallets with creator and timestamp') + usage='usage: %prog [options] [wallet file] [method]', + description=description) + parser.add_option('-p', '--privkey', action='store_true', @@ -56,8 +65,8 @@ default=0) (options, args) = parser.parse_args() -#if the index_cache stored in wallet.json is longer than the default -#then set maxmixdepth to the length of index_cache +# if the index_cache stored in wallet.json is longer than the default +# then set maxmixdepth to the length of index_cache maxmixdepth_configured = True if not options.maxmixdepth: maxmixdepth_configured = False @@ -65,7 +74,7 @@ noseed_methods = ['generate', 'recover', 'listwallets'] methods = ['display', 'displayall', 'summary', 'showseed', 'importprivkey' - ] + noseed_methods + ] + noseed_methods noscan_methods = ['showseed', 'importprivkey'] if len(args) < 1: @@ -84,7 +93,7 @@ extend_mixdepth=not maxmixdepth_configured, storepassword=(method == 'importprivkey')) if method not in noscan_methods: - common.bc_interface.sync_wallet(wallet) + bc_interface.sync_wallet(wallet) if method == 'display' or method == 'displayall' or method == 'summary': @@ -92,6 +101,7 @@ def printd(s): if method != 'summary': print s + total_balance = 0 for m in range(wallet.max_mix_depth): printd('mixing depth %d m/0/%d/' % (m, m)) @@ -108,16 +118,16 @@ def printd(s): balance_depth += balance used = ('used' if k < wallet.index[m][forchange] else ' new') privkey = btc.encode_privkey( - wallet.get_key(m, forchange, k), 'wif_compressed', - get_p2pk_vbyte()) if options.showprivkey else '' + wallet.get_key(m, forchange, k), 'wif_compressed', + get_p2pk_vbyte()) if options.showprivkey else '' if method == 'displayall' or balance > 0 or (used == ' new' and - forchange == 0): + forchange == 0): printd(' m/0/%d/%d/%03d %-35s%s %.8f btc %s' % ( m, forchange, k, addr, used, balance / 1e8, privkey)) if m in wallet.imported_privkeys: printd(' import addresses') for privkey in wallet.imported_privkeys[m]: - addr = btc.privtoaddr(privkey, common.get_p2pk_vbyte()) + addr = btc.privtoaddr(privkey, get_p2pk_vbyte()) balance = 0.0 for addrvalue in wallet.unspent.values(): if addr == addrvalue['address']: @@ -125,8 +135,8 @@ def printd(s): used = (' used' if balance > 0.0 else 'empty') balance_depth += balance wip_privkey = btc.encode_privkey( - privkey, 'wif_compressed', - get_p2pk_vbyte()) if options.showprivkey else '' + privkey, 'wif_compressed', + get_p2pk_vbyte()) if options.showprivkey else '' printd(' ' * 13 + '%-35s%s %.8f btc %s' % (addr, used, balance / 1e8, wip_privkey)) total_balance += balance_depth @@ -135,16 +145,16 @@ def printd(s): elif method == 'generate' or method == 'recover': if method == 'generate': seed = btc.sha256(os.urandom(64))[:32] - words = old_mnemonic.mn_encode(seed) + words = mn_encode(seed) print 'Write down this wallet recovery seed\n\n' + ' '.join( - words) + '\n' + words) + '\n' elif method == 'recover': words = raw_input('Input 12 word recovery seed: ') - words = words.split() #default for split is 1 or more whitespace chars + words = words.split() # default for split is 1 or more whitespace chars if len(words) != 12: print 'ERROR: Recovery seed phrase must be exactly 12 words.' sys.exit(0) - seed = old_mnemonic.mn_decode(words) + seed = mn_decode(words) print seed password = getpass.getpass('Enter wallet encryption passphrase: ') password2 = getpass.getpass('Reenter wallet encryption passphrase: ') @@ -152,12 +162,12 @@ def printd(s): print 'ERROR. Passwords did not match' sys.exit(0) password_key = btc.bin_dbl_sha256(password) - encrypted_seed = slowaes.encryptData(password_key, seed.decode('hex')) + encrypted_seed = encryptData(password_key, seed.decode('hex')) timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S") walletfile = json.dumps({'creator': 'joinmarket project', 'creation_time': timestamp, 'encrypted_seed': encrypted_seed.encode('hex'), - 'network': common.get_network()}) + 'network': get_network()}) walletname = raw_input('Input wallet file name (default: wallet.json): ') if len(walletname) == 0: walletname = 'wallet.json' @@ -174,7 +184,7 @@ def printd(s): elif method == 'showseed': hexseed = wallet.seed print 'hexseed = ' + hexseed - words = old_mnemonic.mn_encode(hexseed) + words = mn_encode(hexseed) print 'Wallet recovery seed\n\n' + ' '.join(words) + '\n' elif method == 'importprivkey': print('WARNING: This imported key will not be recoverable with your 12 ' + @@ -188,30 +198,30 @@ def printd(s): privkeys = privkeys.split(',') if ',' in privkeys else privkeys.split() # TODO read also one key for each line for privkey in privkeys: - #TODO is there any point in only accepting wif format? check what other wallets do + # TODO is there any point in only accepting wif format? check what other wallets do privkey_format = btc.get_privkey_format(privkey) if privkey_format not in ['wif', 'wif_compressed']: print 'ERROR: privkey not in wallet import format' print privkey, 'skipped' continue if privkey_format == 'wif': - #TODO if they actually use an unc privkey, make sure the unc address is used - #r = raw_input('WARNING: Using uncompressed private key, the vast ' + + # TODO if they actually use an unc privkey, make sure the unc address is used + # r = raw_input('WARNING: Using uncompressed private key, the vast ' + # 'majority of JoinMarket transactions use compressed keys\n' + # 'being so unusual is bad for privacy. Continue? (y/n):') - #if r != 'y': + # if r != 'y': # sys.exit(0) print 'Uncompressed privkeys not supported (yet)' print privkey, 'skipped' continue privkey_bin = btc.encode_privkey(privkey, 'hex').decode('hex') - encrypted_privkey = slowaes.encryptData(wallet.password_key, - privkey_bin) + encrypted_privkey = encryptData(wallet.password_key, + privkey_bin) if 'imported_keys' not in wallet.walletdata: wallet.walletdata['imported_keys'] = [] wallet.walletdata['imported_keys'].append( - {'encrypted_privkey': encrypted_privkey.encode('hex'), - 'mixdepth': options.mixdepth}) + {'encrypted_privkey': encrypted_privkey.encode('hex'), + 'mixdepth': options.mixdepth}) if wallet.walletdata['imported_keys']: fd = open(wallet.path, 'w') fd.write(json.dumps(wallet.walletdata)) @@ -231,7 +241,7 @@ def printd(s): try: walletfile = fd.read() walletjson = json.loads(walletfile) - #Add filename to json format + # Add filename to json format walletjson['filename'] = possible_wallet walletjsons.append(walletjson) except ValueError: diff --git a/yield-generator.py b/yield-generator.py index 9a706617..c802c97b 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -1,35 +1,40 @@ #! /usr/bin/env python +from __future__ import absolute_import -import time, os, binascii, sys, datetime -import pprint -data_dir = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(data_dir, 'joinmarket')) +import datetime +import os +import time -from maker import * -from irc import IRCMessageChannel, random_nick -import bitcoin as btc -import common, blockchaininterface +from joinmarket import Maker, random_nick, set_nickname, get_network, get_log, \ + DUST_THRESHOLD, calc_cj_fee, load_program_config, bc_interface, \ + BlockrInterface, Wallet, IRCMessageChannel, debug_dump_object, config -from socket import gethostname +# data_dir = os.path.dirname(os.path.realpath(__file__)) +# sys.path.insert(0, os.path.join(data_dir, 'joinmarket')) + +# import blockchaininterface txfee = 1000 cjfee = '0.002' # 0.2% fee nickname = random_nick() +set_nickname(nickname) nickserv_password = '' minsize = int( - 1.2 * txfee / float(cjfee) -) #minimum size is such that you always net profit at least 20% of the miner fee + 1.2 * txfee / float(cjfee) +) # minimum size is such that you always net profit at least 20% of the miner fee mix_levels = 5 +log = get_log() + -#is a maker for the purposes of generating a yield from held +# is a maker for the purposes of generating a yield from held # bitcoins without ruining privacy for the taker, the taker could easily check # the history of the utxos this bot sends, so theres not much incentive # to ruin the privacy for barely any more yield -#sell-side algorithm: -#add up the value of each utxo for each mixing depth, +# sell-side algorithm: +# add up the value of each utxo for each mixing depth, # announce a relative-fee order of the highest balance -#spent from utxos that try to make the highest balance even higher +# spent from utxos that try to make the highest balance even higher # so try to keep coins concentrated in one mixing depth class YieldGenerator(Maker): statement_file = os.path.join('logs', 'yigen-statement.csv') @@ -42,7 +47,7 @@ def __init__(self, msgchan, wallet): self.tx_unconfirm_timestamp = {} def log_statement(self, data): - if common.get_network() == 'testnet': + if get_network() == 'testnet': return data = [str(d) for d in data] @@ -54,9 +59,10 @@ def on_welcome(self): Maker.on_welcome(self) if not os.path.isfile(self.statement_file): self.log_statement( - ['timestamp', 'cj amount/satoshi', 'my input count', - 'my input value/satoshi', 'cjfee/satoshi', 'earned/satoshi', - 'confirm time/min', 'notes']) + ['timestamp', 'cj amount/satoshi', 'my input count', + 'my input value/satoshi', 'cjfee/satoshi', + 'earned/satoshi', + 'confirm time/min', 'notes']) timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S") self.log_statement([timestamp, '', '', '', '', '', '', 'Connected']) @@ -64,15 +70,15 @@ def on_welcome(self): def create_my_orders(self): mix_balance = self.wallet.get_balance_by_mixdepth() if len([b for m, b in mix_balance.iteritems() if b > 0]) == 0: - debug('do not have any coins left') + log.debug('do not have any coins left') return [] - #print mix_balance + # print mix_balance max_mix = max(mix_balance, key=mix_balance.get) order = {'oid': 0, 'ordertype': 'relorder', 'minsize': minsize, - 'maxsize': mix_balance[max_mix] - common.DUST_THRESHOLD, + 'maxsize': mix_balance[max_mix] - DUST_THRESHOLD, 'txfee': txfee, 'cjfee': cjfee} return [order] @@ -81,46 +87,47 @@ def oid_to_order(self, cjorder, oid, amount): mix_balance = self.wallet.get_balance_by_mixdepth() max_mix = max(mix_balance, key=mix_balance.get) - #algo attempts to make the largest-balance mixing depth get an even larger balance - debug('finding suitable mixdepth') + # algo attempts to make the largest-balance mixing depth get an even larger balance + log.debug('finding suitable mixdepth') mixdepth = (max_mix - 1) % self.wallet.max_mix_depth while True: if mixdepth in mix_balance and mix_balance[mixdepth] >= amount: break mixdepth = (mixdepth - 1) % self.wallet.max_mix_depth - #mixdepth is the chosen depth we'll be spending from + # mixdepth is the chosen depth we'll be spending from cj_addr = self.wallet.get_receive_addr( - (mixdepth + 1) % self.wallet.max_mix_depth) + (mixdepth + 1) % self.wallet.max_mix_depth) change_addr = self.wallet.get_change_addr(mixdepth) utxos = self.wallet.select_utxos(mixdepth, amount) my_total_in = sum([va['value'] for va in utxos.values()]) real_cjfee = calc_cj_fee(cjorder.ordertype, cjorder.cjfee, amount) change_value = my_total_in - amount - cjorder.txfee + real_cjfee - if change_value <= common.DUST_THRESHOLD: - debug('change value=%d below dust threshold, finding new utxos' % - (change_value)) + if change_value <= DUST_THRESHOLD: + log.debug( + 'change value=%d below dust threshold, finding new utxos' % + (change_value)) try: utxos = self.wallet.select_utxos(mixdepth, - amount + common.DUST_THRESHOLD) + amount + DUST_THRESHOLD) except Exception: - debug( - 'dont have the required UTXOs to make a output above the dust threshold, quitting') + log.debug( + 'dont have the required UTXOs to make a output above the dust threshold, quitting') return None, None, None return utxos, cj_addr, change_addr def on_tx_unconfirmed(self, cjorder, txid, removed_utxos): self.tx_unconfirm_timestamp[cjorder.cj_addr] = int(time.time()) - #if the balance of the highest-balance mixing depth change then reannounce it + # if the balance of the highest-balance mixing depth change then reannounce it oldorder = self.orderlist[0] if len(self.orderlist) > 0 else None neworders = self.create_my_orders() if len(neworders) == 0: - return ([0], []) #cancel old order - if oldorder: #oldorder may not exist when this is called from on_tx_confirmed + return ([0], []) # cancel old order + if oldorder: # oldorder may not exist when this is called from on_tx_confirmed if oldorder['maxsize'] == neworders[0]['maxsize']: - return ([], []) #change nothing - #announce new order, replacing the old order + return ([], []) # change nothing + # announce new order, replacing the old order return ([], [neworders[0]]) def on_tx_confirmed(self, cjorder, confirmations, txid): @@ -131,17 +138,17 @@ def on_tx_confirmed(self, cjorder, confirmations, txid): confirm_time = 0 timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S") self.log_statement([timestamp, cjorder.cj_amount, len( - cjorder.utxos), sum([av['value'] for av in cjorder.utxos.values( - )]), cjorder.real_cjfee, cjorder.real_cjfee - cjorder.txfee, round( + cjorder.utxos), sum([av['value'] for av in cjorder.utxos.values( + )]), cjorder.real_cjfee, cjorder.real_cjfee - cjorder.txfee, round( confirm_time / 60.0, 2), '']) return self.on_tx_unconfirmed(cjorder, txid, None) def main(): - common.load_program_config() + load_program_config() import sys seed = sys.argv[1] - if isinstance(common.bc_interface, blockchaininterface.BlockrInterface): + if isinstance(bc_interface, BlockrInterface): print '\nYou are running a yield generator by polling the blockr.io website' print 'This is quite bad for privacy. That site is owned by coinbase.com' print 'Also your bot will run faster and more efficently, you can be immediately notified of new bitcoin network' @@ -152,25 +159,27 @@ def main(): return wallet = Wallet(seed, max_mix_depth=mix_levels) - common.bc_interface.sync_wallet(wallet) + bc_interface.sync_wallet(wallet) + + # nickname is set way above + # nickname - common.nickname = nickname - debug('starting yield generator') - irc = IRCMessageChannel(common.nickname, - realname='btcint=' + common.config.get( - "BLOCKCHAIN", "blockchain_source"), + log.debug('starting yield generator') + irc = IRCMessageChannel(nickname, + realname='btcint=' + config.get( + "BLOCKCHAIN", "blockchain_source"), password=nickserv_password) maker = YieldGenerator(irc, wallet) try: - debug('connecting to irc') + log.debug('connecting to irc') irc.run() except: - debug('CRASHING, DUMPING EVERYTHING') + log.debug('CRASHING, DUMPING EVERYTHING') debug_dump_object(wallet, ['addr_cache', 'keys', 'seed']) debug_dump_object(maker) debug_dump_object(irc) import traceback - debug(traceback.format_exc()) + log.debug(traceback.format_exc()) if __name__ == "__main__":