From 7cfbf3858eb673be4b3b4009b0ead65255677e36 Mon Sep 17 00:00:00 2001 From: Belcher Date: Thu, 11 Dec 2014 01:27:00 +0000 Subject: [PATCH 001/409] made entire utxo sweep, seed from sys.argv for now --- bip32-tool.py | 2 +- common.py | 4 +-- maker.py | 5 ++-- taker.py | 71 ++++++++++++++++++++++++++++++++++++++------------- 4 files changed, 59 insertions(+), 23 deletions(-) diff --git a/bip32-tool.py b/bip32-tool.py index a5f3b3d1..1cc38fd8 100644 --- a/bip32-tool.py +++ b/bip32-tool.py @@ -11,7 +11,7 @@ # m/0/n/1/k kth change address, for mixing depth n -seed = btc.sha256('dont use brainwallets') +seed = sys.argv[1] #btc.sha256('dont use brainwallets') #seed = '256 bits of randomness' diff --git a/common.py b/common.py index 4ec84098..a4188c2a 100644 --- a/common.py +++ b/common.py @@ -6,8 +6,8 @@ import json server = 'irc.freenode.net' -port = 6667 channel = '#joinmarket' +port = 6667 command_prefix = '!' MAX_PRIVMSG_LEN = 450 @@ -142,7 +142,7 @@ def calc_cj_fee(ordertype, cjfee, cj_amount): if ordertype == 'absorder': real_cjfee = int(cjfee) elif ordertype == 'relorder': - real_cjfee = int(Decimal(cjfee) * Decimal(cj_amount)) + real_cjfee = int((Decimal(cjfee) * Decimal(cj_amount)).quantize(Decimal(1))) else: raise RuntimeError('unknown order type: ' + str(ordertype)) return real_cjfee diff --git a/maker.py b/maker.py index 98ee11b1..7e0d41de 100644 --- a/maker.py +++ b/maker.py @@ -7,8 +7,9 @@ import sqlite3 import base64 -nickname = 'cj-maker' -seed = btc.sha256('dont use brainwallets except for holding testnet coins') +from socket import gethostname +nickname = 'cj-maker' + btc.sha256(gethostname())[:6] +seed = sys.argv[1] #btc.sha256('dont use brainwallets except for holding testnet coins') class CoinJoinOrder(object): def __init__(self, irc, nick, oid, amount): diff --git a/taker.py b/taker.py index 8badb809..ceb00890 100644 --- a/taker.py +++ b/taker.py @@ -7,11 +7,12 @@ import sqlite3, sys, base64 import threading, time -nickname = 'cj-taker' -seed = btc.sha256('your brainwallet goes here') +from socket import gethostname +nickname = 'cj-taker-' + btc.sha256(gethostname())[:6] +seed = sys.argv[1] #btc.sha256('your brainwallet goes here') my_utxo = '5cf68d4c42132f8f0bef8573454036953ddb3ba77a3bf3797d9862b7102d65cd:0' -my_tx_fee_contribution = 10000 +my_tx_fee = 10000 class CoinJoinTX(object): def __init__(self, irc, cj_amount, counterparties, oids, my_utxos, my_cj_addr, @@ -43,10 +44,12 @@ def recv_tx_parts(self, irc, nick, utxo_list, cj_addr, change_addr): order = db.execute('SELECT ordertype, txfee, cjfee FROM ' 'orderbook WHERE oid=? AND counterparty=?', (self.active_orders[nick], nick)).fetchone() + total_input = calc_total_input_value(self.utxos[nick]) real_cjfee = calc_cj_fee(order['ordertype'], order['cjfee'], self.cj_amount) self.outputs.append({'address': change_addr, 'value': - calc_total_input_value(self.utxos[nick]) - self.cj_amount - - order['txfee'] + real_cjfee}) + total_input - self.cj_amount - order['txfee'] + real_cjfee}) + print 'fee breakdown for %s totalin=%d cjamount=%d txfee=%d realcjfee=%d' % (nick, + total_input, self.cj_amount, order['txfee'], real_cjfee) self.outputs.append({'address': cj_addr, 'value': self.cj_amount}) self.cjfee_total += real_cjfee if len(self.nonrespondants) > 0: @@ -59,6 +62,8 @@ def recv_tx_parts(self, irc, nick, utxo_list, cj_addr, change_addr): my_total_in += int(usvals['value']) my_change_value = my_total_in - self.cj_amount - self.cjfee_total - self.my_txfee + print 'fee breakdown for me totalin=%d txfee=%d cjfee_total=%d' % (my_total_in, + self.my_txfee, self.cjfee_total) if self.my_change_addr == None: if my_change_value != 0: print 'WARNING CHANGE NOT BEING USED\nCHANGEVALUE = ' + str(my_change_value) @@ -125,21 +130,50 @@ def add_signature(self, sigb64): algo_thread = None #how long to wait for all the orders to arrive before starting to do coinjoins -ORDER_ARRIVAL_WAIT_TIME = 3 +ORDER_ARRIVAL_WAIT_TIME = 2 def choose_order(cj_amount): sqlorders = db.execute('SELECT * FROM orderbook;').fetchall() - #for o in sqlorders: - # print '(%s %s %d %d-%d %d %s)' % (o['counterparty'], o['ordertype'], o['oid'], - # o['minsize'], o['maxsize'], o['txfee'], o['cjfee']) - - orders = [(o['counterparty'], o['oid'], o['txfee'] + - calc_cj_fee(o['ordertype'], o['cjfee'], cj_amount)) + orders = [(o['counterparty'], o['oid'], calc_cj_fee(o['ordertype'], o['cjfee'], cj_amount)) for o in sqlorders if cj_amount >= o['minsize'] or cj_amount <= o['maxsize']] orders = sorted(orders, key=lambda k: k[2]) print 'orders = ' + str(orders) - return orders[0] #choose the cheapest + return orders[0] #choose the cheapest, later this will be chosen differently + +def choose_sweep_order(my_total_input, my_tx_fee): + ''' + choose an order given that we want to be left with no change + i.e. sweep an entire group of utxos + + solve for mychange = 0 + ABS FEE + mychange = totalin - cjamount - mytxfee - absfee + => cjamount = totalin - mytxfee - absfee + REL FEE + mychange = totalin - cjamount - mytxfee - relfee*cjamount + => 0 = totalin - mytxfee - cjamount*(1 + relfee) + => cjamount = (totalin - mytxfee) / (1 + relfee) + ''' + def calc_zero_change_cj_amount(ordertype, cjfee): + cj_amount = None + if ordertype == 'absorder': + cj_amount = my_total_input - my_tx_fee - cjfee + elif ordertype == 'relorder': + cj_amount = (my_total_input - my_tx_fee) / (Decimal(cjfee) + 1) + cj_amount = int(cj_amount.quantize(Decimal(1))) + else: + raise RuntimeError('unknown order type: ' + str(ordertype)) + return cj_amount + + sqlorders = db.execute('SELECT * FROM orderbook;').fetchall() + orders = [(o['counterparty'], o['oid'], calc_zero_change_cj_amount(o['ordertype'], o['cjfee']), + o['minsize'], o['maxsize']) for o in sqlorders] + #filter cj_amounts that are not in range + orders = [o[:3] for o in orders if o[2] >= o[3] and o[2] <= o[4]] + orders = sorted(orders, key=lambda k: k[2]) + print 'sweep orders = ' + str(orders) + return orders[-1] #choose one with the highest cj_amount, most left over after paying everything else #thread which does the buy-side algorithm # chooses which coinjoins to initiate and when @@ -167,12 +201,13 @@ def run(self): self.irc.shutdown() #break - utxo, addrvalue = self.initial_unspents.popitem() - counterparty, oid, fee = choose_order(addrvalue['value']) - cj_amount = addrvalue['value'] - fee + #utxo, addrvalue = self.initial_unspents.popitem() + utxo, addrvalue = [(k, v) for k, v in self.initial_unspents.iteritems() if v['value'] == 200000000][0] + counterparty, oid, cj_amount = choose_sweep_order(addrvalue['value'], my_tx_fee) self.finished_cj = False cjtx = CoinJoinTX(self.irc, cj_amount, [counterparty], [int(oid)], - [utxo], wallet.get_receive_addr(mixing_depth=1), None, my_tx_fee_contribution, self.finished_cj_callback) + [utxo], wallet.get_receive_addr(mixing_depth=1), None, + my_tx_fee, self.finished_cj_callback) #algorithm for making ''' single_cj_amount = 112000000 @@ -250,7 +285,7 @@ def on_pubmsg(irc, nick, message): #!fill [counterparty] [oid] [amount] cjtx = CoinJoinTX(irc, int(amount), [counterparty], [int(oid)], [my_utxo], wallet.get_receive_addr(mixing_depth=1), - wallet.get_change_addr(mixing_depth=0), my_tx_fee_contribution) + wallet.get_change_addr(mixing_depth=0), my_tx_fee) #self.connection.quit("Using irc.client.py") From 236d9e9c7f7222b88c58d7e1445626f06d77d58a Mon Sep 17 00:00:00 2001 From: Belcher Date: Thu, 11 Dec 2014 02:10:11 +0000 Subject: [PATCH 002/409] changed readme --- README.txt | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/README.txt b/README.txt index 07c9fd19..fa63f28c 100644 --- a/README.txt +++ b/README.txt @@ -7,10 +7,9 @@ HOWTO try 1. use bip32-tool.py to output a bunch of addresses send testnet coins to one mixing-depth=0 receive address do this for two wallet seeds, one for each taker and maker + seeds are taken as a command line argument -2. open taker.py and maker.py and set the wallet seed for each - one in the source code - also for taker.py set the unspent transaction output (utxo) variable +2. for taker.py set the unspent transaction output (utxo) variable for the coin you want to spend 3. join irc.freenode.net #joinmarket and run both taker.py and maker.py @@ -50,6 +49,9 @@ some other notes below.. #TODO #ask people on the testnet stuff to code up a few trading algos to see if the interface/protocol that # iv invented is general enough +a few algos: +fees proportional to how many utxos used, since the marginal cost is unrelated to your cj amount, only to + the amount of utxos you use up #TODO think of names #cj-market, cjex, but this isnt really an exchange @@ -80,6 +82,7 @@ some other notes below.. # but that wont stop mitm # after chats on irc, easiest is to do Trust On First Use, maker sends a pubkey over # TOFU requires a human to verify each first time, might not be practical +# skip the human verification, it will probably be okay # also theres some algorithm for detecting mitm #TODO implement something against dust @@ -88,7 +91,26 @@ some other notes below.. #TODO completely abstract away the irc stuff, so it can be switched to something else # e.g. twitter but more likely darkwallet obelisk and/or electrum server +TODO combine the taker and maker code into one file where you can make different kinds of + bot which combine both roles +e.g. tumbler.py repeatedly takes orders on the same coins again and again in an effort + to improve privacy and break the link between them, make sure to split up and combine them again + in random amounts, because the income-collector will also be splitting and combining coins +e.g. patient-tumbler.py which waits a while being a maker, then just starts to take orders + after a time limit for people who want to mix coins but dont mind waiting until a fixed upper time limit +e.g. income-collector.py which acts as a maker solely for the purpose of making money + might need to take orders at some point, for very small outputs which have a small probability of being filled +e.g. single-tx.py which takes a single order, using it to send coins to some address + typically as a payment, so this is what the electrum plugin would look like + +TODO +code a gui where a human can see the state of the orderbook and easily choose orders to fill +code a gui that easily explains to a human how they can choose a fee for their income-collector.py +both are important for market forces, since markets emerge from human decisions and actions + #TODO add random delays to the orderbook stuff so there isnt such a traffic spike when a new bot joins +#two options, random delay !orderbook for ones which dont mind, !orderbook without delay for bots +# which need the orders asap #TODO make sure the outputs are in random order # i.e. so its not like the taker always gets outputs 0,1 and maker 2,3 From 036ec3d1bcdd94f486d4e897c0c7c2c8d98f2f6f Mon Sep 17 00:00:00 2001 From: Belcher Date: Thu, 11 Dec 2014 22:33:16 +0000 Subject: [PATCH 003/409] random.shuffle applied to outputs --- README.txt | 8 ++++---- taker.py | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.txt b/README.txt index fa63f28c..6dc378e9 100644 --- a/README.txt +++ b/README.txt @@ -96,6 +96,8 @@ TODO combine the taker and maker code into one file where you can make different e.g. tumbler.py repeatedly takes orders on the same coins again and again in an effort to improve privacy and break the link between them, make sure to split up and combine them again in random amounts, because the income-collector will also be splitting and combining coins + random intervals between blocks included might be worth it too, since income-collector.py + will appear to have coins which dont get mixed again for a while e.g. patient-tumbler.py which waits a while being a maker, then just starts to take orders after a time limit for people who want to mix coins but dont mind waiting until a fixed upper time limit e.g. income-collector.py which acts as a maker solely for the purpose of making money @@ -112,11 +114,9 @@ both are important for market forces, since markets emerge from human decisions #two options, random delay !orderbook for ones which dont mind, !orderbook without delay for bots # which need the orders asap -#TODO make sure the outputs are in random order -# i.e. so its not like the taker always gets outputs 0,1 and maker 2,3 -#from random import shuffle - #TODO error checking so you cant crash the bot by sending malformed orders +when an error happens, send back a !error command so the counterparty knows + something went wrong, and then cancel that partly filled order #TODO make an ordertype where maker publishes the utxo he will use # this is a way to auction off the use of a desirable coin, maybe a diff --git a/taker.py b/taker.py index ceb00890..c1ba625f 100644 --- a/taker.py +++ b/taker.py @@ -5,7 +5,7 @@ import bitcoin as btc import sqlite3, sys, base64 -import threading, time +import threading, time, random from socket import gethostname nickname = 'cj-taker-' + btc.sha256(gethostname())[:6] @@ -70,6 +70,7 @@ def recv_tx_parts(self, irc, nick, utxo_list, cj_addr, change_addr): else: self.outputs.append({'address': self.my_change_addr, 'value': my_change_value}) utxo_tx = [dict([('output', u)]) for u in sum(self.utxos.values(), [])] + random.shuffle(self.outputs) tx = btc.mktx(utxo_tx, self.outputs) txb64 = base64.b64encode(tx.decode('hex')) n = MAX_PRIVMSG_LEN From ec528453391dd48793e14f8b1b5669574c6712b2 Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 12 Dec 2014 01:42:11 +0000 Subject: [PATCH 004/409] restructured code in preperation for other plans --- README.txt | 2 + common.py | 10 ++- irclib.py | 50 ++++++------ maker.py | 181 ++++++++++++++++++++---------------------- taker.py | 226 ++++++++++++++++++++++++++--------------------------- 5 files changed, 232 insertions(+), 237 deletions(-) diff --git a/README.txt b/README.txt index 6dc378e9..e1f7b712 100644 --- a/README.txt +++ b/README.txt @@ -78,6 +78,8 @@ fees proportional to how many utxos used, since the marginal cost is unrelated t #TODO option for how many blocks deep to wait before using a utxo for more mixing # 1 confirm is probably enough +TODO implement rate limiting for irc.privmsg to stop the bot being killed due to flood + #TODO encrypt messages between taker and maker, to stop trivial server eavesdropping # but that wont stop mitm # after chats on irc, easiest is to do Trust On First Use, maker sends a pubkey over diff --git a/common.py b/common.py index a4188c2a..5f124a72 100644 --- a/common.py +++ b/common.py @@ -5,10 +5,11 @@ import datetime import json -server = 'irc.freenode.net' -channel = '#joinmarket' -port = 6667 +HOST = 'irc.freenode.net' +CHANNEL = '#joinmarket' +PORT = 6667 +#TODO make this var all in caps command_prefix = '!' MAX_PRIVMSG_LEN = 450 @@ -119,6 +120,9 @@ def find_unspent_addresses(self): for m in range(MAX_MIX_DEPTH): for forchange in [0, 1]: addrs += [self.get_addr(m, forchange, n) for n in range(self.index[m][forchange])] + if len(addrs) == 0: + print 'no tx used' + return #TODO send a pull request to pybitcointools # unspent() doesnt tell you which address, you get a bunch of utxos diff --git a/irclib.py b/irclib.py index 0cef9c86..39e2686e 100644 --- a/irclib.py +++ b/irclib.py @@ -34,13 +34,13 @@ def run(self): #handle one channel at a time class IRCClient(object): - def __init__(self): - self.on_privmsg = None - self.on_pubmsg = None - self.on_welcome = None - self.on_set_topic = None - self.on_leave = None - self.on_nick_change = None #TODO implement + + def on_privmsg(self, nick, message): pass + def on_pubmsg(self, nick, message): pass + def on_welcome(self): pass + def on_set_topic(self, newtopic): pass + def on_leave(self, nick): pass + #TODO implement on_nick_change def close(self): self.send_raw("QUIT") @@ -53,7 +53,7 @@ def pubmsg(self, message): self.send_raw("PRIVMSG " + self.channel + " :" + message) def privmsg(self, nick, message): - print '>> ' + nick + ' :' + message + #print '>> ' + nick + ' :' + message self.send_raw("PRIVMSG " + nick + " :" + message) def send_raw(self, line): @@ -70,11 +70,9 @@ def __handle_privmsg(self, source, target, message): #self.send_raw('PRIVMSG ' + nick + ' :\x01VERSION #TODO ctcp version here, since some servers dont let you get on without if target == self.nick: - if self.on_privmsg != None: - self.on_privmsg(self, nick, message) + self.on_privmsg(nick, message) else: - if self.on_privmsg != None: - self.on_pubmsg(self, nick, message) + self.on_pubmsg(nick, message) def __handle_line(self, line): line = line.rstrip() @@ -98,27 +96,22 @@ def __handle_line(self, line): self.send_raw('NICK ' + self.nick) elif chunks[1] == '366': #end of names list self.connect_attempts = 0 - if self.on_welcome != None: - self.on_welcome(self) + self.on_welcome() elif chunks[1] == '332' or chunks[1] == 'TOPIC': #channel topic topic = get_irc_text(line) - if self.on_set_topic != None: - self.on_set_topic(self, topic) + self.on_set_topic(topic) elif chunks[1] == 'QUIT': nick = get_irc_nick(chunks[0]) if nick == self.nick: raise IOError('we quit') else: - if self.on_leave != None: - self.on_leave(self, nick) + self.on_leave(nick) elif chunks[1] == 'KICK': target = chunks[3] - if self.on_leave != None: - self.on_leave(self, nick) + self.on_leave(nick) elif chunks[1] == 'PART': nick = get_irc_nick(chunks[0]) - if self.on_leave != None: - self.on_leave(self, nick) + self.on_leave(nick) elif chunks[1] == 'JOIN': channel = chunks[2][1:] nick = get_irc_nick(chunks[0]) @@ -160,9 +153,20 @@ def run(self, server, port, nick, channel, username='username', realname='realna finally: self.fd.close() self.sock.close() - self.connect_attempts += 1 + print 'disconnected irc' time.sleep(10) + self.connect_attempts += 1 print 'reconnecting' print 'ending irc' self.give_up = True +def irc_privmsg_size_throttle(irc, target, lines, prefix=''): + line = '' + for l in lines: + line += l + if len(line) > MAX_PRIVMSG_LEN: + irc.privmsg(target, prefix + line) + line = '' + if len(line) > 0: + irc.privmsg(target, prefix + line) + diff --git a/maker.py b/maker.py index 7e0d41de..6c34c85e 100644 --- a/maker.py +++ b/maker.py @@ -12,44 +12,45 @@ seed = sys.argv[1] #btc.sha256('dont use brainwallets except for holding testnet coins') class CoinJoinOrder(object): - def __init__(self, irc, nick, oid, amount): + def __init__(self, maker, nick, oid, amount): + self.maker = maker self.oid = oid self.cj_amount = amount order = db.execute('SELECT * FROM myorders WHERE oid=?;', (oid,)).fetchone() if amount <= order['minsize'] or amount >= order['maxsize']: - irc.privmsg(nick, command_prefix + 'error Amount out of range') + maker.privmsg(nick, command_prefix + 'error amount out of range') #TODO logic for this error causing the order to be removed from list of open orders - self.utxos, self.mixing_depth = oid_to_order(oid, amount) + self.utxos, self.mixing_depth = oid_to_order(maker.wallet, oid, amount) self.ordertype = order['ordertype'] self.txfee = order['txfee'] self.cjfee = order['cjfee'] - self.cj_addr = wallet.get_receive_addr(self.mixing_depth) - self.change_addr = wallet.get_change_addr(self.mixing_depth - 1) + self.cj_addr = maker.wallet.get_receive_addr(self.mixing_depth) + self.change_addr = maker.wallet.get_change_addr(self.mixing_depth - 1) self.b64txparts = [] #even if the order ends up never being furfilled, you dont want someone # pretending to fill all your orders to find out which addresses you use - irc.privmsg(nick, command_prefix + 'myparts ' + ','.join(self.utxos) + ' ' + + maker.privmsg(nick, command_prefix + 'myparts ' + ','.join(self.utxos) + ' ' + self.cj_addr + ' ' + self.change_addr) def recv_tx_part(self, b64txpart): self.b64txparts.append(b64txpart) #TODO this is a dos opportunity, flood someone with !txpart #repeatedly to fill up their memory - def recv_tx(self, irc, nick, b64txpart): + def recv_tx(self, nick, b64txpart): self.b64txparts.append(b64txpart) tx = base64.b64decode(''.join(self.b64txparts)).encode('hex') txd = btc.deserialize(tx) goodtx, errmsg = self.verify_unsigned_tx(txd) if not goodtx: - irc.privmsg(nick, command_prefix + 'error ' + errmsg) + self.maker.privmsg(nick, command_prefix + 'error ' + errmsg) return False sigs = [] for index, ins in enumerate(txd['ins']): utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) - if utxo not in wallet.unspent: + if utxo not in self.maker.wallet.unspent: continue - addr = wallet.unspent[utxo]['address'] - txs = btc.sign(tx, index, wallet.get_key_from_addr(addr)) + addr = self.maker.wallet.unspent[utxo]['address'] + txs = btc.sign(tx, index, self.maker.wallet.get_key_from_addr(addr)) sigs.append(base64.b64encode(btc.deserialize(txs)['ins'][index]['script'].decode('hex'))) if len(sigs) == 0: print 'ERROR no private keys found' @@ -60,10 +61,10 @@ def recv_tx(self, irc, nick, b64txpart): prev_sigline = sigline sigline = sigline + command_prefix + 'sig ' + sig if len(sigline) > MAX_PRIVMSG_LEN: - irc.privmsg(nick, prev_sigline) + self.maker.privmsg(nick, prev_sigline) sigline = command_prefix + 'sig ' + sig if len(sigline) > 0: - irc.privmsg(nick, sigline) + self.maker.privmsg(nick, sigline) return True def verify_unsigned_tx(self, txd): @@ -72,7 +73,7 @@ def verify_unsigned_tx(self, txd): return False, 'my utxos are not contained' my_total_in = 0 for u in self.utxos: - usvals = wallet.unspent[u] + usvals = self.maker.wallet.unspent[u] my_total_in += int(usvals['value']) real_cjfee = calc_cj_fee(self.ordertype, self.cjfee, self.cj_amount) @@ -97,12 +98,9 @@ def verify_unsigned_tx(self, txd): return False, 'cj or change addr not in tx outputs exactly once' return True, None -wallet = Wallet(seed) -active_orders = {} - #these two functions create_my_orders() and oid_to_uxto() define the # sell-side pricing algorithm of this bot -def create_my_orders(): +def create_my_orders(wallet): db.execute("CREATE TABLE myorders(oid INTEGER, ordertype TEXT, " + "minsize INTEGER, maxsize INTEGER, txfee INTEGER, cjfee TEXT);") @@ -123,7 +121,7 @@ def create_my_orders(): oid += 1 ''' -def oid_to_order(oid, amount): +def oid_to_order(wallet, oid, amount): unspent = [] for utxo, addrvalue in wallet.unspent.iteritems(): unspent.append({'value': addrvalue['value'], 'utxo': utxo}) @@ -136,74 +134,69 @@ def oid_to_order(oid, amount): return [unspent['utxo']] ''' -#TODO this belongs in irclib.py -def irc_privmsg_size_throttle(irc, target, lines, prefix=''): - line = '' - for l in lines: - line += l - if len(line) > MAX_PRIVMSG_LEN: - irc.privmsg(target, prefix + line) - line = '' - if len(line) > 0: - irc.privmsg(target, prefix + line) - -def privmsg_all_orders(irc, target): - orderdb_keys = ['ordertype', 'oid', 'minsize', 'maxsize', 'txfee', 'cjfee'] - orderline = '' - for order in db.execute('SELECT * FROM myorders;').fetchall(): - elem_list = [str(order[k]) for k in orderdb_keys] - orderline += (command_prefix + ' '.join(elem_list)) - if len(orderline) > MAX_PRIVMSG_LEN: - irc.privmsg(target, orderline) - orderline = '' - if len(orderline) > 0: - irc.privmsg(target, orderline) + +class Maker(irclib.IRCClient): + def __init__(self, wallet): + self.active_orders = {} + self.wallet = wallet + + def privmsg_all_orders(self, target): + orderdb_keys = ['ordertype', 'oid', 'minsize', 'maxsize', 'txfee', 'cjfee'] + orderline = '' + for order in db.execute('SELECT * FROM myorders;').fetchall(): + elem_list = [str(order[k]) for k in orderdb_keys] + orderline += (command_prefix + ' '.join(elem_list)) + if len(orderline) > MAX_PRIVMSG_LEN: + self.privmsg(target, orderline) + orderline = '' + if len(orderline) > 0: + self.privmsg(target, orderline) + + def on_welcome(self): + self.privmsg_all_orders(CHANNEL) + + def on_privmsg(self, nick, message): + #debug("privmsg nick=%s message=%s" % (nick, message)) + if message[0] != command_prefix: + return + command_lines = message.split(command_prefix) + for command_line in command_lines: + chunks = command_line.split(" ") + if chunks[0] == 'fill': + oid = chunks[1] + amount = int(chunks[2]) + self.active_orders[nick] = CoinJoinOrder(self, nick, oid, amount) + elif chunks[0] == 'txpart': + b64txpart = chunks[1] #TODO check nick appears in active_orders + self.active_orders[nick].recv_tx_part(b64txpart) + elif chunks[0] == 'tx': + b64txpart = chunks[1] + self.active_orders[nick].recv_tx(nick, b64txpart) + + + #each order has an id for referencing to and looking up + # using the same id again overwrites it, they'll be plenty of times when an order + # has to be modified and its better to just have !order rather than !cancelorder then !order + def on_pubmsg(self, nick, message): + #debug("pubmsg nick=%s message=%s" % (nick, message)) + if message[0] == command_prefix: + chunks = message[1:].split(" ") + if chunks[0] == '%quit' or chunks[0] == '%makerquit': + self.shutdown() + elif chunks[0] == '%say': #% is a way to remind me its a testing cmd + self.pubmsg(message[6:]) + elif chunks[0] == '%rm': + self.pubmsg('!cancel ' + chunks[1]) + elif chunks[0] == 'orderbook': + self.privmsg_all_orders(nick) -def on_welcome(irc): - privmsg_all_orders(irc, channel) - -def on_privmsg(irc, nick, message): - #debug("privmsg nick=%s message=%s" % (nick, message)) - if message[0] != command_prefix: - return - command_lines = message.split(command_prefix) - for command_line in command_lines: - chunks = command_line.split(" ") - if chunks[0] == 'fill': - oid = chunks[1] - amount = int(chunks[2]) - active_orders[nick] = CoinJoinOrder(irc, nick, oid, amount) - elif chunks[0] == 'txpart': - b64txpart = chunks[1] #TODO check nick appears in active_orders - active_orders[nick].recv_tx_part(b64txpart) - elif chunks[0] == 'tx': - b64txpart = chunks[1] - active_orders[nick].recv_tx(irc, nick, b64txpart) - - -#each order has an id for referencing to and looking up -# using the same id again overwrites it, they'll be plenty of times when an order -# has to be modified and its better to just have !order rather than !cancelorder then !order -def on_pubmsg(irc, nick, message): - #debug("pubmsg nick=%s message=%s" % (nick, message)) - if message[0] == command_prefix: - chunks = message[1:].split(" ") - if chunks[0] == '%quit' or chunks[0] == '%makerquit': - irc.shutdown() - elif chunks[0] == '%say': #% is a way to remind me its a testing cmd - irc.pubmsg(message[6:]) - elif chunks[0] == '%rm': - irc.pubmsg('!cancel ' + chunks[1]) - elif chunks[0] == 'orderbook': - privmsg_all_orders(irc, nick) - -def on_set_topic(irc, newtopic): - chunks = newtopic.split('|') - try: - print chunks[1] - print chunks[3] - except IndexError: - pass + def on_set_topic(self, newtopic): + chunks = newtopic.split('|') + try: + print chunks[1].strip() + print chunks[3].strip() + except IndexError: + pass def main(): #TODO using sqlite3 to store my own orders is overkill, just @@ -212,17 +205,15 @@ def main(): con = sqlite3.connect(":memory:") con.row_factory = sqlite3.Row db = con.cursor() + + wallet = Wallet(seed) wallet.download_wallet_history() wallet.find_unspent_addresses() - create_my_orders() - - print 'starting irc' - irc = irclib.IRCClient() - irc.on_privmsg = on_privmsg - irc.on_pubmsg = on_pubmsg - irc.on_welcome = on_welcome - irc.on_set_topic = on_set_topic - irc.run(server, port, nickname, channel) + print 'downloaded wallet history' + + create_my_orders(wallet) + maker = Maker(wallet) + maker.run(HOST, PORT, nickname, CHANNEL) if __name__ == "__main__": main() diff --git a/taker.py b/taker.py index c1ba625f..86d2791e 100644 --- a/taker.py +++ b/taker.py @@ -15,17 +15,18 @@ my_tx_fee = 10000 class CoinJoinTX(object): - def __init__(self, irc, cj_amount, counterparties, oids, my_utxos, my_cj_addr, + def __init__(self, taker, cj_amount, counterparties, oids, my_utxos, my_cj_addr, my_change_addr, my_txfee, finishcallback=None): ''' if my_change is None then there wont be a change address thats used if you want to entirely coinjoin one utxo with no change left over ''' + self.taker = taker self.cj_amount = cj_amount self.active_orders = dict(zip(counterparties, oids)) self.nonrespondants = list(counterparties) self.my_utxos = my_utxos - self.utxos = {irc.nick: my_utxos} + self.utxos = {taker.nick: my_utxos} self.finishcallback = finishcallback self.my_txfee = my_txfee self.outputs = [{'address': my_cj_addr, 'value': self.cj_amount}] @@ -33,15 +34,15 @@ def __init__(self, irc, cj_amount, counterparties, oids, my_utxos, my_cj_addr, self.cjfee_total = 0 self.latest_tx = None for c, oid in zip(counterparties, oids): - irc.privmsg(c, command_prefix + 'fill ' + str(oid) + ' ' + str(cj_amount)) + taker.privmsg(c, command_prefix + 'fill ' + str(oid) + ' ' + str(cj_amount)) - def recv_tx_parts(self, irc, nick, utxo_list, cj_addr, change_addr): + def recv_tx_parts(self, nick, utxo_list, cj_addr, change_addr): if nick not in self.nonrespondants: debug('nick(' + nick + ') not in nonrespondants ' + str(self.nonrespondants)) return self.utxos[nick] = utxo_list self.nonrespondants.remove(nick) - order = db.execute('SELECT ordertype, txfee, cjfee FROM ' + order = self.taker.db.execute('SELECT ordertype, txfee, cjfee FROM ' 'orderbook WHERE oid=? AND counterparty=?', (self.active_orders[nick], nick)).fetchone() total_input = calc_total_input_value(self.utxos[nick]) @@ -58,7 +59,7 @@ def recv_tx_parts(self, irc, nick, utxo_list, cj_addr, change_addr): my_total_in = 0 for u in self.my_utxos: - usvals = wallet.unspent[u] + usvals = self.taker.wallet.unspent[u] my_total_in += int(usvals['value']) my_change_value = my_total_in - self.cj_amount - self.cjfee_total - self.my_txfee @@ -77,19 +78,19 @@ def recv_tx_parts(self, irc, nick, utxo_list, cj_addr, change_addr): txparts = [txb64[i:i+n] for i in range(0, len(txb64), n)] for p in txparts[:-1]: for nickk in self.active_orders.keys(): - irc.privmsg(nickk, command_prefix + 'txpart' + p) + self.taker.privmsg(nickk, command_prefix + 'txpart' + p) for nickk in self.active_orders.keys(): - irc.privmsg(nickk, command_prefix + 'tx ' + txparts[-1]) - #now sign it ourselves here + self.taker.privmsg(nickk, command_prefix + 'tx ' + txparts[-1]) + #now sign it ourselves here for index, ins in enumerate(btc.deserialize(tx)['ins']): utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) if utxo not in self.my_utxos: continue - if utxo not in wallet.unspent: + if utxo not in self.taker.wallet.unspent: continue - addr = wallet.unspent[utxo]['address'] - tx = btc.sign(tx, index, wallet.get_key_from_addr(addr)) + addr = self.taker.wallet.unspent[utxo]['address'] + tx = btc.sign(tx, index, self.taker.wallet.get_key_from_addr(addr)) self.latest_tx = btc.deserialize(tx) def add_signature(self, sigb64): @@ -98,6 +99,8 @@ def add_signature(self, sigb64): inserted_sig = False tx = btc.serialize(self.latest_tx) for index, ins in enumerate(self.latest_tx['ins']): + if ins['script'] != '': + continue ftx = btc.blockr_fetchtx(ins['outpoint']['hash'], get_network()) src_val = btc.deserialize(ftx)['outs'][ ins['outpoint']['index'] ] sig_good = btc.verify_tx_input(tx, index, src_val['script'], *btc.deserialize_script(sig)) @@ -125,7 +128,6 @@ def add_signature(self, sigb64): if self.finishcallback != None: self.finishcallback() -wallet = Wallet(seed) cjtx = None algo_thread = None @@ -179,10 +181,10 @@ def calc_zero_change_cj_amount(ordertype, cjfee): #thread which does the buy-side algorithm # chooses which coinjoins to initiate and when class AlgoThread(threading.Thread): - def __init__(self, irc, initial_unspents): + def __init__(self, taker, initial_unspents): threading.Thread.__init__(self) self.daemon = True - self.irc = irc + self.taker = taker self.initial_unspents = initial_unspents self.finished_cj = False @@ -199,15 +201,15 @@ def run(self): #TODO just make this do one tx and then stop if len(self.initial_unspents) == 0: print 'finished mixing, closing...' - self.irc.shutdown() + self.taker.shutdown() #break #utxo, addrvalue = self.initial_unspents.popitem() utxo, addrvalue = [(k, v) for k, v in self.initial_unspents.iteritems() if v['value'] == 200000000][0] counterparty, oid, cj_amount = choose_sweep_order(addrvalue['value'], my_tx_fee) self.finished_cj = False - cjtx = CoinJoinTX(self.irc, cj_amount, [counterparty], [int(oid)], - [utxo], wallet.get_receive_addr(mixing_depth=1), None, + cjtx = CoinJoinTX(self.taker, cj_amount, [counterparty], [int(oid)], + [utxo], self.taker.wallet.get_receive_addr(mixing_depth=1), None, my_tx_fee, self.finished_cj_callback) #algorithm for making ''' @@ -226,110 +228,102 @@ def run(self): print 'woken algo thread' -def add_order(nick, chunks): - db.execute('INSERT INTO orderbook VALUES(?, ?, ?, ?, ?, ?, ?);', - (nick, chunks[1], chunks[0], chunks[2], chunks[3], chunks[4], chunks[5])) - -def on_privmsg(irc, nick, message): - #debug("privmsg nick=%s message=%s" % (nick, message)) - if message[0] != command_prefix: - return - - for command in message[1:].split(command_prefix): - chunks = command.split(" ") - if chunks[0] in ordername_list: - add_order(nick, chunks) - elif chunks[0] == 'myparts': - utxo_list = chunks[1].split(',') - cj_addr = chunks[2] - change_addr = chunks[3] - cjtx.recv_tx_parts(irc, nick, utxo_list, cj_addr, change_addr) - elif chunks[0] == 'sig': - sig = chunks[1] - cjtx.add_signature(sig) - -#each order has an id for referencing to and looking up -# using the same id again overwrites it, they'll be plenty of times when an order -# has to be modified and its better to just have !order rather than !cancelorder then !order -def on_pubmsg(irc, nick, message): - global cjtx - print("pubmsg nick=%s message=%s" % (nick, message)) - if message[0] != command_prefix: - return - - for command in message[1:].split(command_prefix): - #commands starting with % are for testing and will be removed in the final version - chunks = command.split(" ") - if chunks[0] == '%quit' or chunks[0] == '%takerquit': - irc.shutdown() - elif chunks[0] == 'cancel': - #!cancel [oid] - try: - oid = int(chunks[1]) - db.execute("DELETE FROM orderbook WHERE counterparty=? AND oid=?;", - (nick, oid)) - except ValueError as e: - debug("!cancel " + repr(e)) - return - elif chunks[0] in ordername_list: - add_order(nick, chunks) - elif chunks[0] == '%showob': - print('printing orderbook') - for o in db.execute('SELECT * FROM orderbook;').fetchall(): - print '(%s %s %d %d-%d %d %s)' % (o['counterparty'], o['ordertype'], o['oid'], - o['minsize'], o['maxsize'], o['txfee'], o['cjfee']) - print('done') - elif chunks[0] == '%fill': - counterparty = chunks[1] - oid = chunks[2] - amount = chunks[3] - #!fill [counterparty] [oid] [amount] - cjtx = CoinJoinTX(irc, int(amount), [counterparty], [int(oid)], - [my_utxo], wallet.get_receive_addr(mixing_depth=1), - wallet.get_change_addr(mixing_depth=0), my_tx_fee) - - #self.connection.quit("Using irc.client.py") - -def on_welcome(irc): - global algo_thread - irc.pubmsg(command_prefix + 'orderbook') - algo_thread = AlgoThread(irc, wallet.unspent.copy()) - #algo_thread.start() - -def on_set_topic(irc, newtopic): - chunks = newtopic.split('|') - try: - print chunks[1] - print chunks[2] - except IndexError: - pass - -''' -for m in range(2): - print 'mixing depth ' + str(m) - for forchange in range(2): - print ' forchange=' + str(forchange) - for n in range(3): - #print ' ' + str(n) + ' ' + btc.privtoaddr(wallet.get_key(m, forchange, n), 0x6f) -''' +class Taker(irclib.IRCClient): + def __init__(self, wallet): + con = sqlite3.connect(":memory:", check_same_thread=False) + con.row_factory = sqlite3.Row + self.db = con.cursor() + self.db.execute("CREATE TABLE orderbook(counterparty TEXT, oid INTEGER, ordertype TEXT, " + + "minsize INTEGER, maxsize INTEGER, txfee INTEGER, cjfee TEXT);") + + self.wallet = wallet + + def add_order(self, nick, chunks): + self.db.execute('INSERT INTO orderbook VALUES(?, ?, ?, ?, ?, ?, ?);', + (nick, chunks[1], chunks[0], chunks[2], chunks[3], chunks[4], chunks[5])) + + def on_privmsg(self, nick, message): + #debug("privmsg nick=%s message=%s" % (nick, message)) + if message[0] != command_prefix: + return + + for command in message[1:].split(command_prefix): + chunks = command.split(" ") + if chunks[0] in ordername_list: + self.add_order(nick, chunks) + elif chunks[0] == 'myparts': + utxo_list = chunks[1].split(',') + cj_addr = chunks[2] + change_addr = chunks[3] + cjtx.recv_tx_parts(nick, utxo_list, cj_addr, change_addr) + elif chunks[0] == 'sig': + sig = chunks[1] + cjtx.add_signature(sig) + + #each order has an id for referencing to and looking up + # using the same id again overwrites it, they'll be plenty of times when an order + # has to be modified and its better to just have !order rather than !cancelorder then !order + def on_pubmsg(self, nick, message): + global cjtx + print("pubmsg nick=%s message=%s" % (nick, message)) + if message[0] != command_prefix: + return + + for command in message[1:].split(command_prefix): + #commands starting with % are for testing and will be removed in the final version + chunks = command.split(" ") + if chunks[0] == '%quit' or chunks[0] == '%takerquit': + self.shutdown() + elif chunks[0] == 'cancel': + #!cancel [oid] + try: + oid = int(chunks[1]) + self.db.execute("DELETE FROM orderbook WHERE counterparty=? AND oid=?;", + (nick, oid)) + except ValueError as e: + debug("!cancel " + repr(e)) + return + elif chunks[0] in ordername_list: + self.add_order(nick, chunks) + elif chunks[0] == '%showob': + print('printing orderbook') + for o in self.db.execute('SELECT * FROM orderbook;').fetchall(): + print '(%s %s %d %d-%d %d %s)' % (o['counterparty'], o['ordertype'], o['oid'], + o['minsize'], o['maxsize'], o['txfee'], o['cjfee']) + print('done') + elif chunks[0] == '%fill': + counterparty = chunks[1] + oid = chunks[2] + amount = chunks[3] + #!fill [counterparty] [oid] [amount] + cjtx = CoinJoinTX(self, int(amount), [counterparty], [int(oid)], + [my_utxo], self.wallet.get_receive_addr(mixing_depth=1), + self.wallet.get_change_addr(mixing_depth=0), my_tx_fee) + + #self.connection.quit("Using irc.client.py") + + def on_welcome(self): + global algo_thread + self.pubmsg(command_prefix + 'orderbook') + algo_thread = AlgoThread(self, self.wallet.unspent.copy()) + #algo_thread.start() + + def on_set_topic(self, newtopic): + chunks = newtopic.split('|') + try: + print chunks[1].strip() + print chunks[2].strip() + except IndexError: + pass def main(): - global db - con = sqlite3.connect(":memory:", check_same_thread=False) - con.row_factory = sqlite3.Row - db = con.cursor() - db.execute("CREATE TABLE orderbook(counterparty TEXT, oid INTEGER, ordertype TEXT, " - + "minsize INTEGER, maxsize INTEGER, txfee INTEGER, cjfee TEXT);") + wallet = Wallet(seed) wallet.download_wallet_history() wallet.find_unspent_addresses() print 'starting irc' - irc = irclib.IRCClient() - irc.on_privmsg = on_privmsg - irc.on_pubmsg = on_pubmsg - irc.on_welcome = on_welcome - irc.on_set_topic = on_set_topic - irc.run(server, port, nickname, channel) + taker = Taker(wallet) + taker.run(HOST, PORT, nickname, CHANNEL) if __name__ == "__main__": main() From 7c2435100a7b2271b3de31097bdcdec6ad7a2b3f Mon Sep 17 00:00:00 2001 From: Belcher Date: Mon, 15 Dec 2014 21:30:42 +0000 Subject: [PATCH 005/409] replaced sql db with simple python datastructures --- maker.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/maker.py b/maker.py index 6c34c85e..1b05c156 100644 --- a/maker.py +++ b/maker.py @@ -8,7 +8,7 @@ import base64 from socket import gethostname -nickname = 'cj-maker' + btc.sha256(gethostname())[:6] +nickname = 'cj-maker-' + btc.sha256(gethostname())[:6] seed = sys.argv[1] #btc.sha256('dont use brainwallets except for holding testnet coins') class CoinJoinOrder(object): @@ -16,7 +16,7 @@ def __init__(self, maker, nick, oid, amount): self.maker = maker self.oid = oid self.cj_amount = amount - order = db.execute('SELECT * FROM myorders WHERE oid=?;', (oid,)).fetchone() + order = [o for o in orderlist if o['oid'] == oid][0] if amount <= order['minsize'] or amount >= order['maxsize']: maker.privmsg(nick, command_prefix + 'error amount out of range') #TODO logic for this error causing the order to be removed from list of open orders @@ -101,18 +101,20 @@ def verify_unsigned_tx(self, txd): #these two functions create_my_orders() and oid_to_uxto() define the # sell-side pricing algorithm of this bot def create_my_orders(wallet): - db.execute("CREATE TABLE myorders(oid INTEGER, ordertype TEXT, " - + "minsize INTEGER, maxsize INTEGER, txfee INTEGER, cjfee TEXT);") #tells the highest value possible made by combining all utxos #fee is 0.2% of the cj amount total_value = 0 for utxo, addrvalue in wallet.unspent.iteritems(): total_value += addrvalue['value'] - db.execute('INSERT INTO myorders VALUES(?, ?, ?, ?, ?, ?);', - (0, 'relorder', 0, total_value, 10000, '0.002')) + order = {'oid': 0, 'ordertype': 'relorder', 'minsize': 0, + 'maxsize': total_value, 'txfee': 10000, 'cjfee': '0.002'} + global orderlist + orderlist = [order] ''' + db.execute("CREATE TABLE myorders(oid INTEGER, ordertype TEXT, " + + "minsize INTEGER, maxsize INTEGER, txfee INTEGER, cjfee TEXT);") #simple algorithm where each utxo we have becomes an order oid = 0 for un in db.execute('SELECT * FROM unspent;').fetchall(): @@ -141,10 +143,10 @@ def __init__(self, wallet): self.wallet = wallet def privmsg_all_orders(self, target): - orderdb_keys = ['ordertype', 'oid', 'minsize', 'maxsize', 'txfee', 'cjfee'] + order_keys = ['ordertype', 'oid', 'minsize', 'maxsize', 'txfee', 'cjfee'] orderline = '' - for order in db.execute('SELECT * FROM myorders;').fetchall(): - elem_list = [str(order[k]) for k in orderdb_keys] + for order in orderlist: + elem_list = [str(order[k]) for k in order_keys] orderline += (command_prefix + ' '.join(elem_list)) if len(orderline) > MAX_PRIVMSG_LEN: self.privmsg(target, orderline) @@ -163,7 +165,7 @@ def on_privmsg(self, nick, message): for command_line in command_lines: chunks = command_line.split(" ") if chunks[0] == 'fill': - oid = chunks[1] + oid = int(chunks[1]) amount = int(chunks[2]) self.active_orders[nick] = CoinJoinOrder(self, nick, oid, amount) elif chunks[0] == 'txpart': @@ -201,10 +203,6 @@ def on_set_topic(self, newtopic): def main(): #TODO using sqlite3 to store my own orders is overkill, just # use a python data structure - global db - con = sqlite3.connect(":memory:") - con.row_factory = sqlite3.Row - db = con.cursor() wallet = Wallet(seed) wallet.download_wallet_history() From 698d7842cad560be5eac9b906552591226107f7a Mon Sep 17 00:00:00 2001 From: Belcher Date: Mon, 15 Dec 2014 21:40:29 +0000 Subject: [PATCH 006/409] added some more thoughts --- README.txt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.txt b/README.txt index e1f7b712..184a4d9e 100644 --- a/README.txt +++ b/README.txt @@ -73,7 +73,10 @@ fees proportional to how many utxos used, since the marginal cost is unrelated t #TODO use electrum json_rpc instead of the pybitcointools stuff # problem, i dont think that supports testnet # bitcoind json_rpc obviously supports testnet, but someone else can download -# the blockchain +# the blockchain, actually it seems you cant replace pybitcointools with bitcoind +# cant look up any txid or address +# could use a websocket api for learning when new blocks/tx appear +# could use python-bitcoinlib to be a node in the p2p network #TODO option for how many blocks deep to wait before using a utxo for more mixing # 1 confirm is probably enough @@ -85,6 +88,7 @@ TODO implement rate limiting for irc.privmsg to stop the bot being killed due to # after chats on irc, easiest is to do Trust On First Use, maker sends a pubkey over # TOFU requires a human to verify each first time, might not be practical # skip the human verification, it will probably be okay +# make the irc nick be a hash of the pubkey # also theres some algorithm for detecting mitm #TODO implement something against dust @@ -106,6 +110,8 @@ e.g. income-collector.py which acts as a maker solely for the purpose of making might need to take orders at some point, for very small outputs which have a small probability of being filled e.g. single-tx.py which takes a single order, using it to send coins to some address typically as a payment, so this is what the electrum plugin would look like +e.g. gui-taker.py has a gui which shows the user the orderbook and they can easily fill and order + and see other statistics, could be easily done by opening a http port and sending a html form and graphics TODO code a gui where a human can see the state of the orderbook and easily choose orders to fill From 9e49a9c72c23426b77798bc04e719c81ad9516f4 Mon Sep 17 00:00:00 2001 From: Belcher Date: Wed, 17 Dec 2014 20:49:06 +0000 Subject: [PATCH 007/409] fixed some bugs, in wallet downloading history --- bip32-tool.py | 5 +++-- common.py | 19 +++++++++---------- maker.py | 7 +++---- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/bip32-tool.py b/bip32-tool.py index 1cc38fd8..f7b00304 100644 --- a/bip32-tool.py +++ b/bip32-tool.py @@ -1,5 +1,6 @@ import bitcoin as btc +import sys #structure for cj market wallet # m/0/ root key @@ -21,14 +22,14 @@ addr_vbyte = 0x6f #testnet m_0 = btc.bip32_ckd(master, 0) -for n in range(3): +for n in range(2): print 'mixing depth ' + str(n) + ' m/0/' + str(n) + '/' m_0_n = btc.bip32_ckd(m_0, n) for forchange in range(2): print(' ' + ('receive' if forchange==0 else 'change') + ' addresses m/0/%d/%d/' % (n, forchange)) m_0_n_c = btc.bip32_ckd(m_0_n, forchange) - for k in range(4): + for k in range(15): m_0_n_c_k = btc.bip32_ckd(m_0_n_c, k) priv = btc.bip32_extract_key(m_0_n_c_k) print' m/0/%d/%d/%d/ ' % (n, forchange, k) + btc.privtoaddr(priv, addr_vbyte)# + ' ' + btc.encode_privkey(priv, 'wif') diff --git a/common.py b/common.py index 5f124a72..d5ca6a02 100644 --- a/common.py +++ b/common.py @@ -22,7 +22,7 @@ def get_network(): return 'testnet' #TODO change this name into get_addr_ver() or something -def get_vbyte(): +def get_addr_vbyte(): if get_network() == 'testnet': return 0x6f else: @@ -52,7 +52,7 @@ def get_key(self, mixing_depth, forchange, i): return btc.bip32_extract_key(btc.bip32_ckd(self.keys[mixing_depth][forchange], i)) def get_addr(self, mixing_depth, forchange, i): - return btc.privtoaddr(self.get_key(mixing_depth, forchange, i), get_vbyte()) + return btc.privtoaddr(self.get_key(mixing_depth, forchange, i), get_addr_vbyte()) def get_new_addr(self, mixing_depth, forchange): index = self.index[mixing_depth] @@ -78,13 +78,14 @@ def download_wallet_history(self, gaplimit=6): sets Wallet internal indexes to be at the next unused address ''' addr_req_count = 20 - + for mix_depth in range(MAX_MIX_DEPTH): for forchange in [0, 1]: unused_addr_count = 0 + last_used_addr = '' while unused_addr_count < gaplimit: addrs = [self.get_new_addr(mix_depth, forchange) for i in range(addr_req_count)] - + #TODO send a pull request to pybitcointools # because this surely should be possible with a function from it if get_network() == 'testnet': @@ -93,8 +94,6 @@ def download_wallet_history(self, gaplimit=6): blockr_url = 'http://btc.blockr.io/api/v1/address/txs/' res = btc.make_request(blockr_url+','.join(addrs)) data = json.loads(res)['data'] - last_used_addr = '' - unused_addr_count = 0 for dat in data: if dat['nb_txs'] != 0: last_used_addr = dat['address'] @@ -102,10 +101,10 @@ def download_wallet_history(self, gaplimit=6): unused_addr_count += 1 if unused_addr_count >= gaplimit: break - if last_used_addr == '': - self.index[mix_depth][forchange] = 0 - else: - self.index[mix_depth][forchange] = self.addr_cache[last_used_addr][2] + 1 + if last_used_addr == '': + self.index[mix_depth][forchange] = 0 + else: + self.index[mix_depth][forchange] = self.addr_cache[last_used_addr][2] + 1 def find_unspent_addresses(self): ''' diff --git a/maker.py b/maker.py index 1b05c156..14409945 100644 --- a/maker.py +++ b/maker.py @@ -85,7 +85,7 @@ def verify_unsigned_tx(self, txd): times_seen_cj_addr = 0 times_seen_change_addr = 0 for outs in txd['outs']: - addr = btc.script_to_address(outs['script'], get_vbyte()) + addr = btc.script_to_address(outs['script'], get_addr_vbyte()) if addr == self.cj_addr: times_seen_cj_addr += 1 if outs['value'] != self.cj_amount: @@ -201,14 +201,13 @@ def on_set_topic(self, newtopic): pass def main(): - #TODO using sqlite3 to store my own orders is overkill, just - # use a python data structure - wallet = Wallet(seed) wallet.download_wallet_history() wallet.find_unspent_addresses() print 'downloaded wallet history' + + create_my_orders(wallet) maker = Maker(wallet) maker.run(HOST, PORT, nickname, CHANNEL) From 8aa7ef0ed3c434d6d5163777dff51f708f73bb03 Mon Sep 17 00:00:00 2001 From: Belcher Date: Tue, 23 Dec 2014 00:49:45 +0000 Subject: [PATCH 008/409] wait for confirms code, easy to write maker algos --- README.txt | 11 ++++ bip32-tool.py | 8 ++- common.py | 48 ++++++++++++-- irclib.py | 25 +++---- maker.py | 177 +++++++++++++++++++++++++++++++++++--------------- taker.py | 11 ++-- 6 files changed, 205 insertions(+), 75 deletions(-) diff --git a/README.txt b/README.txt index 184a4d9e..1c13ce34 100644 --- a/README.txt +++ b/README.txt @@ -82,6 +82,7 @@ fees proportional to how many utxos used, since the marginal cost is unrelated t # 1 confirm is probably enough TODO implement rate limiting for irc.privmsg to stop the bot being killed due to flood +i suggest creating a thread that only dispatches/writes to the irc socket #TODO encrypt messages between taker and maker, to stop trivial server eavesdropping # but that wont stop mitm @@ -113,6 +114,11 @@ e.g. single-tx.py which takes a single order, using it to send coins to some add e.g. gui-taker.py has a gui which shows the user the orderbook and they can easily fill and order and see other statistics, could be easily done by opening a http port and sending a html form and graphics +TODO need to move onto the bip44 structure of HD wallets + +TODO +probably a good idea to have a debug.log where loads of information is dumped + TODO code a gui where a human can see the state of the orderbook and easily choose orders to fill code a gui that easily explains to a human how they can choose a fee for their income-collector.py @@ -122,6 +128,11 @@ both are important for market forces, since markets emerge from human decisions #two options, random delay !orderbook for ones which dont mind, !orderbook without delay for bots # which need the orders asap +TODO +the add_addr_notify() stuff doesnt work, so if theres several CoinJoinOrder's open it will start a few + threads to do the notifying, they could race condition or other multithreaded errors +i suggest to create a single thread that sorts out all the stuff + #TODO error checking so you cant crash the bot by sending malformed orders when an error happens, send back a !error command so the counterparty knows something went wrong, and then cancel that partly filled order diff --git a/bip32-tool.py b/bip32-tool.py index f7b00304..d6e26b82 100644 --- a/bip32-tool.py +++ b/bip32-tool.py @@ -15,6 +15,7 @@ seed = sys.argv[1] #btc.sha256('dont use brainwallets') #seed = '256 bits of randomness' +print_privkey = len(sys.argv) > 2 master = btc.bip32_master_key(seed)#, btc.TESTNET_PRIVATE) print 'master = ' + master @@ -29,10 +30,13 @@ print(' ' + ('receive' if forchange==0 else 'change') + ' addresses m/0/%d/%d/' % (n, forchange)) m_0_n_c = btc.bip32_ckd(m_0_n, forchange) - for k in range(15): + for k in range(3): m_0_n_c_k = btc.bip32_ckd(m_0_n_c, k) priv = btc.bip32_extract_key(m_0_n_c_k) - print' m/0/%d/%d/%d/ ' % (n, forchange, k) + btc.privtoaddr(priv, addr_vbyte)# + ' ' + btc.encode_privkey(priv, 'wif') + output = btc.privtoaddr(priv, addr_vbyte) + if print_privkey: + output += ' ' + btc.encode_privkey(priv, 'wif', addr_vbyte) + print' m/0/%d/%d/%d/ ' % (n, forchange, k) + output ''' diff --git a/common.py b/common.py index d5ca6a02..1fe86c3b 100644 --- a/common.py +++ b/common.py @@ -1,9 +1,8 @@ import bitcoin as btc from decimal import Decimal -import sys -import datetime -import json +import sys, datetime, json, time +import threading HOST = 'irc.freenode.net' CHANNEL = '#joinmarket' @@ -11,7 +10,7 @@ #TODO make this var all in caps command_prefix = '!' -MAX_PRIVMSG_LEN = 450 +MAX_PRIVMSG_LEN = 400 ordername_list = ["absorder", "relorder"] @@ -139,6 +138,47 @@ def find_unspent_addresses(self): self.unspent[u['tx']+':'+str(u['n'])] = {'address': dat['address'], 'value': int(u['amount'].replace('.', ''))} +#awful way of doing this, but works for now +# later use websocket api for people who dont download the blockchain +# and -walletnotify for people who do +def add_addr_notify(address, unconfirmfun, confirmfun): + + class NotifyThread(threading.Thread): + def __init__(self, address, unconfirmfun, confirmfun): + threading.Thread.__init__(self) + self.daemon = True + self.address = address + self.unconfirmfun = unconfirmfun + self.confirmfun = confirmfun + + def run(self): + while True: + time.sleep(5) + if get_network() == 'testnet': + blockr_url = 'http://tbtc.blockr.io/api/v1/address/balance/' + else: + blockr_url = 'http://btc.blockr.io/api/v1/address/balance/' + res = btc.make_request(blockr_url + self.address + '?confirmations=0') + data = json.loads(res)['data'] + if data['balance'] > 0: + break + self.unconfirmfun(data['balance']*1e8) + while True: + time.sleep(5 * 60) + if get_network() == 'testnet': + blockr_url = 'http://tbtc.blockr.io/api/v1/address/txs/' + else: + blockr_url = 'http://btc.blockr.io/api/v1/address/txs/' + res = btc.make_request(blockr_url + self.address + '?confirmations=0') + data = json.loads(res)['data'] + if data['nb_txs'] == 0: + continue + if data['txs'][0]['confirmations'] >= 1: #confirmation threshold + break + self.confirmfun(data['txs'][0]['confirmations'], + data['txs'][0]['tx'], data['txs'][0]['amount']*1e8) + + NotifyThread(address, unconfirmfun, confirmfun).start() def calc_cj_fee(ordertype, cjfee, cj_amount): real_cjfee = None diff --git a/irclib.py b/irclib.py index 39e2686e..98b27182 100644 --- a/irclib.py +++ b/irclib.py @@ -20,17 +20,20 @@ def __init__(self, irc): def run(self): while not self.irc.give_up: time.sleep(PING_INTERVAL) - self.irc.ping_reply = False - #maybe use this to calculate the lag one day - self.irc.lockcond.acquire() - self.irc.send_raw('PING LAG' + str(int(time.time() * 1000))) - self.irc.lockcond.wait(PING_TIMEOUT) - self.irc.lockcond.release() - if not self.irc.ping_reply: - print 'irc ping timed out' - self.irc.close() - self.irc.fd.close() - self.irc.sock.close() + try: + self.irc.ping_reply = False + #maybe use this to calculate the lag one day + self.irc.lockcond.acquire() + self.irc.send_raw('PING LAG' + str(int(time.time() * 1000))) + self.irc.lockcond.wait(PING_TIMEOUT) + self.irc.lockcond.release() + if not self.irc.ping_reply: + print 'irc ping timed out' + self.irc.close() + self.irc.fd.close() + self.irc.sock.close() + except IOError: + pass #handle one channel at a time class IRCClient(object): diff --git a/maker.py b/maker.py index 14409945..4292d039 100644 --- a/maker.py +++ b/maker.py @@ -16,21 +16,23 @@ def __init__(self, maker, nick, oid, amount): self.maker = maker self.oid = oid self.cj_amount = amount - order = [o for o in orderlist if o['oid'] == oid][0] + order = [o for o in maker.orderlist if o['oid'] == oid][0] if amount <= order['minsize'] or amount >= order['maxsize']: maker.privmsg(nick, command_prefix + 'error amount out of range') #TODO logic for this error causing the order to be removed from list of open orders - self.utxos, self.mixing_depth = oid_to_order(maker.wallet, oid, amount) + self.utxos, self.mixing_depth = maker.oid_to_order(oid, amount) self.ordertype = order['ordertype'] self.txfee = order['txfee'] self.cjfee = order['cjfee'] self.cj_addr = maker.wallet.get_receive_addr(self.mixing_depth) self.change_addr = maker.wallet.get_change_addr(self.mixing_depth - 1) self.b64txparts = [] - #even if the order ends up never being furfilled, you dont want someone - # pretending to fill all your orders to find out which addresses you use + #always a new address even if the order ends up never being + # furfilled, you dont want someone pretending to fill all your + # orders to find out which addresses you use maker.privmsg(nick, command_prefix + 'myparts ' + ','.join(self.utxos) + ' ' + self.cj_addr + ' ' + self.change_addr) + def recv_tx_part(self, b64txpart): self.b64txparts.append(b64txpart) #TODO this is a dos opportunity, flood someone with !txpart @@ -38,8 +40,8 @@ def recv_tx_part(self, b64txpart): def recv_tx(self, nick, b64txpart): self.b64txparts.append(b64txpart) - tx = base64.b64decode(''.join(self.b64txparts)).encode('hex') - txd = btc.deserialize(tx) + self.tx = base64.b64decode(''.join(self.b64txparts)).encode('hex') + txd = btc.deserialize(self.tx) goodtx, errmsg = self.verify_unsigned_tx(txd) if not goodtx: self.maker.privmsg(nick, command_prefix + 'error ' + errmsg) @@ -50,10 +52,11 @@ def recv_tx(self, nick, b64txpart): if utxo not in self.maker.wallet.unspent: continue addr = self.maker.wallet.unspent[utxo]['address'] - txs = btc.sign(tx, index, self.maker.wallet.get_key_from_addr(addr)) + txs = btc.sign(self.tx, index, self.maker.wallet.get_key_from_addr(addr)) sigs.append(base64.b64encode(btc.deserialize(txs)['ins'][index]['script'].decode('hex'))) if len(sigs) == 0: print 'ERROR no private keys found' + add_addr_notify(self.change_addr, self.unconfirm_callback, self.confirm_callback) #TODO make this a function in irclib.py sigline = '' @@ -67,6 +70,26 @@ def recv_tx(self, nick, b64txpart): self.maker.privmsg(nick, sigline) return True + def unconfirm_callback(self, value): + to_cancel, to_announce = self.maker.on_tx_unconfirmed(self, value) + self.handle_modified_orders(to_cancel, to_announce) + + def confirm_callback(self, confirmations, txid, value): + to_cancel, to_announce = self.maker.on_tx_confirmed(self, + confirmations, txid, value) + self.handle_modified_orders(to_cancel, to_announce) + + def handle_modified_orders(self, to_cancel, to_announce): + for oid in to_cancel: + order = [o for o in self.maker.orderlist if o['oid'] == oid][0] + self.maker.orderlist.remove(order) + if len(to_cancel) > 0: + clines = ['!cancel ' + str(oid) for oid in to_cancel] + self.maker.pubmsg(''.join(clines)) + if len(to_announce) > 0: + self.maker.privmsg_all_orders(CHANNEL, to_announce) + self.maker.orderlist.append(to_announce) + def verify_unsigned_tx(self, txd): tx_utxos = set([ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) for ins in txd['ins']]) if not tx_utxos.issuperset(set(self.utxos)): @@ -79,7 +102,7 @@ def verify_unsigned_tx(self, txd): real_cjfee = calc_cj_fee(self.ordertype, self.cjfee, self.cj_amount) expected_change_value = (my_total_in - self.cj_amount - self.txfee + real_cjfee) - debug('earned fee = ' + str(real_cjfee)) + debug('earned = ' + str(real_cjfee - self.txfee)) debug('mycjaddr, mychange = ' + self.cj_addr + ', ' + self.change_addr) times_seen_cj_addr = 0 @@ -93,56 +116,21 @@ def verify_unsigned_tx(self, txd): if addr == self.change_addr: times_seen_change_addr += 1 if outs['value'] != expected_change_value: - return False, 'wrong change, i expect ' + str(expected_change_address) + return False, 'wrong change, i expect ' + str(expected_change_value) if times_seen_cj_addr != 1 or times_seen_change_addr != 1: return False, 'cj or change addr not in tx outputs exactly once' return True, None -#these two functions create_my_orders() and oid_to_uxto() define the -# sell-side pricing algorithm of this bot -def create_my_orders(wallet): - - #tells the highest value possible made by combining all utxos - #fee is 0.2% of the cj amount - total_value = 0 - for utxo, addrvalue in wallet.unspent.iteritems(): - total_value += addrvalue['value'] - - order = {'oid': 0, 'ordertype': 'relorder', 'minsize': 0, - 'maxsize': total_value, 'txfee': 10000, 'cjfee': '0.002'} - global orderlist - orderlist = [order] - ''' - db.execute("CREATE TABLE myorders(oid INTEGER, ordertype TEXT, " - + "minsize INTEGER, maxsize INTEGER, txfee INTEGER, cjfee TEXT);") - #simple algorithm where each utxo we have becomes an order - oid = 0 - for un in db.execute('SELECT * FROM unspent;').fetchall(): - db.execute('INSERT INTO myorders VALUES(?, ?, ?, ?, ?, ?);', - (oid, 'absorder', 0, un['value'], 10000, '100000')) - oid += 1 - ''' - -def oid_to_order(wallet, oid, amount): - unspent = [] - for utxo, addrvalue in wallet.unspent.iteritems(): - unspent.append({'value': addrvalue['value'], 'utxo': utxo}) - inputs = btc.select(unspent, amount) - #TODO this raises an exception if you dont have enough money, id rather it just returned None - mixing_depth = 1 - return [i['utxo'] for i in inputs], mixing_depth - ''' - unspent = db.execute('SELECT * FROM unspent WHERE value > ?;', (amount,)).fetchone() - return [unspent['utxo']] - ''' - - class Maker(irclib.IRCClient): def __init__(self, wallet): self.active_orders = {} self.wallet = wallet + self.nextoid = -1 + self.orderlist = self.create_my_orders() - def privmsg_all_orders(self, target): + def privmsg_all_orders(self, target, orderlist=None): + if orderlist == None: + orderlist = self.orderlist order_keys = ['ordertype', 'oid', 'minsize', 'maxsize', 'txfee', 'cjfee'] orderline = '' for order in orderlist: @@ -166,7 +154,7 @@ def on_privmsg(self, nick, message): chunks = command_line.split(" ") if chunks[0] == 'fill': oid = int(chunks[1]) - amount = int(chunks[2]) + amount = int(chunks[2]) #TODO make sure that nick doesnt already have an open order self.active_orders[nick] = CoinJoinOrder(self, nick, oid, amount) elif chunks[0] == 'txpart': b64txpart = chunks[1] #TODO check nick appears in active_orders @@ -200,16 +188,97 @@ def on_set_topic(self, newtopic): except IndexError: pass + #these functions + # create_my_orders() + # oid_to_uxto() + # on_tx_unconfirmed() + # on_tx_confirmed() + #define the sell-side pricing algorithm of this bot + #still might be a bad way of doing things, we'll see + def create_my_orders(self): + + ''' + #tells the highest value possible made by combining all utxos + #fee is 0.2% of the cj amount + total_value = 0 + for utxo, addrvalue in self.wallet.unspent.iteritems(): + total_value += addrvalue['value'] + + order = {'oid': 0, 'ordertype': 'relorder', 'minsize': 0, + 'maxsize': total_value, 'txfee': 10000, 'cjfee': '0.002'} + return [order] + ''' + + #each utxo is a single absolute-fee order + orderlist = [] + for utxo, addrvalue in self.wallet.unspent.iteritems(): + order = {'oid': self.get_next_oid(), 'ordertype': 'absorder', 'minsize': 0, + 'maxsize': addrvalue['value'], 'txfee': 10000, 'cjfee': 100000, + 'utxo': utxo} + orderlist.append(order) + #yes you can add keys there that are never used by the rest of the Maker code + # so im adding utxo here + return orderlist + + + #has to return a list of utxos and mixing depth the cj address will be in + # the change address will be in mixing_depth-1 + def oid_to_order(self, oid, amount): + ''' + unspent = [] + for utxo, addrvalue in self.wallet.unspent.iteritems(): + unspent.append({'value': addrvalue['value'], 'utxo': utxo}) + inputs = btc.select(unspent, amount) + #TODO this raises an exception if you dont have enough money, id rather it just returned None + mixing_depth = 1 + return [i['utxo'] for i in inputs], mixing_depth + ''' + + order = [o for o in self.orderlist if o['oid'] == oid][0] + mixing_depth = 1 #TODO in this toy algo sort out mixing depth + return [order['utxo']], mixing_depth + + def get_next_oid(self): + self.nextoid += 1 + return self.nextoid + + #gets called when the tx is seen on the network + #must return which orders to cancel or recreate + def on_tx_unconfirmed(self, order, value): + print 'tx unconfirmed' + return ([order.oid], []) + + #gets called when the tx is included in a block + #must return which orders to cancel or recreate + # and i have to think about how that will work for both + # the blockchain explorer api method and the bitcoid walletnotify + def on_tx_confirmed(self, order, confirmations, txid, value): + print 'tx confirmed' + to_announce = [] + txd = btc.deserialize(order.tx) + for i, out in enumerate(txd['outs']): + addr = btc.script_to_address(out['script'], get_addr_vbyte()) + if addr == order.change_addr: + neworder = {'oid': self.get_next_oid(), 'ordertype': 'absorder', 'minsize': 0, + 'maxsize': out['value'], 'txfee': 10000, 'cjfee': 100000, + 'utxo': txid + ':' + str(i)} + to_announce.append(neworder) + if addr == order.cj_addr: + neworder = {'oid': self.get_next_oid(), 'ordertype': 'absorder', 'minsize': 0, + 'maxsize': out['value'], 'txfee': 10000, 'cjfee': 100000, + 'utxo': txid + ':' + str(i)} + to_announce.append(neworder) + return ([], to_announce) + + def main(): + print 'downloading wallet history' wallet = Wallet(seed) wallet.download_wallet_history() wallet.find_unspent_addresses() - print 'downloaded wallet history' - - - create_my_orders(wallet) maker = Maker(wallet) + print 'connecting to irc' maker.run(HOST, PORT, nickname, CHANNEL) if __name__ == "__main__": diff --git a/taker.py b/taker.py index 86d2791e..482ae5af 100644 --- a/taker.py +++ b/taker.py @@ -4,13 +4,12 @@ import irclib import bitcoin as btc -import sqlite3, sys, base64 -import threading, time, random +import sqlite3, sys, base64, threading, time, random from socket import gethostname nickname = 'cj-taker-' + btc.sha256(gethostname())[:6] seed = sys.argv[1] #btc.sha256('your brainwallet goes here') -my_utxo = '5cf68d4c42132f8f0bef8573454036953ddb3ba77a3bf3797d9862b7102d65cd:0' +my_utxo = '5b0b416a5f5b8d8b08873de5e5b2df46c9190f4f5a440db638ce38f651b1bbf6:0' my_tx_fee = 10000 @@ -314,9 +313,13 @@ def on_set_topic(self, newtopic): print chunks[1].strip() print chunks[2].strip() except IndexError: - pass + pass + + def on_leave(self, nick): + self.db.execute('DELETE FROM orderbook WHERE counterparty=?;', (nick,)) def main(): + print 'downloading wallet history' wallet = Wallet(seed) wallet.download_wallet_history() wallet.find_unspent_addresses() From 17d5fe3a69edf5ebfb0848f4970cf648222450da Mon Sep 17 00:00:00 2001 From: Belcher Date: Tue, 23 Dec 2014 12:29:11 +0000 Subject: [PATCH 009/409] added mixing depth handling for unspent --- README.txt | 19 ++++++++----------- common.py | 45 ++++++++++++++++++++++++++++----------------- maker.py | 6 +++--- taker.py | 7 +++++-- 4 files changed, 44 insertions(+), 33 deletions(-) diff --git a/README.txt b/README.txt index 1c13ce34..888e9aa2 100644 --- a/README.txt +++ b/README.txt @@ -9,24 +9,21 @@ HOWTO try do this for two wallet seeds, one for each taker and maker seeds are taken as a command line argument -2. for taker.py set the unspent transaction output (utxo) variable - for the coin you want to spend +2. join irc.freenode.net #joinmarket and run both taker.py and maker.py -3. join irc.freenode.net #joinmarket and run both taker.py and maker.py - -4. when both bots join and have announced their orders, use this +3. when both bots join and have announced their orders, use this command to start a coinjoining - !%fill [counterparty] [order-id] [cj-amount] + !%fill [counterparty] [order-id] [cj-amount] [utxo] so for example if the maker is called 'cj-maker' and you want to mix 1.9btc - !%fill cj-maker 0 190000000 + !%fill cj-maker 0 190000000 5cf68d4c42132f8f0bef8573454036953ddb3ba77a3bf3797d9862b7102d65cd:1 all values are in satoshis, the first order has order-id 0 and it counts up +you can use !%unspent to see a printout of taker's unspent transaction outputs +and !%showob to see the orderbook -5. watch the outputs of both bots, soon enough taker.py will say it has completed - a transaction, it will not do pushtx() but instead print the tx hex - you can examine this, with a blockchain explorer or my coin-jumble app and - push it to the network yourself, or not, whatever +4. watch the outputs of both bots, soon enough taker.py will say it has completed + a transaction, maker will wait for the transaction to be seen and confirmed theres lots that needs to be done some other notes below.. diff --git a/common.py b/common.py index 1fe86c3b..67a19dec 100644 --- a/common.py +++ b/common.py @@ -112,31 +112,42 @@ def find_unspent_addresses(self): you know which addresses have been used ''' + addr_req_count = 20 + #TODO handle the case where there are so many addresses it cant # fit into one api call (>50 or so) - addrs = [] + addrs = {} for m in range(MAX_MIX_DEPTH): for forchange in [0, 1]: - addrs += [self.get_addr(m, forchange, n) for n in range(self.index[m][forchange])] + for n in range(self.index[m][forchange]): + addrs[self.get_addr(m, forchange, n)] = m if len(addrs) == 0: print 'no tx used' return - #TODO send a pull request to pybitcointools - # unspent() doesnt tell you which address, you get a bunch of utxos - # but dont know which privkey to sign with - if get_network() == 'testnet': - blockr_url = 'http://tbtc.blockr.io/api/v1/address/unspent/' - elif network == 'btc': - blockr_url = 'http://btc.blockr.io/api/v1/address/unspent/' - res = btc.make_request(blockr_url+','.join(addrs)) - data = json.loads(res)['data'] - if 'unspent' in data: - data = [data] - for dat in data: - for u in dat['unspent']: - self.unspent[u['tx']+':'+str(u['n'])] = {'address': - dat['address'], 'value': int(u['amount'].replace('.', ''))} + i = 0 + addrkeys = addrs.keys() + while i < len(addrkeys): + inc = min(len(addrkeys) - i, addr_req_count) + req = addrkeys[i:i + inc] + i += inc + + #TODO send a pull request to pybitcointools + # unspent() doesnt tell you which address, you get a bunch of utxos + # but dont know which privkey to sign with + if get_network() == 'testnet': + blockr_url = 'http://tbtc.blockr.io/api/v1/address/unspent/' + elif network == 'btc': + blockr_url = 'http://btc.blockr.io/api/v1/address/unspent/' + res = btc.make_request(blockr_url+','.join(req)) + data = json.loads(res)['data'] + if 'unspent' in data: + data = [data] + for dat in data: + for u in dat['unspent']: + self.unspent[u['tx']+':'+str(u['n'])] = {'address': + dat['address'], 'value': int(u['amount'].replace('.', '')), + 'mixdepth': addrs[dat['address']]} #awful way of doing this, but works for now # later use websocket api for people who dont download the blockchain diff --git a/maker.py b/maker.py index 4292d039..f904391f 100644 --- a/maker.py +++ b/maker.py @@ -214,10 +214,10 @@ def create_my_orders(self): for utxo, addrvalue in self.wallet.unspent.iteritems(): order = {'oid': self.get_next_oid(), 'ordertype': 'absorder', 'minsize': 0, 'maxsize': addrvalue['value'], 'txfee': 10000, 'cjfee': 100000, - 'utxo': utxo} + 'utxo': utxo, 'mixdepth': addrvalue['mixdepth']} orderlist.append(order) #yes you can add keys there that are never used by the rest of the Maker code - # so im adding utxo here + # so im adding utxo and mixdepth here return orderlist @@ -235,7 +235,7 @@ def oid_to_order(self, oid, amount): ''' order = [o for o in self.orderlist if o['oid'] == oid][0] - mixing_depth = 1 #TODO in this toy algo sort out mixing depth + mixing_depth = order['mixdepth'] + 1 return [order['utxo']], mixing_depth def get_next_oid(self): diff --git a/taker.py b/taker.py index 482ae5af..4918a0ed 100644 --- a/taker.py +++ b/taker.py @@ -9,7 +9,6 @@ from socket import gethostname nickname = 'cj-taker-' + btc.sha256(gethostname())[:6] seed = sys.argv[1] #btc.sha256('your brainwallet goes here') -my_utxo = '5b0b416a5f5b8d8b08873de5e5b2df46c9190f4f5a440db638ce38f651b1bbf6:0' my_tx_fee = 10000 @@ -290,11 +289,15 @@ def on_pubmsg(self, nick, message): print '(%s %s %d %d-%d %d %s)' % (o['counterparty'], o['ordertype'], o['oid'], o['minsize'], o['maxsize'], o['txfee'], o['cjfee']) print('done') + elif chunks[0] == '%unspent': + from pprint import pprint + pprint(self.wallet.unspent) elif chunks[0] == '%fill': counterparty = chunks[1] oid = chunks[2] amount = chunks[3] - #!fill [counterparty] [oid] [amount] + my_utxo = chunks[4] + #!fill [counterparty] [oid] [amount] [utxo] cjtx = CoinJoinTX(self, int(amount), [counterparty], [int(oid)], [my_utxo], self.wallet.get_receive_addr(mixing_depth=1), self.wallet.get_change_addr(mixing_depth=0), my_tx_fee) From 9079b58413320cf999e2918ac5d25dd7481ca346 Mon Sep 17 00:00:00 2001 From: Belcher Date: Tue, 23 Dec 2014 14:31:11 +0000 Subject: [PATCH 010/409] moved some code away from taker.py --- taker.py | 152 ++------------------------------------------------- testtaker.py | 73 +++++++++++++++++++++++++ tumbler.py | 108 ++++++++++++++++++++++++++++++++++++ 3 files changed, 187 insertions(+), 146 deletions(-) create mode 100644 testtaker.py create mode 100644 tumbler.py diff --git a/taker.py b/taker.py index 4918a0ed..7080a8b4 100644 --- a/taker.py +++ b/taker.py @@ -4,13 +4,7 @@ import irclib import bitcoin as btc -import sqlite3, sys, base64, threading, time, random - -from socket import gethostname -nickname = 'cj-taker-' + btc.sha256(gethostname())[:6] -seed = sys.argv[1] #btc.sha256('your brainwallet goes here') - -my_tx_fee = 10000 +import sqlite3, base64, threading, time, random class CoinJoinTX(object): def __init__(self, taker, cj_amount, counterparties, oids, my_utxos, my_cj_addr, @@ -126,116 +120,14 @@ def add_signature(self, sigb64): if self.finishcallback != None: self.finishcallback() -cjtx = None - -algo_thread = None - -#how long to wait for all the orders to arrive before starting to do coinjoins -ORDER_ARRIVAL_WAIT_TIME = 2 - -def choose_order(cj_amount): - - sqlorders = db.execute('SELECT * FROM orderbook;').fetchall() - orders = [(o['counterparty'], o['oid'], calc_cj_fee(o['ordertype'], o['cjfee'], cj_amount)) - for o in sqlorders if cj_amount >= o['minsize'] or cj_amount <= o['maxsize']] - orders = sorted(orders, key=lambda k: k[2]) - print 'orders = ' + str(orders) - return orders[0] #choose the cheapest, later this will be chosen differently - -def choose_sweep_order(my_total_input, my_tx_fee): - ''' - choose an order given that we want to be left with no change - i.e. sweep an entire group of utxos - - solve for mychange = 0 - ABS FEE - mychange = totalin - cjamount - mytxfee - absfee - => cjamount = totalin - mytxfee - absfee - REL FEE - mychange = totalin - cjamount - mytxfee - relfee*cjamount - => 0 = totalin - mytxfee - cjamount*(1 + relfee) - => cjamount = (totalin - mytxfee) / (1 + relfee) - ''' - def calc_zero_change_cj_amount(ordertype, cjfee): - cj_amount = None - if ordertype == 'absorder': - cj_amount = my_total_input - my_tx_fee - cjfee - elif ordertype == 'relorder': - cj_amount = (my_total_input - my_tx_fee) / (Decimal(cjfee) + 1) - cj_amount = int(cj_amount.quantize(Decimal(1))) - else: - raise RuntimeError('unknown order type: ' + str(ordertype)) - return cj_amount - - sqlorders = db.execute('SELECT * FROM orderbook;').fetchall() - orders = [(o['counterparty'], o['oid'], calc_zero_change_cj_amount(o['ordertype'], o['cjfee']), - o['minsize'], o['maxsize']) for o in sqlorders] - #filter cj_amounts that are not in range - orders = [o[:3] for o in orders if o[2] >= o[3] and o[2] <= o[4]] - orders = sorted(orders, key=lambda k: k[2]) - print 'sweep orders = ' + str(orders) - return orders[-1] #choose one with the highest cj_amount, most left over after paying everything else - -#thread which does the buy-side algorithm -# chooses which coinjoins to initiate and when -class AlgoThread(threading.Thread): - def __init__(self, taker, initial_unspents): - threading.Thread.__init__(self) - self.daemon = True - self.taker = taker - self.initial_unspents = initial_unspents - self.finished_cj = False - - def finished_cj_callback(self): - self.finished_cj = True - print 'finished cj' - - def run(self): - global cjtx - time.sleep(ORDER_ARRIVAL_WAIT_TIME) - #while True: - if 1: - #wait for orders to arrive - #TODO just make this do one tx and then stop - if len(self.initial_unspents) == 0: - print 'finished mixing, closing...' - self.taker.shutdown() - #break - - #utxo, addrvalue = self.initial_unspents.popitem() - utxo, addrvalue = [(k, v) for k, v in self.initial_unspents.iteritems() if v['value'] == 200000000][0] - counterparty, oid, cj_amount = choose_sweep_order(addrvalue['value'], my_tx_fee) - self.finished_cj = False - cjtx = CoinJoinTX(self.taker, cj_amount, [counterparty], [int(oid)], - [utxo], self.taker.wallet.get_receive_addr(mixing_depth=1), None, - my_tx_fee, self.finished_cj_callback) - #algorithm for making - ''' - single_cj_amount = 112000000 - unspent = [] - for utxo, addrvalue in self.initial_unspents.iteritems(): - unspent.append({'value': addrvalue['value'], 'utxo': utxo}) - inputs = btc.select(unspent, single_cj_amount) - my_utxos = [i['utxo'] for i in inputs] - counterparty, oid = choose_order(single_cj_amount) - cjtx = CoinJoinTX(self.irc, int(single_cj_amount), [counterparty], [int(oid)], - my_utxos, wallet.get_receive_addr(mixing_depth=1), wallet.get_change_addr(mixing_depth=0)) - ''' - while not self.finished_cj: - time.sleep(5) - print 'woken algo thread' - - class Taker(irclib.IRCClient): - def __init__(self, wallet): + def __init__(self): con = sqlite3.connect(":memory:", check_same_thread=False) con.row_factory = sqlite3.Row self.db = con.cursor() self.db.execute("CREATE TABLE orderbook(counterparty TEXT, oid INTEGER, ordertype TEXT, " + "minsize INTEGER, maxsize INTEGER, txfee INTEGER, cjfee TEXT);") - self.wallet = wallet - def add_order(self, nick, chunks): self.db.execute('INSERT INTO orderbook VALUES(?, ?, ?, ?, ?, ?, ?);', (nick, chunks[1], chunks[0], chunks[2], chunks[3], chunks[4], chunks[5])) @@ -249,21 +141,12 @@ def on_privmsg(self, nick, message): chunks = command.split(" ") if chunks[0] in ordername_list: self.add_order(nick, chunks) - elif chunks[0] == 'myparts': - utxo_list = chunks[1].split(',') - cj_addr = chunks[2] - change_addr = chunks[3] - cjtx.recv_tx_parts(nick, utxo_list, cj_addr, change_addr) - elif chunks[0] == 'sig': - sig = chunks[1] - cjtx.add_signature(sig) #each order has an id for referencing to and looking up # using the same id again overwrites it, they'll be plenty of times when an order # has to be modified and its better to just have !order rather than !cancelorder then !order def on_pubmsg(self, nick, message): - global cjtx - print("pubmsg nick=%s message=%s" % (nick, message)) + #print("pubmsg nick=%s message=%s" % (nick, message)) if message[0] != command_prefix: return @@ -283,32 +166,11 @@ def on_pubmsg(self, nick, message): return elif chunks[0] in ordername_list: self.add_order(nick, chunks) - elif chunks[0] == '%showob': - print('printing orderbook') - for o in self.db.execute('SELECT * FROM orderbook;').fetchall(): - print '(%s %s %d %d-%d %d %s)' % (o['counterparty'], o['ordertype'], o['oid'], - o['minsize'], o['maxsize'], o['txfee'], o['cjfee']) - print('done') - elif chunks[0] == '%unspent': - from pprint import pprint - pprint(self.wallet.unspent) - elif chunks[0] == '%fill': - counterparty = chunks[1] - oid = chunks[2] - amount = chunks[3] - my_utxo = chunks[4] - #!fill [counterparty] [oid] [amount] [utxo] - cjtx = CoinJoinTX(self, int(amount), [counterparty], [int(oid)], - [my_utxo], self.wallet.get_receive_addr(mixing_depth=1), - self.wallet.get_change_addr(mixing_depth=0), my_tx_fee) #self.connection.quit("Using irc.client.py") def on_welcome(self): - global algo_thread self.pubmsg(command_prefix + 'orderbook') - algo_thread = AlgoThread(self, self.wallet.unspent.copy()) - #algo_thread.start() def on_set_topic(self, newtopic): chunks = newtopic.split('|') @@ -322,13 +184,11 @@ def on_leave(self, nick): self.db.execute('DELETE FROM orderbook WHERE counterparty=?;', (nick,)) def main(): - print 'downloading wallet history' - wallet = Wallet(seed) - wallet.download_wallet_history() - wallet.find_unspent_addresses() + from socket import gethostname + nickname = 'cj-taker-' + btc.sha256(gethostname())[:6] print 'starting irc' - taker = Taker(wallet) + taker = Taker() taker.run(HOST, PORT, nickname, CHANNEL) if __name__ == "__main__": diff --git a/testtaker.py b/testtaker.py new file mode 100644 index 00000000..3556dfab --- /dev/null +++ b/testtaker.py @@ -0,0 +1,73 @@ + +from taker import * + + +my_tx_fee = 10000 + +class TestTaker(Taker): + + def __init__(self, wallet): + Taker.__init__(self) + self.wallet = wallet + + def on_privmsg(self, nick, message): + Taker.on_privmsg(self, nick, message) + #debug("privmsg nick=%s message=%s" % (nick, message)) + if message[0] != command_prefix: + return + for command in message[1:].split(command_prefix): + chunks = command.split(" ") + if chunks[0] == 'myparts': + utxo_list = chunks[1].split(',') + cj_addr = chunks[2] + change_addr = chunks[3] + self.cjtx.recv_tx_parts(nick, utxo_list, cj_addr, change_addr) + elif chunks[0] == 'sig': + sig = chunks[1] + self.cjtx.add_signature(sig) + + + def on_pubmsg(self, nick, message): + Taker.on_pubmsg(self, nick, message) + if message[0] != command_prefix: + return + for command in message[1:].split(command_prefix): + #commands starting with % are for testing and will be removed in the final version + chunks = command.split(" ") + if chunks[0] == '%showob': + print('printing orderbook') + for o in self.db.execute('SELECT * FROM orderbook;').fetchall(): + print '(%s %s %d %d-%d %d %s)' % (o['counterparty'], o['ordertype'], o['oid'], + o['minsize'], o['maxsize'], o['txfee'], o['cjfee']) + print('done') + elif chunks[0] == '%unspent': + from pprint import pprint + pprint(self.wallet.unspent) + elif chunks[0] == '%fill': + counterparty = chunks[1] + oid = chunks[2] + amount = chunks[3] + my_utxo = chunks[4] + #!fill [counterparty] [oid] [amount] [utxo] + print 'making cjtx' + self.cjtx = CoinJoinTX(self, int(amount), [counterparty], [int(oid)], + [my_utxo], self.wallet.get_receive_addr(mixing_depth=1), + self.wallet.get_change_addr(mixing_depth=0), my_tx_fee) +def main(): + import sys + seed = sys.argv[1] #btc.sha256('your brainwallet goes here') + from socket import gethostname + nickname = 'testtakr-' + btc.sha256(gethostname())[:6] + + print 'downloading wallet history' + wallet = Wallet(seed) + wallet.download_wallet_history() + wallet.find_unspent_addresses() + + print 'starting irc' + taker = TestTaker(wallet) + taker.run(HOST, PORT, nickname, CHANNEL) + +if __name__ == "__main__": + main() + print('done') diff --git a/tumbler.py b/tumbler.py new file mode 100644 index 00000000..caf92eab --- /dev/null +++ b/tumbler.py @@ -0,0 +1,108 @@ + + +algo_thread = None + +#how long to wait for all the orders to arrive before starting to do coinjoins +ORDER_ARRIVAL_WAIT_TIME = 2 + +def choose_order(cj_amount): + + sqlorders = db.execute('SELECT * FROM orderbook;').fetchall() + orders = [(o['counterparty'], o['oid'], calc_cj_fee(o['ordertype'], o['cjfee'], cj_amount)) + for o in sqlorders if cj_amount >= o['minsize'] or cj_amount <= o['maxsize']] + orders = sorted(orders, key=lambda k: k[2]) + print 'orders = ' + str(orders) + return orders[0] #choose the cheapest, later this will be chosen differently + +def choose_sweep_order(my_total_input, my_tx_fee): + ''' + choose an order given that we want to be left with no change + i.e. sweep an entire group of utxos + + solve for mychange = 0 + ABS FEE + mychange = totalin - cjamount - mytxfee - absfee + => cjamount = totalin - mytxfee - absfee + REL FEE + mychange = totalin - cjamount - mytxfee - relfee*cjamount + => 0 = totalin - mytxfee - cjamount*(1 + relfee) + => cjamount = (totalin - mytxfee) / (1 + relfee) + ''' + def calc_zero_change_cj_amount(ordertype, cjfee): + cj_amount = None + if ordertype == 'absorder': + cj_amount = my_total_input - my_tx_fee - cjfee + elif ordertype == 'relorder': + cj_amount = (my_total_input - my_tx_fee) / (Decimal(cjfee) + 1) + cj_amount = int(cj_amount.quantize(Decimal(1))) + else: + raise RuntimeError('unknown order type: ' + str(ordertype)) + return cj_amount + + sqlorders = db.execute('SELECT * FROM orderbook;').fetchall() + orders = [(o['counterparty'], o['oid'], calc_zero_change_cj_amount(o['ordertype'], o['cjfee']), + o['minsize'], o['maxsize']) for o in sqlorders] + #filter cj_amounts that are not in range + orders = [o[:3] for o in orders if o[2] >= o[3] and o[2] <= o[4]] + orders = sorted(orders, key=lambda k: k[2]) + print 'sweep orders = ' + str(orders) + return orders[-1] #choose one with the highest cj_amount, most left over after paying everything else + +#thread which does the buy-side algorithm +# chooses which coinjoins to initiate and when +class AlgoThread(threading.Thread): + def __init__(self, taker, initial_unspents): + threading.Thread.__init__(self) + self.daemon = True + self.taker = taker + self.initial_unspents = initial_unspents + self.finished_cj = False + + def finished_cj_callback(self): + self.finished_cj = True + print 'finished cj' + + def run(self): + global cjtx + time.sleep(ORDER_ARRIVAL_WAIT_TIME) + #while True: + if 1: + #wait for orders to arrive + #TODO just make this do one tx and then stop + if len(self.initial_unspents) == 0: + print 'finished mixing, closing...' + self.taker.shutdown() + #break + + #utxo, addrvalue = self.initial_unspents.popitem() + utxo, addrvalue = [(k, v) for k, v in self.initial_unspents.iteritems() if v['value'] == 200000000][0] + counterparty, oid, cj_amount = choose_sweep_order(addrvalue['value'], my_tx_fee) + self.finished_cj = False + cjtx = CoinJoinTX(self.taker, cj_amount, [counterparty], [int(oid)], + [utxo], self.taker.wallet.get_receive_addr(mixing_depth=1), None, + my_tx_fee, self.finished_cj_callback) + #algorithm for making + ''' + single_cj_amount = 112000000 + unspent = [] + for utxo, addrvalue in self.initial_unspents.iteritems(): + unspent.append({'value': addrvalue['value'], 'utxo': utxo}) + inputs = btc.select(unspent, single_cj_amount) + my_utxos = [i['utxo'] for i in inputs] + counterparty, oid = choose_order(single_cj_amount) + cjtx = CoinJoinTX(self.irc, int(single_cj_amount), [counterparty], [int(oid)], + my_utxos, wallet.get_receive_addr(mixing_depth=1), wallet.get_change_addr(mixing_depth=0)) + ''' + while not self.finished_cj: + time.sleep(5) + print 'woken algo thread' + +def main(): + print 'downloading wallet history' + wallet = Wallet(seed) + wallet.download_wallet_history() + wallet.find_unspent_addresses() + +if __name__ == "__main__": + main() + print('done') From 3bd09630b4e965bb994a5c50ad92c0725a55827a Mon Sep 17 00:00:00 2001 From: Belcher Date: Tue, 23 Dec 2014 14:32:33 +0000 Subject: [PATCH 011/409] changed test instructions --- README.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.txt b/README.txt index 888e9aa2..7219e8ea 100644 --- a/README.txt +++ b/README.txt @@ -9,7 +9,7 @@ HOWTO try do this for two wallet seeds, one for each taker and maker seeds are taken as a command line argument -2. join irc.freenode.net #joinmarket and run both taker.py and maker.py +2. join irc.freenode.net #joinmarket and run both testtaker.py and maker.py 3. when both bots join and have announced their orders, use this command to start a coinjoining From 0640fa4241a2089a20946bf660e31035c33fa9d5 Mon Sep 17 00:00:00 2001 From: Belcher Date: Tue, 23 Dec 2014 21:25:38 +0000 Subject: [PATCH 012/409] coded gui-taker.py to show the orderbook --- gui-taker.py | 115 +++++++++++++++++++++++++++++++++++++++++++++++++ irclib.py | 2 + orderbook.html | 48 +++++++++++++++++++++ taker.py | 3 ++ 4 files changed, 168 insertions(+) create mode 100644 gui-taker.py create mode 100644 orderbook.html diff --git a/gui-taker.py b/gui-taker.py new file mode 100644 index 00000000..8007172e --- /dev/null +++ b/gui-taker.py @@ -0,0 +1,115 @@ + +from taker import * + +import BaseHTTPServer, SimpleHTTPServer, threading +from decimal import Decimal + +import io +import base64 + +def create_depth_graph(db): + try: + import matplotlib.pyplot as plt + except ImportError: + return 'Install matplotlib to see graphs' + fig = plt.figure() + plt.plot(range(10), range(10)) + plt.grid() + plt.title('this graph shows nothing but there could be a graph about the orderbook here later') + + imbuf = io.BytesIO() + fig.savefig(imbuf, format='png') + b64 = base64.b64encode(imbuf.getvalue()) + return '' + #fd = open('fig.png', 'wb') + #fd.write(imbuf.getvalue()) + #fd.close() + +def do_nothing(arg): + return arg + +def ordertype_display(ordertype): + ordertypes = {'absorder': 'Absolute Fee', 'relorder': 'Relative Fee'} + return ordertypes[ordertype] + +def satoshi_to_unit(sat): + return str(Decimal(sat) / Decimal(1e8)) + +class OrderbookPageRequestHeader(SimpleHTTPServer.SimpleHTTPRequestHandler): + def __init__(self, request, client_address, base_server): + self.taker = base_server.taker + SimpleHTTPServer.SimpleHTTPRequestHandler.__init__(self, request, client_address, base_server) + + def create_orderbook_table(self): + result = '' + rows = self.taker.db.execute('SELECT * FROM orderbook;').fetchall() + for o in rows: + result += '' + order_keys_display = (('ordertype', ordertype_display), ('counterparty', do_nothing), + ('oid', str), ('cjfee', satoshi_to_unit), ('txfee', satoshi_to_unit), + ('minsize', satoshi_to_unit), ('maxsize', satoshi_to_unit)) + for key, displayer in order_keys_display: + result += '' + displayer(o[key]) + '' + result += '' + return len(rows), result + + def get_counterparty_count(self): + counterparties = self.taker.db.execute('SELECT DISTINCT counterparty FROM orderbook;').fetchall() + return str(len(counterparties)) + + def do_GET(self): + #SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self) + #print 'httpd received ' + self.path + ' request' + if self.path == '/': + fd = open('orderbook.html', 'r') + orderbook_fmt = fd.read() + fd.close() + ordercount, ordertable = self.create_orderbook_table() + replacements = { + 'ORDERCOUNT': str(ordercount), + 'CPCOUNT': self.get_counterparty_count(), + 'ORDERTABLE': ordertable, + 'DEPTHGRAPH': create_depth_graph(self.taker.db)} + orderbook_page = orderbook_fmt + for key, rep in replacements.iteritems(): + orderbook_page = orderbook_page.replace(key, rep) + + self.send_response(200) + self.send_header('Content-type', 'text/html') + self.send_header('Content-length', len(orderbook_page)) + self.end_headers() + self.wfile.write(orderbook_page) + + + +class HTTPDThread(threading.Thread): + def __init__(self, taker): + threading.Thread.__init__(self) + self.daemon = True + self.taker = taker + def run(self): + hostport = ('localhost', 62601) + httpd = BaseHTTPServer.HTTPServer(hostport, OrderbookPageRequestHeader) + httpd.taker = self.taker + print 'started http server, visit http://{0}:{1}/'.format(*hostport) + httpd.serve_forever() + + +class GUITaker(Taker): + def on_welcome(self): + Taker.on_welcome(self) + HTTPDThread(self).start() + +def main(): + from socket import gethostname + nickname = 'guitaker-' + btc.sha256(gethostname())[:6] + + print 'starting irc' + taker = GUITaker() + taker.run(HOST, PORT, nickname, CHANNEL) + + #create_depth_graph() + +if __name__ == "__main__": + main() + print('done') diff --git a/irclib.py b/irclib.py index 98b27182..b2845cfc 100644 --- a/irclib.py +++ b/irclib.py @@ -43,6 +43,7 @@ def on_pubmsg(self, nick, message): pass def on_welcome(self): pass def on_set_topic(self, newtopic): pass def on_leave(self, nick): pass + def on_disconnect(self): pass #TODO implement on_nick_change def close(self): @@ -156,6 +157,7 @@ def run(self, server, port, nick, channel, username='username', realname='realna finally: self.fd.close() self.sock.close() + self.on_disconnect() print 'disconnected irc' time.sleep(10) self.connect_attempts += 1 diff --git a/orderbook.html b/orderbook.html new file mode 100644 index 00000000..3ce4c062 --- /dev/null +++ b/orderbook.html @@ -0,0 +1,48 @@ + + + +Joinmarket Orderbook + + + + + +
+

Joinmarket Orderbook

+

ORDERCOUNT orders found by CPCOUNT counterparties

+ +DEPTHGRAPH + + + + + + + + + + + +ORDERTABLE +
TypeCounterpartyOrder IDFeeMiner Fee ContributionMinimum SizeMaximum Size
+ +
+ + + diff --git a/taker.py b/taker.py index 7080a8b4..b5462c0c 100644 --- a/taker.py +++ b/taker.py @@ -183,6 +183,9 @@ def on_set_topic(self, newtopic): def on_leave(self, nick): self.db.execute('DELETE FROM orderbook WHERE counterparty=?;', (nick,)) + def on_disconnect(self): + self.db.execute('DELETE FROM orderbook;') + def main(): from socket import gethostname nickname = 'cj-taker-' + btc.sha256(gethostname())[:6] From 074d5cf16f84bc48ae531a820cb54594aad7f08a Mon Sep 17 00:00:00 2001 From: Belcher Date: Tue, 23 Dec 2014 22:51:15 +0000 Subject: [PATCH 013/409] tested 3party coinjoins, added many debug() --- irclib.py | 11 ++++++----- maker.py | 16 +++++++++++----- taker.py | 13 +++++++------ testtaker.py | 20 +++++++++++++++++--- 4 files changed, 41 insertions(+), 19 deletions(-) diff --git a/irclib.py b/irclib.py index b2845cfc..94c4e722 100644 --- a/irclib.py +++ b/irclib.py @@ -1,7 +1,6 @@ -import socket -import threading -import time +import socket, threading, time +from common import debug PING_INTERVAL = 40 PING_TIMEOUT = 10 @@ -54,14 +53,16 @@ def shutdown(self): self.give_up = True def pubmsg(self, message): + #debug('pubmsg ' + message) self.send_raw("PRIVMSG " + self.channel + " :" + message) def privmsg(self, nick, message): - #print '>> ' + nick + ' :' + message + #debug('privmsg to ' + nick + ' ' + message) self.send_raw("PRIVMSG " + nick + " :" + message) def send_raw(self, line): - #print('>> ' + line) + if not line.startswith('PING LAG'): + debug('sendraw ' + line) self.sock.sendall(line + '\r\n') def __handle_privmsg(self, source, target, message): diff --git a/maker.py b/maker.py index f904391f..69e4201e 100644 --- a/maker.py +++ b/maker.py @@ -27,10 +27,11 @@ def __init__(self, maker, nick, oid, amount): self.cj_addr = maker.wallet.get_receive_addr(self.mixing_depth) self.change_addr = maker.wallet.get_change_addr(self.mixing_depth - 1) self.b64txparts = [] + debug('new cjorder nick=%s oid=%d amount=%d' % (nick, oid, amount)) #always a new address even if the order ends up never being # furfilled, you dont want someone pretending to fill all your # orders to find out which addresses you use - maker.privmsg(nick, command_prefix + 'myparts ' + ','.join(self.utxos) + ' ' + + maker.privmsg(nick, command_prefix + 'addrs ' + ','.join(self.utxos) + ' ' + self.cj_addr + ' ' + self.change_addr) def recv_tx_part(self, b64txpart): @@ -42,10 +43,14 @@ def recv_tx(self, nick, b64txpart): self.b64txparts.append(b64txpart) self.tx = base64.b64decode(''.join(self.b64txparts)).encode('hex') txd = btc.deserialize(self.tx) + import pprint + debug('obtained tx\n' + pprint.pformat(txd)) goodtx, errmsg = self.verify_unsigned_tx(txd) if not goodtx: + debug('not a good tx, reason=' + errmsg) self.maker.privmsg(nick, command_prefix + 'error ' + errmsg) return False + debug('goodtx') sigs = [] for index, ins in enumerate(txd['ins']): utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) @@ -58,6 +63,7 @@ def recv_tx(self, nick, b64txpart): print 'ERROR no private keys found' add_addr_notify(self.change_addr, self.unconfirm_callback, self.confirm_callback) + debug('sending sigs ' + str(sigs)) #TODO make this a function in irclib.py sigline = '' for sig in sigs: @@ -71,10 +77,12 @@ def recv_tx(self, nick, b64txpart): return True def unconfirm_callback(self, value): + debug('saw tx on network') to_cancel, to_announce = self.maker.on_tx_unconfirmed(self, value) self.handle_modified_orders(to_cancel, to_announce) def confirm_callback(self, confirmations, txid, value): + debug('tx in a block') to_cancel, to_announce = self.maker.on_tx_confirmed(self, confirmations, txid, value) self.handle_modified_orders(to_cancel, to_announce) @@ -146,7 +154,7 @@ def on_welcome(self): self.privmsg_all_orders(CHANNEL) def on_privmsg(self, nick, message): - #debug("privmsg nick=%s message=%s" % (nick, message)) + debug("privmsg nick=%s message=%s" % (nick, message)) if message[0] != command_prefix: return command_lines = message.split(command_prefix) @@ -168,7 +176,7 @@ def on_privmsg(self, nick, message): # using the same id again overwrites it, they'll be plenty of times when an order # has to be modified and its better to just have !order rather than !cancelorder then !order def on_pubmsg(self, nick, message): - #debug("pubmsg nick=%s message=%s" % (nick, message)) + debug("pubmsg nick=%s message=%s" % (nick, message)) if message[0] == command_prefix: chunks = message[1:].split(" ") if chunks[0] == '%quit' or chunks[0] == '%makerquit': @@ -245,7 +253,6 @@ def get_next_oid(self): #gets called when the tx is seen on the network #must return which orders to cancel or recreate def on_tx_unconfirmed(self, order, value): - print 'tx unconfirmed' return ([order.oid], []) #gets called when the tx is included in a block @@ -253,7 +260,6 @@ def on_tx_unconfirmed(self, order, value): # and i have to think about how that will work for both # the blockchain explorer api method and the bitcoid walletnotify def on_tx_confirmed(self, order, confirmations, txid, value): - print 'tx confirmed' to_announce = [] txd = btc.deserialize(order.tx) for i, out in enumerate(txd['outs']): diff --git a/taker.py b/taker.py index b5462c0c..b883c5a2 100644 --- a/taker.py +++ b/taker.py @@ -28,7 +28,7 @@ def __init__(self, taker, cj_amount, counterparties, oids, my_utxos, my_cj_addr, for c, oid in zip(counterparties, oids): taker.privmsg(c, command_prefix + 'fill ' + str(oid) + ' ' + str(cj_amount)) - def recv_tx_parts(self, nick, utxo_list, cj_addr, change_addr): + def recv_addrs(self, nick, utxo_list, cj_addr, change_addr): if nick not in self.nonrespondants: debug('nick(' + nick + ') not in nonrespondants ' + str(self.nonrespondants)) return @@ -64,13 +64,15 @@ def recv_tx_parts(self, nick, utxo_list, cj_addr, change_addr): self.outputs.append({'address': self.my_change_addr, 'value': my_change_value}) utxo_tx = [dict([('output', u)]) for u in sum(self.utxos.values(), [])] random.shuffle(self.outputs) - tx = btc.mktx(utxo_tx, self.outputs) + tx = btc.mktx(utxo_tx, self.outputs) + import pprint + debug('obtained tx\n' + pprint.pformat(btc.deserialize(tx))) txb64 = base64.b64encode(tx.decode('hex')) n = MAX_PRIVMSG_LEN txparts = [txb64[i:i+n] for i in range(0, len(txb64), n)] for p in txparts[:-1]: for nickk in self.active_orders.keys(): - self.taker.privmsg(nickk, command_prefix + 'txpart' + p) + self.taker.privmsg(nickk, command_prefix + 'txpart ' + p) for nickk in self.active_orders.keys(): self.taker.privmsg(nickk, command_prefix + 'tx ' + txparts[-1]) @@ -133,7 +135,7 @@ def add_order(self, nick, chunks): (nick, chunks[1], chunks[0], chunks[2], chunks[3], chunks[4], chunks[5])) def on_privmsg(self, nick, message): - #debug("privmsg nick=%s message=%s" % (nick, message)) + debug("privmsg nick=%s message=%s" % (nick, message)) if message[0] != command_prefix: return @@ -146,10 +148,9 @@ def on_privmsg(self, nick, message): # using the same id again overwrites it, they'll be plenty of times when an order # has to be modified and its better to just have !order rather than !cancelorder then !order def on_pubmsg(self, nick, message): - #print("pubmsg nick=%s message=%s" % (nick, message)) + debug("pubmsg nick=%s message=%s" % (nick, message)) if message[0] != command_prefix: return - for command in message[1:].split(command_prefix): #commands starting with % are for testing and will be removed in the final version chunks = command.split(" ") diff --git a/testtaker.py b/testtaker.py index 3556dfab..b9fedfc4 100644 --- a/testtaker.py +++ b/testtaker.py @@ -17,17 +17,18 @@ def on_privmsg(self, nick, message): return for command in message[1:].split(command_prefix): chunks = command.split(" ") - if chunks[0] == 'myparts': + if chunks[0] == 'addrs': utxo_list = chunks[1].split(',') cj_addr = chunks[2] change_addr = chunks[3] - self.cjtx.recv_tx_parts(nick, utxo_list, cj_addr, change_addr) + self.cjtx.recv_addrs(nick, utxo_list, cj_addr, change_addr) elif chunks[0] == 'sig': sig = chunks[1] self.cjtx.add_signature(sig) def on_pubmsg(self, nick, message): + print("pubmsg nick=%s message=%s" % (nick, message)) Taker.on_pubmsg(self, nick, message) if message[0] != command_prefix: return @@ -44,15 +45,28 @@ def on_pubmsg(self, nick, message): from pprint import pprint pprint(self.wallet.unspent) elif chunks[0] == '%fill': + #!fill [counterparty] [oid] [amount] [utxo] counterparty = chunks[1] oid = chunks[2] amount = chunks[3] my_utxo = chunks[4] - #!fill [counterparty] [oid] [amount] [utxo] print 'making cjtx' self.cjtx = CoinJoinTX(self, int(amount), [counterparty], [int(oid)], [my_utxo], self.wallet.get_receive_addr(mixing_depth=1), self.wallet.get_change_addr(mixing_depth=0), my_tx_fee) + elif chunks[0] == '%2fill': + #!2fill [amount] [utxo] [counterparty1] [oid1] [counterparty2] [oid2] + amount = int(chunks[1]) + my_utxo = chunks[2] + cp1 = chunks[3] + oid1 = int(chunks[4]) + cp2 = chunks[5] + oid2 = int(chunks[6]) + print 'creating cjtx' + self.cjtx = CoinJoinTX(self, amount, [cp1, cp2], [oid1, oid2], + [my_utxo], self.wallet.get_receive_addr(mixing_depth=1), + self.wallet.get_change_addr(mixing_depth=0), my_tx_fee) + def main(): import sys seed = sys.argv[1] #btc.sha256('your brainwallet goes here') From 782a22350b7bba600acd5d10533702c0cf4bb26f Mon Sep 17 00:00:00 2001 From: Belcher Date: Thu, 25 Dec 2014 14:16:18 +0000 Subject: [PATCH 014/409] various small additions and fixes --- README.txt | 10 ++++++++++ common.py | 43 ++++++++++++++++++++++++++++++---------- gui-taker.py | 19 +++++++++++++----- irclib.py | 5 ++++- maker.py | 56 +++++++++++++++++++++++++++------------------------- testtaker.py | 2 -- 6 files changed, 90 insertions(+), 45 deletions(-) diff --git a/README.txt b/README.txt index 7219e8ea..085eb045 100644 --- a/README.txt +++ b/README.txt @@ -111,6 +111,16 @@ e.g. single-tx.py which takes a single order, using it to send coins to some add e.g. gui-taker.py has a gui which shows the user the orderbook and they can easily fill and order and see other statistics, could be easily done by opening a http port and sending a html form and graphics +TODO +implement this the thing that gmaxwell wrote about in the original coinjoin post, as a kind of tumbler +"Isn't the anonymity set size limited by how many parties you can get in a single transaction?" + +"Not quite. The anonymity set size of a single transaction is limited by the number of parties in it, obviously. And transaction size limits as well as failure (retry) risk mean that really huge joint transactions would not be wise. But because these transactions are cheap, there is no limit to the number of transactions you can cascade. + +In particular, if you have can build transactions with m participants per transaction you can create a sequence of m*3 transactions which form a three-stage switching network that permits any of m^2 final outputs to have come from any of m^2 original inputs (e.g. using three stages of 32 transactions with 32 inputs each 1024 users can be joined with a total of 96 transactions). This allows the anonymity set to be any size, limited only by participation." +https://en.wikipedia.org/wiki/Clos_network +Not sure if it will actually be possible in this liquidity maker/taker system + TODO need to move onto the bip44 structure of HD wallets TODO diff --git a/common.py b/common.py index 67a19dec..ccc614c9 100644 --- a/common.py +++ b/common.py @@ -5,9 +5,11 @@ import threading HOST = 'irc.freenode.net' -CHANNEL = '#joinmarket' +CHANNEL = '#joinmarket-pit-test' PORT = 6667 +#for the mainnet its #joinmarket-pit + #TODO make this var all in caps command_prefix = '!' MAX_PRIVMSG_LEN = 400 @@ -27,17 +29,17 @@ def get_addr_vbyte(): else: return 0x00 -MAX_MIX_DEPTH = 2 #for now class Wallet(object): - def __init__(self, seed): + def __init__(self, seed, max_mix_depth=2): + self.max_mix_depth = max_mix_depth master = btc.bip32_master_key(seed) m_0 = btc.bip32_ckd(master, 0) - mixing_depth_keys = [btc.bip32_ckd(m_0, c) for c in range(MAX_MIX_DEPTH)] + mixing_depth_keys = [btc.bip32_ckd(m_0, c) for c in range(max_mix_depth)] self.keys = [(btc.bip32_ckd(m, 0), btc.bip32_ckd(m, 1)) for m in mixing_depth_keys] - #self.index = [[0, 0]]*MAX_MIX_DEPTH + #self.index = [[0, 0]]*max_mix_depth self.index = [] - for i in range(MAX_MIX_DEPTH): + for i in range(max_mix_depth): self.index.append([0, 0]) #example @@ -72,13 +74,35 @@ def get_key_from_addr(self, addr): else: return None + def remove_old_utxos(self, tx): + removed_utxos = {} + for ins in tx['ins']: + utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) + if utxo not in self.unspent: + continue + removed_utxos[utxo] = self.unspent[utxo] + del self.unspent[utxo] + return removed_utxos + + def add_new_utxos(self, tx, txid): + added_utxos = {} + for index, outs in enumerate(tx['outs']): + addr = btc.script_to_address(outs['script'], get_addr_vbyte()) + if addr not in self.addr_cache: + continue + addrdict = {'address': addr, 'value': outs['value']} + utxo = txid + ':' + str(index) + added_utxos[utxo] = addrdict + self.unspent[utxo] = addrdict + return added_utxos + def download_wallet_history(self, gaplimit=6): ''' sets Wallet internal indexes to be at the next unused address ''' addr_req_count = 20 - for mix_depth in range(MAX_MIX_DEPTH): + for mix_depth in range(self.max_mix_depth): for forchange in [0, 1]: unused_addr_count = 0 last_used_addr = '' @@ -117,7 +141,7 @@ def find_unspent_addresses(self): #TODO handle the case where there are so many addresses it cant # fit into one api call (>50 or so) addrs = {} - for m in range(MAX_MIX_DEPTH): + for m in range(self.max_mix_depth): for forchange in [0, 1]: for n in range(self.index[m][forchange]): addrs[self.get_addr(m, forchange, n)] = m @@ -146,8 +170,7 @@ def find_unspent_addresses(self): for dat in data: for u in dat['unspent']: self.unspent[u['tx']+':'+str(u['n'])] = {'address': - dat['address'], 'value': int(u['amount'].replace('.', '')), - 'mixdepth': addrs[dat['address']]} + dat['address'], 'value': int(u['amount'].replace('.', ''))} #awful way of doing this, but works for now # later use websocket api for people who dont download the blockchain diff --git a/gui-taker.py b/gui-taker.py index 8007172e..e3aee8a3 100644 --- a/gui-taker.py +++ b/gui-taker.py @@ -25,16 +25,25 @@ def create_depth_graph(db): #fd.write(imbuf.getvalue()) #fd.close() -def do_nothing(arg): +def do_nothing(arg, order): return arg -def ordertype_display(ordertype): +def ordertype_display(ordertype, order): ordertypes = {'absorder': 'Absolute Fee', 'relorder': 'Relative Fee'} return ordertypes[ordertype] -def satoshi_to_unit(sat): +def cjfee_display(cjfee, order): + if order['ordertype'] == 'absorder': + return satoshi_to_unit(cjfee, order) + elif order['ordertype'] == 'relorder': + return str(float(cjfee) * 100) + '%' + +def satoshi_to_unit(sat, order): return str(Decimal(sat) / Decimal(1e8)) +def order_str(s, order): + return str(s) + class OrderbookPageRequestHeader(SimpleHTTPServer.SimpleHTTPRequestHandler): def __init__(self, request, client_address, base_server): self.taker = base_server.taker @@ -46,10 +55,10 @@ def create_orderbook_table(self): for o in rows: result += '' order_keys_display = (('ordertype', ordertype_display), ('counterparty', do_nothing), - ('oid', str), ('cjfee', satoshi_to_unit), ('txfee', satoshi_to_unit), + ('oid', order_str), ('cjfee', cjfee_display), ('txfee', satoshi_to_unit), ('minsize', satoshi_to_unit), ('maxsize', satoshi_to_unit)) for key, displayer in order_keys_display: - result += '' + displayer(o[key]) + '' + result += '' + displayer(o[key], o) + '' result += '' return len(rows), result diff --git a/irclib.py b/irclib.py index 94c4e722..4ba26762 100644 --- a/irclib.py +++ b/irclib.py @@ -147,7 +147,10 @@ def run(self, server, port, nick, channel, username='username', realname='realna self.send_raw('NICK ' + nick) try: while 1: - line = self.fd.readline() + try: + line = self.fd.readline() + except AttributeError as e: + raise IOError(repr(e)) if line == None: break if len(line) == 0: diff --git a/maker.py b/maker.py index 69e4201e..d1015cd7 100644 --- a/maker.py +++ b/maker.py @@ -4,8 +4,8 @@ import irclib import bitcoin as btc import sys -import sqlite3 -import base64 +import base64, pprint + from socket import gethostname nickname = 'cj-maker-' + btc.sha256(gethostname())[:6] @@ -20,12 +20,12 @@ def __init__(self, maker, nick, oid, amount): if amount <= order['minsize'] or amount >= order['maxsize']: maker.privmsg(nick, command_prefix + 'error amount out of range') #TODO logic for this error causing the order to be removed from list of open orders - self.utxos, self.mixing_depth = maker.oid_to_order(oid, amount) + self.utxos, cj_mixing_depth, change_mixing_depth = maker.oid_to_order(oid, amount) self.ordertype = order['ordertype'] self.txfee = order['txfee'] self.cjfee = order['cjfee'] - self.cj_addr = maker.wallet.get_receive_addr(self.mixing_depth) - self.change_addr = maker.wallet.get_change_addr(self.mixing_depth - 1) + self.cj_addr = maker.wallet.get_receive_addr(cj_mixing_depth) + self.change_addr = maker.wallet.get_change_addr(change_mixing_depth) self.b64txparts = [] debug('new cjorder nick=%s oid=%d amount=%d' % (nick, oid, amount)) #always a new address even if the order ends up never being @@ -41,23 +41,23 @@ def recv_tx_part(self, b64txpart): def recv_tx(self, nick, b64txpart): self.b64txparts.append(b64txpart) - self.tx = base64.b64decode(''.join(self.b64txparts)).encode('hex') - txd = btc.deserialize(self.tx) + txhex = base64.b64decode(''.join(self.b64txparts)).encode('hex') + self.tx = btc.deserialize(txhex) import pprint - debug('obtained tx\n' + pprint.pformat(txd)) - goodtx, errmsg = self.verify_unsigned_tx(txd) + debug('obtained tx\n' + pprint.pformat(self.tx)) + goodtx, errmsg = self.verify_unsigned_tx(self.tx) if not goodtx: debug('not a good tx, reason=' + errmsg) self.maker.privmsg(nick, command_prefix + 'error ' + errmsg) return False debug('goodtx') sigs = [] - for index, ins in enumerate(txd['ins']): + for index, ins in enumerate(self.tx['ins']): utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) if utxo not in self.maker.wallet.unspent: continue addr = self.maker.wallet.unspent[utxo]['address'] - txs = btc.sign(self.tx, index, self.maker.wallet.get_key_from_addr(addr)) + txs = btc.sign(txhex, index, self.maker.wallet.get_key_from_addr(addr)) sigs.append(base64.b64encode(btc.deserialize(txs)['ins'][index]['script'].decode('hex'))) if len(sigs) == 0: print 'ERROR no private keys found' @@ -76,15 +76,17 @@ def recv_tx(self, nick, b64txpart): self.maker.privmsg(nick, sigline) return True - def unconfirm_callback(self, value): - debug('saw tx on network') - to_cancel, to_announce = self.maker.on_tx_unconfirmed(self, value) + def unconfirm_callback(self, balance): + removed_utxos = self.maker.wallet.remove_old_utxos(self.tx) + debug('saw tx on network, removed_utxos=\n' + pprint.pformat(removed_utxos)) + to_cancel, to_announce = self.maker.on_tx_unconfirmed(self, balance, removed_utxos) self.handle_modified_orders(to_cancel, to_announce) - def confirm_callback(self, confirmations, txid, value): - debug('tx in a block') + def confirm_callback(self, confirmations, txid, balance): + added_utxos = self.maker.wallet.add_new_utxos(self.tx, txid) + debug('tx in a block, added_utxos=' + pprint.pformat(added_utxos)) to_cancel, to_announce = self.maker.on_tx_confirmed(self, - confirmations, txid, value) + confirmations, txid, balance, added_utxos) self.handle_modified_orders(to_cancel, to_announce) def handle_modified_orders(self, to_cancel, to_announce): @@ -222,7 +224,7 @@ def create_my_orders(self): for utxo, addrvalue in self.wallet.unspent.iteritems(): order = {'oid': self.get_next_oid(), 'ordertype': 'absorder', 'minsize': 0, 'maxsize': addrvalue['value'], 'txfee': 10000, 'cjfee': 100000, - 'utxo': utxo, 'mixdepth': addrvalue['mixdepth']} + 'utxo': utxo, 'mixdepth': self.wallet.addr_cache[addrvalue['address']][0]} orderlist.append(order) #yes you can add keys there that are never used by the rest of the Maker code # so im adding utxo and mixdepth here @@ -243,8 +245,9 @@ def oid_to_order(self, oid, amount): ''' order = [o for o in self.orderlist if o['oid'] == oid][0] - mixing_depth = order['mixdepth'] + 1 - return [order['utxo']], mixing_depth + cj_mixing_depth = order['mixdepth'] + 1 + change_mixing_depth = order['mixdepth'] + return [order['utxo']], cj_mixing_depth, change_mixing_depth def get_next_oid(self): self.nextoid += 1 @@ -252,24 +255,23 @@ def get_next_oid(self): #gets called when the tx is seen on the network #must return which orders to cancel or recreate - def on_tx_unconfirmed(self, order, value): - return ([order.oid], []) + def on_tx_unconfirmed(self, cjorder, balance, removed_utxos): + return ([cjorder.oid], []) #gets called when the tx is included in a block #must return which orders to cancel or recreate # and i have to think about how that will work for both # the blockchain explorer api method and the bitcoid walletnotify - def on_tx_confirmed(self, order, confirmations, txid, value): + def on_tx_confirmed(self, cjorder, confirmations, txid, balance, added_utxos): to_announce = [] - txd = btc.deserialize(order.tx) - for i, out in enumerate(txd['outs']): + for i, out in enumerate(cjorder.tx['outs']): addr = btc.script_to_address(out['script'], get_addr_vbyte()) - if addr == order.change_addr: + if addr == cjorder.change_addr: neworder = {'oid': self.get_next_oid(), 'ordertype': 'absorder', 'minsize': 0, 'maxsize': out['value'], 'txfee': 10000, 'cjfee': 100000, 'utxo': txid + ':' + str(i)} to_announce.append(neworder) - if addr == order.cj_addr: + if addr == cjorder.cj_addr: neworder = {'oid': self.get_next_oid(), 'ordertype': 'absorder', 'minsize': 0, 'maxsize': out['value'], 'txfee': 10000, 'cjfee': 100000, 'utxo': txid + ':' + str(i)} diff --git a/testtaker.py b/testtaker.py index b9fedfc4..101206a9 100644 --- a/testtaker.py +++ b/testtaker.py @@ -1,7 +1,6 @@ from taker import * - my_tx_fee = 10000 class TestTaker(Taker): @@ -28,7 +27,6 @@ def on_privmsg(self, nick, message): def on_pubmsg(self, nick, message): - print("pubmsg nick=%s message=%s" % (nick, message)) Taker.on_pubmsg(self, nick, message) if message[0] != command_prefix: return From 88f086f29e740e8c9c02229002324b975037da24 Mon Sep 17 00:00:00 2001 From: Belcher Date: Thu, 25 Dec 2014 16:13:05 +0000 Subject: [PATCH 015/409] bugfix --- maker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maker.py b/maker.py index d1015cd7..efecacfe 100644 --- a/maker.py +++ b/maker.py @@ -98,7 +98,7 @@ def handle_modified_orders(self, to_cancel, to_announce): self.maker.pubmsg(''.join(clines)) if len(to_announce) > 0: self.maker.privmsg_all_orders(CHANNEL, to_announce) - self.maker.orderlist.append(to_announce) + self.maker.orderlist += to_announce def verify_unsigned_tx(self, txd): tx_utxos = set([ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) for ins in txd['ins']]) From 6ee6886f668e80f0a1ecd4aa354de9b5619d3f2c Mon Sep 17 00:00:00 2001 From: Belcher Date: Thu, 25 Dec 2014 17:50:50 +0000 Subject: [PATCH 016/409] shows you address balances now --- bip32-tool.py | 110 ++++++++++++++++---------------------------------- 1 file changed, 34 insertions(+), 76 deletions(-) diff --git a/bip32-tool.py b/bip32-tool.py index d6e26b82..fbe1ff3a 100644 --- a/bip32-tool.py +++ b/bip32-tool.py @@ -1,6 +1,9 @@ import bitcoin as btc +from common import Wallet + import sys +from optparse import OptionParser #structure for cj market wallet # m/0/ root key @@ -12,87 +15,42 @@ # m/0/n/1/k kth change address, for mixing depth n -seed = sys.argv[1] #btc.sha256('dont use brainwallets') +parser = OptionParser(usage='usage: %prog [options] [seed]', + description='Does useful little lasts involving your bip32 wallet.') +parser.add_option('-p', '--privkey', action='store_true', dest='showprivkey', + help='print private key along with address') +parser.add_option('-m', '--maxmixdepth', action='store', type='int', dest='maxmixdepth', + default=2, help='maximum mixing depth to look for') +parser.add_option('-g', '--gap-limit', action='store', dest='gaplimit', + help='gap limit for wallet', default=6) +(options, args) = parser.parse_args() + +if len(args) != 1: + parser.error('Needs a seed') + sys.exit(0) +seed = args[0] #seed = '256 bits of randomness' -print_privkey = len(sys.argv) > 2 - -master = btc.bip32_master_key(seed)#, btc.TESTNET_PRIVATE) -print 'master = ' + master +print_privkey = options.showprivkey -addr_vbyte = 0x6f #testnet +wallet = Wallet(seed, options.maxmixdepth) +print 'downloading wallet history' +wallet.download_wallet_history(options.gaplimit) +wallet.find_unspent_addresses() -m_0 = btc.bip32_ckd(master, 0) -for n in range(2): - print 'mixing depth ' + str(n) + ' m/0/' + str(n) + '/' - m_0_n = btc.bip32_ckd(m_0, n) - for forchange in range(2): +for m in range(wallet.max_mix_depth): + print 'mixing depth %d m/0/%d/' % (m, m) + for forchange in [0, 1]: print(' ' + ('receive' if forchange==0 else 'change') + - ' addresses m/0/%d/%d/' % (n, forchange)) - m_0_n_c = btc.bip32_ckd(m_0_n, forchange) - for k in range(3): - m_0_n_c_k = btc.bip32_ckd(m_0_n_c, k) - priv = btc.bip32_extract_key(m_0_n_c_k) - output = btc.privtoaddr(priv, addr_vbyte) - if print_privkey: - output += ' ' + btc.encode_privkey(priv, 'wif', addr_vbyte) - print' m/0/%d/%d/%d/ ' % (n, forchange, k) + output - - -''' -#default key on http://bip32.org/ -m_priv =\ -'xprv9s21ZrQH143K2JF8RafpqtKiTbsbaxEeUaMnNHsm5o6wCW3z8ySyH4UxFVSfZ8n7ESu7fgir8imbZKLYVBxFPND1pniTZ81vKfd45EHKX73' - -m_pub = btc.bip32_privtopub(m_priv) -print 'm_pub = ' + m_pub - -print 'prv(hex) = ' + btc.bip32_extract_key(m_priv) -print 'prv(wif) = ' + btc.encode_privkey(btc.bip32_extract_key(m_priv), 'wif_compressed') -print 'pub(hex) = ' + btc.bip32_extract_key(m_pub) -print 'addr = ' + btc.pubtoaddr(btc.bip32_extract_key(m_pub)) - -def print_pub_priv(prefix, priv): - pub = btc.bip32_privtopub(priv) - print prefix - print 'pub = ' + pub - print 'prv = ' + priv - -#bip32 test vector -print '\nbip32 test vector\n' -m_priv =\ -'xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi' - -#i_H = i + 2**31 -chain_m_0h = btc.bip32_ckd(m_priv, 0 + 2**31) -print_pub_priv('chain m/0H', chain_m_0h) - -chain_m_0h_1 = btc.bip32_ckd(chain_m_0h, 1) -print_pub_priv('chain m/0H/1', chain_m_0h_1) - -chain_m_0h_1_2h = btc.bip32_ckd(chain_m_0h_1, 2 + 2**31) -print_pub_priv('chain m/0H/1/2H', chain_m_0h_1_2h) - - -#bip32 test vector 2 -print '\nbip32 test vector 2\n' -m_priv =\ -'xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U' - -print 'master(hex) = ' + btc.bip32_extract_key(m_priv) - -chain_m_0 = btc.bip32_ckd(m_priv, 0) -print_pub_priv('chain m/0', chain_m_0) + ' addresses m/0/%d/%d/' % (m, forchange)) + for k in range(wallet.index[m][forchange] + options.gaplimit): + addr = wallet.get_addr(m, forchange, k) + balance = 0.0 + for addrvalue in wallet.unspent.values(): + if addr == addrvalue['address']: + balance += addrvalue['value'] + used = ('used' if k < wallet.index[m][forchange] else ' new') + print ' m/0/%d/%d/%2d %s %s %.8fbtc' % (m, forchange, k, addr, used, balance/1e8) -chain_m_0_214blahH = btc.bip32_ckd(chain_m_0, 2147483647 + 2**31) -print_pub_priv('chain m/0/2147483647H', chain_m_0_214blahH) -chain_m_0_214blahH_1 = btc.bip32_ckd(chain_m_0_214blahH, 1) -print_pub_priv('chain m/0/2147483647H/1', chain_m_0_214blahH_1) -''' -''' -seed = '256 bits of randomness' -m_priv = btc.bip32_master_key(seed) -print 'new seed = ' + m_priv -''' From dc1f7aa5cd264847804429c662af81a28b6db710 Mon Sep 17 00:00:00 2001 From: Belcher Date: Thu, 25 Dec 2014 22:25:54 +0000 Subject: [PATCH 017/409] combine method to combine small utxo into big one --- README.txt | 8 +++---- bip32-tool.py | 60 ++++++++++++++++++++++++++++++++++++--------------- 2 files changed, 47 insertions(+), 21 deletions(-) diff --git a/README.txt b/README.txt index 085eb045..34a4a7c0 100644 --- a/README.txt +++ b/README.txt @@ -99,12 +99,12 @@ TODO combine the taker and maker code into one file where you can make different bot which combine both roles e.g. tumbler.py repeatedly takes orders on the same coins again and again in an effort to improve privacy and break the link between them, make sure to split up and combine them again - in random amounts, because the income-collector will also be splitting and combining coins - random intervals between blocks included might be worth it too, since income-collector.py + in random amounts, because the yield-generator will also be splitting and combining coins + random intervals between blocks included might be worth it too, since yield-generator.py will appear to have coins which dont get mixed again for a while e.g. patient-tumbler.py which waits a while being a maker, then just starts to take orders after a time limit for people who want to mix coins but dont mind waiting until a fixed upper time limit -e.g. income-collector.py which acts as a maker solely for the purpose of making money +e.g. yield-generator.py which acts as a maker solely for the purpose of making money might need to take orders at some point, for very small outputs which have a small probability of being filled e.g. single-tx.py which takes a single order, using it to send coins to some address typically as a payment, so this is what the electrum plugin would look like @@ -128,7 +128,7 @@ probably a good idea to have a debug.log where loads of information is dumped TODO code a gui where a human can see the state of the orderbook and easily choose orders to fill -code a gui that easily explains to a human how they can choose a fee for their income-collector.py +code a gui that easily explains to a human how they can choose a fee for their yield-generator.py both are important for market forces, since markets emerge from human decisions and actions #TODO add random delays to the orderbook stuff so there isnt such a traffic spike when a new bot joins diff --git a/bip32-tool.py b/bip32-tool.py index fbe1ff3a..457e4011 100644 --- a/bip32-tool.py +++ b/bip32-tool.py @@ -15,8 +15,11 @@ # m/0/n/1/k kth change address, for mixing depth n -parser = OptionParser(usage='usage: %prog [options] [seed]', - description='Does useful little lasts involving your bip32 wallet.') +parser = OptionParser(usage='usage: %prog [options] [seed] [method]', + description='Does useful little lasts involving your bip32 wallet. The' + + ' method is one of the following: Display- shows all addresses and balances' + + '. Combine- combines all utxos into one output for each mixing level. Used for' + + ' testing and is detrimental to privacy.') parser.add_option('-p', '--privkey', action='store_true', dest='showprivkey', help='print private key along with address') parser.add_option('-m', '--maxmixdepth', action='store', type='int', dest='maxmixdepth', @@ -25,10 +28,13 @@ help='gap limit for wallet', default=6) (options, args) = parser.parse_args() -if len(args) != 1: +if len(args) < 1: parser.error('Needs a seed') sys.exit(0) seed = args[0] + +method = ('display' if len(args) == 1 else args[1].lower()) + #seed = '256 bits of randomness' print_privkey = options.showprivkey @@ -38,19 +44,39 @@ wallet.download_wallet_history(options.gaplimit) wallet.find_unspent_addresses() -for m in range(wallet.max_mix_depth): - print 'mixing depth %d m/0/%d/' % (m, m) - for forchange in [0, 1]: - print(' ' + ('receive' if forchange==0 else 'change') + - ' addresses m/0/%d/%d/' % (m, forchange)) - for k in range(wallet.index[m][forchange] + options.gaplimit): - addr = wallet.get_addr(m, forchange, k) - balance = 0.0 - for addrvalue in wallet.unspent.values(): - if addr == addrvalue['address']: +if method == 'display': + for m in range(wallet.max_mix_depth): + print 'mixing depth %d m/0/%d/' % (m, m) + for forchange in [0, 1]: + print(' ' + ('receive' if forchange==0 else 'change') + + ' addresses m/0/%d/%d/' % (m, forchange)) + for k in range(wallet.index[m][forchange] + options.gaplimit): + addr = wallet.get_addr(m, forchange, k) + balance = 0.0 + for addrvalue in wallet.unspent.values(): + if addr == addrvalue['address']: + balance += addrvalue['value'] + used = ('used' if k < wallet.index[m][forchange] else ' new') + print ' m/0/%d/%d/%02d %s %s %.8fbtc' % (m, forchange, k, addr, used, balance/1e8) +elif method == 'combine': + ins = [] + outs = [] + for m in range(wallet.max_mix_depth): + for forchange in [0, 1]: + balance = 0 + for k in range(wallet.index[m][forchange]): + addr = wallet.get_addr(m, forchange, k) + for utxo, addrvalue in wallet.unspent.iteritems(): + if addr != addrvalue['address']: + continue + ins.append({'output': utxo}) balance += addrvalue['value'] - used = ('used' if k < wallet.index[m][forchange] else ' new') - print ' m/0/%d/%d/%2d %s %s %.8fbtc' % (m, forchange, k, addr, used, balance/1e8) - - + if balance > 0: + destaddr = wallet.get_addr(m, forchange, wallet.index[m][forchange]) + outs.append({'address': destaddr, 'value': balance}) + tx = btc.mktx(ins, outs) + for index, utxo in enumerate(ins): + addr = wallet.unspent[utxo['output']]['address'] + tx = btc.sign(tx, index, wallet.get_key_from_addr(addr)) + print tx From 3ea158bea616a537ca0d8b62c3c05e2649bb8a43 Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 26 Dec 2014 00:31:12 +0000 Subject: [PATCH 018/409] yield generator, make money from your coins --- maker.py | 6 ++- yield-generator.py | 110 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 yield-generator.py diff --git a/maker.py b/maker.py index efecacfe..97d6482a 100644 --- a/maker.py +++ b/maker.py @@ -84,7 +84,7 @@ def unconfirm_callback(self, balance): def confirm_callback(self, confirmations, txid, balance): added_utxos = self.maker.wallet.add_new_utxos(self.tx, txid) - debug('tx in a block, added_utxos=' + pprint.pformat(added_utxos)) + debug('tx in a block, added_utxos=\n' + pprint.pformat(added_utxos)) to_cancel, to_announce = self.maker.on_tx_confirmed(self, confirmations, txid, balance, added_utxos) self.handle_modified_orders(to_cancel, to_announce) @@ -98,6 +98,10 @@ def handle_modified_orders(self, to_cancel, to_announce): self.maker.pubmsg(''.join(clines)) if len(to_announce) > 0: self.maker.privmsg_all_orders(CHANNEL, to_announce) + for ann in to_announce: + oldorder_s = [order for order in self.maker.orderlist if order['oid'] == ann['oid']] + if len(oldorder_s) > 0: + self.maker.orderlist.remove(oldorder_s[0]) self.maker.orderlist += to_announce def verify_unsigned_tx(self, txd): diff --git a/yield-generator.py b/yield-generator.py new file mode 100644 index 00000000..8f59e683 --- /dev/null +++ b/yield-generator.py @@ -0,0 +1,110 @@ +#! /usr/bin/env python + +from maker import * +import bitcoin as btc + +import pprint +import pdb + +txfee = 1000 +cjfee = '0.01' # 1% fee +mix_levels = 4 + +#is a maker for the purposes of generating a yield from held bitcoins +#for each mixing level, adds up the balance of all the addresses and put up +# a relative fee order, oid=mix depth +class YieldGenerator(Maker): + def __init__(self, wallet): + Maker.__init__(self, wallet) + + def get_mix_utxo_list(self): + mix_utxo_list = {} + for utxo, addrvalue in self.wallet.unspent.iteritems(): + mixdepth = self.wallet.addr_cache[addrvalue['address']][0] + if mixdepth not in mix_utxo_list: + mix_utxo_list[mixdepth] = [] + mix_utxo_list[mixdepth].append(utxo) + return mix_utxo_list + + def create_my_orders(self): + mix_utxo_list = self.get_mix_utxo_list() + orderlist = [] + for mixdepth, utxo_list in mix_utxo_list.iteritems(): + total_value = 0 + for utxo in utxo_list: + total_value += self.wallet.unspent[utxo]['value'] + order = {'oid': mixdepth, 'ordertype': 'relorder', 'minsize': 0, + 'maxsize': total_value, 'txfee': txfee, 'cjfee': cjfee, + 'utxos': utxo_list} + orderlist.append(order) + return orderlist + + def oid_to_order(self, oid, amount): + order = [o for o in self.orderlist if o['oid'] == oid][0] + unspent = [{'utxo': utxo, 'value': self.wallet.unspent[utxo]['value']} + for utxo in order['utxos']] + inputs = btc.select(unspent, amount) + mixdepth = oid + cj_mixdepth = (mixdepth + 1) % self.wallet.max_mix_depth + change_mixdepth = mixdepth + return [i['utxo'] for i in inputs], cj_mixdepth, change_mixdepth + + def on_tx_unconfirmed(self, cjorder, balance, removed_utxos): + #want to replace the current relorders with the same + # thing except reduced maxvalue to take into account the use + source_mixdepth = self.wallet.addr_cache[removed_utxos.values()[0]['address']][0] + debug('source mixdepth = %d' % (source_mixdepth)) + removed_utxos_balance = sum([addrvalue['value'] for addrvalue in removed_utxos.values()]) + debug('removed_utxos_balance = %d' % (removed_utxos_balance)) + + oldorder = [order for order in self.orderlist if order['oid'] == source_mixdepth][0] + neworder = oldorder.copy() + neworder['maxsize'] = oldorder['maxsize'] - removed_utxos_balance + [neworder['utxos'].remove(u) for u in removed_utxos.keys()] + #TODO if the maxsize left is zero or below the dust limit, just cancel the order + debug('neworder\n' + pprint.pformat(neworder)) + + #to_announce = self.create_my_orders() + return ([], [neworder]) + + def on_tx_confirmed(self, cjorder, confirmations, txid, balance, added_utxos): + #add the new available utxos to the maxsize and announce it + # if we dont have a mixdepth of that level, make one + + to_announce = [] + for utxo, addrvalue in added_utxos.iteritems(): + mixdepth = self.wallet.addr_cache[addrvalue['address']][0] + debug('mixdepth=%d' % (mixdepth)) + oldorder_search = [order for order in self.orderlist if order['oid'] == mixdepth] + debug('len=' + str(len(oldorder_search)) + ' oldorder_search=\n' + pprint.pformat(oldorder_search)) + if len(oldorder_search) == 0: + #there were no existing orders at that mixing depth + neworder = {'oid': mixdepth, 'ordertype': 'relorder', 'minsize': 0, + 'maxsize': addrvalue['value'], 'txfee': txfee, 'cjfee': cjfee, + 'utxos': [utxo]} + else: + #assert len(oldorder_search) == 1 + oldorder = oldorder_search[0] + neworder = oldorder.copy() + neworder['maxsize'] = oldorder['maxsize'] + addrvalue['value'] + neworder['utxos'].append(utxo) + to_announce.append(neworder) + return ([], to_announce) + +def main(): + print 'downloading wallet history' + wallet = Wallet(seed, max_mix_depth = mix_levels) + wallet.download_wallet_history() + wallet.find_unspent_addresses() + + + from socket import gethostname + nickname = 'yield-gen-' + btc.sha256(gethostname())[:6] + + maker = YieldGenerator(wallet) + print 'connecting to irc' + maker.run(HOST, PORT, nickname, CHANNEL) + +if __name__ == "__main__": + main() + print('done') From 95fe7b2df89fc3a1be72ea92abe0593ff468e9c7 Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 26 Dec 2014 02:34:30 +0000 Subject: [PATCH 019/409] improvements to web browser interface --- gui-taker.py | 92 +++++++++++++++++++++++++++++++++++--------------- orderbook.html | 50 +++++++++++++++++---------- 2 files changed, 97 insertions(+), 45 deletions(-) diff --git a/gui-taker.py b/gui-taker.py index e3aee8a3..eb1c6c73 100644 --- a/gui-taker.py +++ b/gui-taker.py @@ -7,23 +7,46 @@ import io import base64 -def create_depth_graph(db): +tableheading = ''' + + + + + + + + + + +''' + +def calc_depth_data(db, value): + pass + +def calc_order_size_data(db): + return ordersizes + +def create_size_histogram(db): try: import matplotlib.pyplot as plt except ImportError: return 'Install matplotlib to see graphs' + rows = db.execute('SELECT maxsize FROM orderbook;').fetchall() + ordersizes = [r['maxsize']/1e8 for r in rows] + fig = plt.figure() - plt.plot(range(10), range(10)) + plt.hist(ordersizes, 30, histtype='bar', rwidth=0.8) plt.grid() - plt.title('this graph shows nothing but there could be a graph about the orderbook here later') + #plt.title('Order size distribution') + plt.xlabel('Order sizes / btc') + plt.ylabel('Frequency') + return get_graph_html(fig) +def get_graph_html(fig): imbuf = io.BytesIO() fig.savefig(imbuf, format='png') b64 = base64.b64encode(imbuf.getvalue()) return '' - #fd = open('fig.png', 'wb') - #fd.write(imbuf.getvalue()) - #fd.close() def do_nothing(arg, order): return arg @@ -53,13 +76,13 @@ def create_orderbook_table(self): result = '' rows = self.taker.db.execute('SELECT * FROM orderbook;').fetchall() for o in rows: - result += '' + result += ' \n' order_keys_display = (('ordertype', ordertype_display), ('counterparty', do_nothing), ('oid', order_str), ('cjfee', cjfee_display), ('txfee', satoshi_to_unit), ('minsize', satoshi_to_unit), ('maxsize', satoshi_to_unit)) for key, displayer in order_keys_display: - result += '' - result += '' + result += ' \n' + result += ' \n' return len(rows), result def get_counterparty_count(self): @@ -69,27 +92,42 @@ def get_counterparty_count(self): def do_GET(self): #SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self) #print 'httpd received ' + self.path + ' request' - if self.path == '/': - fd = open('orderbook.html', 'r') - orderbook_fmt = fd.read() - fd.close() - ordercount, ordertable = self.create_orderbook_table() - replacements = { - 'ORDERCOUNT': str(ordercount), - 'CPCOUNT': self.get_counterparty_count(), - 'ORDERTABLE': ordertable, - 'DEPTHGRAPH': create_depth_graph(self.taker.db)} - orderbook_page = orderbook_fmt - for key, rep in replacements.iteritems(): - orderbook_page = orderbook_page.replace(key, rep) - self.send_response(200) - self.send_header('Content-type', 'text/html') - self.send_header('Content-length', len(orderbook_page)) - self.end_headers() - self.wfile.write(orderbook_page) + pages = ['/', '/ordersize'] + if self.path not in pages: + return + fd = open('orderbook.html', 'r') + orderbook_fmt = fd.read() + fd.close() + if self.path == '/': + ordercount, ordertable = self.create_orderbook_table() + replacements = { + 'PAGETITLE': 'Joinmarket Browser Interface', + 'MAINHEADING': 'Joinmarket Orderbook', + 'SECONDHEADING': (str(ordercount) + ' orders found by ' + + self.get_counterparty_count() + ' counterparties'), + 'MAINBODY': tableheading + ordertable + '
TypeCounterpartyOrder IDFeeMiner Fee ContributionMinimum SizeMaximum Size
' + displayer(o[key], o) + '
' + displayer(o[key], o) + '
\n' + } + elif self.path == '/ordersize': + replacements = { + 'PAGETITLE': 'Joinmarket Browser Interface', + 'MAINHEADING': 'Order Sizes', + 'SECONDHEADING': 'Order Size Histogram', + 'MAINBODY': create_size_histogram(self.taker.db) + } + + + orderbook_page = orderbook_fmt + for key, rep in replacements.iteritems(): + orderbook_page = orderbook_page.replace(key, rep) + + self.send_response(200) + self.send_header('Content-Type', 'text/html') + self.send_header('Content-Length', len(orderbook_page)) + self.end_headers() + self.wfile.write(orderbook_page) class HTTPDThread(threading.Thread): def __init__(self, taker): diff --git a/orderbook.html b/orderbook.html index 3ce4c062..2495c727 100644 --- a/orderbook.html +++ b/orderbook.html @@ -1,7 +1,7 @@ -Joinmarket Orderbook +PAGETITLE
-

Joinmarket Orderbook

-

ORDERCOUNT orders found by CPCOUNT counterparties

- -DEPTHGRAPH - - - - - - - - - - - -ORDERTABLE -
TypeCounterpartyOrder IDFeeMiner Fee ContributionMinimum SizeMaximum Size
+ +

MAINHEADING

+ + +

SECONDHEADING

+ +MAINBODY
From 3ba05e755bcde9bfb85fd28d0edd48a2c2bd1387 Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 26 Dec 2014 03:08:33 +0000 Subject: [PATCH 020/409] added shutdown button --- gui-taker.py | 23 ++++++++++++++++------- orderbook.html | 2 +- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/gui-taker.py b/gui-taker.py index eb1c6c73..4e5a8fd4 100644 --- a/gui-taker.py +++ b/gui-taker.py @@ -20,6 +20,10 @@ ''' +shutdownform = '
' + +shutdownpage = '

Successfully Shut down

' + def calc_depth_data(db, value): pass @@ -70,6 +74,7 @@ def order_str(s, order): class OrderbookPageRequestHeader(SimpleHTTPServer.SimpleHTTPRequestHandler): def __init__(self, request, client_address, base_server): self.taker = base_server.taker + self.base_server = base_server SimpleHTTPServer.SimpleHTTPRequestHandler.__init__(self, request, client_address, base_server) def create_orderbook_table(self): @@ -92,15 +97,12 @@ def get_counterparty_count(self): def do_GET(self): #SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self) #print 'httpd received ' + self.path + ' request' - pages = ['/', '/ordersize'] if self.path not in pages: return - fd = open('orderbook.html', 'r') orderbook_fmt = fd.read() fd.close() - if self.path == '/': ordercount, ordertable = self.create_orderbook_table() replacements = { @@ -108,7 +110,7 @@ def do_GET(self): 'MAINHEADING': 'Joinmarket Orderbook', 'SECONDHEADING': (str(ordercount) + ' orders found by ' + self.get_counterparty_count() + ' counterparties'), - 'MAINBODY': tableheading + ordertable + '\n' + 'MAINBODY': shutdownform + tableheading + ordertable + '\n' } elif self.path == '/ordersize': replacements = { @@ -117,18 +119,25 @@ def do_GET(self): 'SECONDHEADING': 'Order Size Histogram', 'MAINBODY': create_size_histogram(self.taker.db) } - - orderbook_page = orderbook_fmt for key, rep in replacements.iteritems(): orderbook_page = orderbook_page.replace(key, rep) - self.send_response(200) self.send_header('Content-Type', 'text/html') self.send_header('Content-Length', len(orderbook_page)) self.end_headers() self.wfile.write(orderbook_page) + def do_POST(self): + if self.path == '/shutdown': + self.taker.shutdown() + self.send_response(200) + self.send_header('Content-Type', 'text/html') + self.send_header('Content-Length', len(shutdownpage)) + self.end_headers() + self.wfile.write(shutdownpage) + self.base_server.__shutdown_request = True + class HTTPDThread(threading.Thread): def __init__(self, taker): threading.Thread.__init__(self) diff --git a/orderbook.html b/orderbook.html index 2495c727..eba54490 100644 --- a/orderbook.html +++ b/orderbook.html @@ -49,9 +49,9 @@
  • Depth Chart
  • UTXOs
    • -

    MAINHEADING

    +

    MAINHEADING

    SECONDHEADING

    MAINBODY From 3cd002000c36c7913fde9a9dc70d18ec108ac035 Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 26 Dec 2014 03:14:25 +0000 Subject: [PATCH 021/409] updated readme --- README.txt | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/README.txt b/README.txt index 34a4a7c0..628e3306 100644 --- a/README.txt +++ b/README.txt @@ -9,7 +9,7 @@ HOWTO try do this for two wallet seeds, one for each taker and maker seeds are taken as a command line argument -2. join irc.freenode.net #joinmarket and run both testtaker.py and maker.py +2. join irc.freenode.net #joinmarket-pit-test and run both testtaker.py and maker.py 3. when both bots join and have announced their orders, use this command to start a coinjoining @@ -50,16 +50,6 @@ a few algos: fees proportional to how many utxos used, since the marginal cost is unrelated to your cj amount, only to the amount of utxos you use up -#TODO think of names -#cj-market, cjex, but this isnt really an exchange -#Indra's Net -#If we now arbitrarily select one of these jewels for inspection and look closely at it, we will discover that in its polished surface there are reflected all the other jewels in the net, infinite in number. Not only that, but each of the jewels reflected in this one jewel is also reflecting all the other jewels, so that there is an infinite reflecting process occurring. -# but it sounds a bit like 'internet' with an accent -#maybe Indra, Indra's mixer -#other allusions, hall of mirrors, mirror labyrinth -#from discussing on irc, a simple name could just be JoinMarket or CoinJoinMarket -# JoinMarket seems the best probably - #TODO dont always pick the lowest cost order, instead have an exponentially decaying # distribution, so most of the time you pick the lowest and sometimes you take higher ones # this represents your uncertainty in sybil attackers, the cheapest may not always be the best @@ -108,6 +98,7 @@ e.g. yield-generator.py which acts as a maker solely for the purpose of making m might need to take orders at some point, for very small outputs which have a small probability of being filled e.g. single-tx.py which takes a single order, using it to send coins to some address typically as a payment, so this is what the electrum plugin would look like +e.g. patient-single-tx.py which does the above but doesnt mind waiting up to a limit e.g. gui-taker.py has a gui which shows the user the orderbook and they can easily fill and order and see other statistics, could be easily done by opening a http port and sending a html form and graphics From 92daac18a3a8a91b407a4229d4fadbfd01c4d2bd Mon Sep 17 00:00:00 2001 From: Belcher Date: Sat, 27 Dec 2014 00:05:26 +0000 Subject: [PATCH 022/409] edited names, changed files --- README.txt | 7 +++-- gui-taker.py | 8 +++-- taker.py | 77 +++++++++++++++++++++++++++++++++++++++++++++-- testtaker.py | 85 ---------------------------------------------------- 4 files changed, 83 insertions(+), 94 deletions(-) delete mode 100644 testtaker.py diff --git a/README.txt b/README.txt index 628e3306..7ef0cb30 100644 --- a/README.txt +++ b/README.txt @@ -4,12 +4,13 @@ you will need to know python somewhat to play around with it also get some testnet coins HOWTO try -1. use bip32-tool.py to output a bunch of addresses +1. create two wallet seeds string (can be just brainwallets if you're only storing testnet btc) + one seed for each maker and taker + use bip32-tool.py to output a bunch of addresses from the seeds send testnet coins to one mixing-depth=0 receive address - do this for two wallet seeds, one for each taker and maker seeds are taken as a command line argument -2. join irc.freenode.net #joinmarket-pit-test and run both testtaker.py and maker.py +2. join irc.freenode.net #joinmarket-pit-test and run both taker.py and yield-generator.py 3. when both bots join and have announced their orders, use this command to start a coinjoining diff --git a/gui-taker.py b/gui-taker.py index 4e5a8fd4..a017dc36 100644 --- a/gui-taker.py +++ b/gui-taker.py @@ -1,5 +1,6 @@ -from taker import * +import taker +from common import * import BaseHTTPServer, SimpleHTTPServer, threading from decimal import Decimal @@ -151,13 +152,14 @@ def run(self): httpd.serve_forever() -class GUITaker(Taker): +class GUITaker(taker.OrderbookWatch): def on_welcome(self): - Taker.on_welcome(self) + taker.OrderbookWatch.on_welcome(self) HTTPDThread(self).start() def main(): from socket import gethostname + import bitcoin as btc nickname = 'guitaker-' + btc.sha256(gethostname())[:6] print 'starting irc' diff --git a/taker.py b/taker.py index b883c5a2..2769b593 100644 --- a/taker.py +++ b/taker.py @@ -122,7 +122,7 @@ def add_signature(self, sigb64): if self.finishcallback != None: self.finishcallback() -class Taker(irclib.IRCClient): +class OrderbookWatch(irclib.IRCClient): def __init__(self): con = sqlite3.connect(":memory:", check_same_thread=False) con.row_factory = sqlite3.Row @@ -187,12 +187,83 @@ def on_leave(self, nick): def on_disconnect(self): self.db.execute('DELETE FROM orderbook;') +my_tx_fee = 10000 + +class Taker(OrderbookWatch): + + def __init__(self, wallet): + OrderbookWatch.__init__(self) + self.wallet = wallet + + def on_privmsg(self, nick, message): + OrderbookWatch.on_privmsg(self, nick, message) + #debug("privmsg nick=%s message=%s" % (nick, message)) + if message[0] != command_prefix: + return + for command in message[1:].split(command_prefix): + chunks = command.split(" ") + if chunks[0] == 'addrs': + utxo_list = chunks[1].split(',') + cj_addr = chunks[2] + change_addr = chunks[3] + self.cjtx.recv_addrs(nick, utxo_list, cj_addr, change_addr) + elif chunks[0] == 'sig': + sig = chunks[1] + self.cjtx.add_signature(sig) + + + def on_pubmsg(self, nick, message): + OrderbookWatch.on_pubmsg(self, nick, message) + if message[0] != command_prefix: + return + for command in message[1:].split(command_prefix): + #commands starting with % are for testing and will be removed in the final version + chunks = command.split(" ") + if chunks[0] == '%showob': + print('printing orderbook') + for o in self.db.execute('SELECT * FROM orderbook;').fetchall(): + print '(%s %s %d %d-%d %d %s)' % (o['counterparty'], o['ordertype'], o['oid'], + o['minsize'], o['maxsize'], o['txfee'], o['cjfee']) + print('done') + elif chunks[0] == '%unspent': + from pprint import pprint + pprint(self.wallet.unspent) + elif chunks[0] == '%fill': + #!fill [counterparty] [oid] [amount] [utxo] + counterparty = chunks[1] + oid = chunks[2] + amount = chunks[3] + my_utxo = chunks[4] + print 'making cjtx' + self.cjtx = CoinJoinTX(self, int(amount), [counterparty], [int(oid)], + [my_utxo], self.wallet.get_receive_addr(mixing_depth=1), + self.wallet.get_change_addr(mixing_depth=0), my_tx_fee) + elif chunks[0] == '%2fill': + #!2fill [amount] [utxo] [counterparty1] [oid1] [counterparty2] [oid2] + amount = int(chunks[1]) + my_utxo = chunks[2] + cp1 = chunks[3] + oid1 = int(chunks[4]) + cp2 = chunks[5] + oid2 = int(chunks[6]) + print 'creating cjtx' + self.cjtx = CoinJoinTX(self, amount, [cp1, cp2], [oid1, oid2], + [my_utxo], self.wallet.get_receive_addr(mixing_depth=1), + self.wallet.get_change_addr(mixing_depth=0), my_tx_fee) + def main(): + import sys + seed = sys.argv[1] #btc.sha256('your brainwallet goes here') from socket import gethostname - nickname = 'cj-taker-' + btc.sha256(gethostname())[:6] + nickname = 'taker-' + btc.sha256(gethostname())[:6] + + print 'downloading wallet history' + wallet = Wallet(seed) + wallet.download_wallet_history() + wallet.find_unspent_addresses() print 'starting irc' - taker = Taker() + taker = Taker(wallet) taker.run(HOST, PORT, nickname, CHANNEL) if __name__ == "__main__": diff --git a/testtaker.py b/testtaker.py deleted file mode 100644 index 101206a9..00000000 --- a/testtaker.py +++ /dev/null @@ -1,85 +0,0 @@ - -from taker import * - -my_tx_fee = 10000 - -class TestTaker(Taker): - - def __init__(self, wallet): - Taker.__init__(self) - self.wallet = wallet - - def on_privmsg(self, nick, message): - Taker.on_privmsg(self, nick, message) - #debug("privmsg nick=%s message=%s" % (nick, message)) - if message[0] != command_prefix: - return - for command in message[1:].split(command_prefix): - chunks = command.split(" ") - if chunks[0] == 'addrs': - utxo_list = chunks[1].split(',') - cj_addr = chunks[2] - change_addr = chunks[3] - self.cjtx.recv_addrs(nick, utxo_list, cj_addr, change_addr) - elif chunks[0] == 'sig': - sig = chunks[1] - self.cjtx.add_signature(sig) - - - def on_pubmsg(self, nick, message): - Taker.on_pubmsg(self, nick, message) - if message[0] != command_prefix: - return - for command in message[1:].split(command_prefix): - #commands starting with % are for testing and will be removed in the final version - chunks = command.split(" ") - if chunks[0] == '%showob': - print('printing orderbook') - for o in self.db.execute('SELECT * FROM orderbook;').fetchall(): - print '(%s %s %d %d-%d %d %s)' % (o['counterparty'], o['ordertype'], o['oid'], - o['minsize'], o['maxsize'], o['txfee'], o['cjfee']) - print('done') - elif chunks[0] == '%unspent': - from pprint import pprint - pprint(self.wallet.unspent) - elif chunks[0] == '%fill': - #!fill [counterparty] [oid] [amount] [utxo] - counterparty = chunks[1] - oid = chunks[2] - amount = chunks[3] - my_utxo = chunks[4] - print 'making cjtx' - self.cjtx = CoinJoinTX(self, int(amount), [counterparty], [int(oid)], - [my_utxo], self.wallet.get_receive_addr(mixing_depth=1), - self.wallet.get_change_addr(mixing_depth=0), my_tx_fee) - elif chunks[0] == '%2fill': - #!2fill [amount] [utxo] [counterparty1] [oid1] [counterparty2] [oid2] - amount = int(chunks[1]) - my_utxo = chunks[2] - cp1 = chunks[3] - oid1 = int(chunks[4]) - cp2 = chunks[5] - oid2 = int(chunks[6]) - print 'creating cjtx' - self.cjtx = CoinJoinTX(self, amount, [cp1, cp2], [oid1, oid2], - [my_utxo], self.wallet.get_receive_addr(mixing_depth=1), - self.wallet.get_change_addr(mixing_depth=0), my_tx_fee) - -def main(): - import sys - seed = sys.argv[1] #btc.sha256('your brainwallet goes here') - from socket import gethostname - nickname = 'testtakr-' + btc.sha256(gethostname())[:6] - - print 'downloading wallet history' - wallet = Wallet(seed) - wallet.download_wallet_history() - wallet.find_unspent_addresses() - - print 'starting irc' - taker = TestTaker(wallet) - taker.run(HOST, PORT, nickname, CHANNEL) - -if __name__ == "__main__": - main() - print('done') From 82182c51044fe22e51a409f3e25e4876db92bb3d Mon Sep 17 00:00:00 2001 From: Belcher Date: Sat, 27 Dec 2014 12:48:46 +0000 Subject: [PATCH 023/409] added error checking code to maker --- README.txt | 5 +++ maker.py | 103 ++++++++++++++++++++++++++++----------------- yield-generator.py | 3 ++ 3 files changed, 72 insertions(+), 39 deletions(-) diff --git a/README.txt b/README.txt index 7ef0cb30..afe5918d 100644 --- a/README.txt +++ b/README.txt @@ -118,6 +118,11 @@ TODO need to move onto the bip44 structure of HD wallets TODO probably a good idea to have a debug.log where loads of information is dumped +TODO +for the !addrs command, firstly change its name since it also includes the utxo inputs + secondly, the utxo list might be longer than can fit in an irc message, so create a + !addrsparts or something command + TODO code a gui where a human can see the state of the orderbook and easily choose orders to fill code a gui that easily explains to a human how they can choose a fee for their yield-generator.py diff --git a/maker.py b/maker.py index 97d6482a..1c5b3b54 100644 --- a/maker.py +++ b/maker.py @@ -3,23 +3,20 @@ from common import * import irclib import bitcoin as btc -import sys import base64, pprint - -from socket import gethostname -nickname = 'cj-maker-' + btc.sha256(gethostname())[:6] -seed = sys.argv[1] #btc.sha256('dont use brainwallets except for holding testnet coins') - class CoinJoinOrder(object): def __init__(self, maker, nick, oid, amount): self.maker = maker self.oid = oid self.cj_amount = amount - order = [o for o in maker.orderlist if o['oid'] == oid][0] + order_s = [o for o in maker.orderlist if o['oid'] == oid] + if len(order_s) == 0: + self.maker.send_error(nick, 'oid not found') + order = order_s[0] if amount <= order['minsize'] or amount >= order['maxsize']: - maker.privmsg(nick, command_prefix + 'error amount out of range') - #TODO logic for this error causing the order to be removed from list of open orders + self.maker.send_error(nick, 'amount out of range') + #TODO return addresses, not mixing depths, so you can coinjoin to outside your own wallet self.utxos, cj_mixing_depth, change_mixing_depth = maker.oid_to_order(oid, amount) self.ordertype = order['ordertype'] self.txfee = order['txfee'] @@ -36,19 +33,25 @@ def __init__(self, maker, nick, oid, amount): def recv_tx_part(self, b64txpart): self.b64txparts.append(b64txpart) - #TODO this is a dos opportunity, flood someone with !txpart - #repeatedly to fill up their memory + size = sum([len(s) for s in self.b64txparts]) + if size > 60000000: #~2MB * 2 * 4/3 + self.maker.send_error(nick, 'tx too large, buffer limit reached') def recv_tx(self, nick, b64txpart): self.b64txparts.append(b64txpart) - txhex = base64.b64decode(''.join(self.b64txparts)).encode('hex') - self.tx = btc.deserialize(txhex) - import pprint + try: + txhex = base64.b64decode(''.join(self.b64txparts)).encode('hex') + except TypeError as e: + self.maker.send_error(nick, 'bad base64 tx. ' + repr(e)) + try: + self.tx = btc.deserialize(txhex) + except IndexError as e: + self.maker.send_error(nick, 'malformed txhex. ' + repr(e)) debug('obtained tx\n' + pprint.pformat(self.tx)) goodtx, errmsg = self.verify_unsigned_tx(self.tx) if not goodtx: debug('not a good tx, reason=' + errmsg) - self.maker.privmsg(nick, command_prefix + 'error ' + errmsg) + self.maker.send_error(nick, errmsg) return False debug('goodtx') sigs = [] @@ -59,10 +62,9 @@ def recv_tx(self, nick, b64txpart): addr = self.maker.wallet.unspent[utxo]['address'] txs = btc.sign(txhex, index, self.maker.wallet.get_key_from_addr(addr)) sigs.append(base64.b64encode(btc.deserialize(txs)['ins'][index]['script'].decode('hex'))) - if len(sigs) == 0: - print 'ERROR no private keys found' - add_addr_notify(self.change_addr, self.unconfirm_callback, self.confirm_callback) + #len(sigs) > 0 guarenteed since i did verify_unsigned_tx() + add_addr_notify(self.change_addr, self.unconfirm_callback, self.confirm_callback) debug('sending sigs ' + str(sigs)) #TODO make this a function in irclib.py sigline = '' @@ -94,7 +96,7 @@ def handle_modified_orders(self, to_cancel, to_announce): order = [o for o in self.maker.orderlist if o['oid'] == oid][0] self.maker.orderlist.remove(order) if len(to_cancel) > 0: - clines = ['!cancel ' + str(oid) for oid in to_cancel] + clines = [command_prefix + 'cancel ' + str(oid) for oid in to_cancel] self.maker.pubmsg(''.join(clines)) if len(to_announce) > 0: self.maker.privmsg_all_orders(CHANNEL, to_announce) @@ -111,7 +113,7 @@ def verify_unsigned_tx(self, txd): my_total_in = 0 for u in self.utxos: usvals = self.maker.wallet.unspent[u] - my_total_in += int(usvals['value']) + my_total_in += usvals['value'] real_cjfee = calc_cj_fee(self.ordertype, self.cjfee, self.cj_amount) expected_change_value = (my_total_in - self.cj_amount @@ -135,6 +137,9 @@ def verify_unsigned_tx(self, txd): return False, 'cj or change addr not in tx outputs exactly once' return True, None +class CJMakerOrderError(StandardError): + pass + class Maker(irclib.IRCClient): def __init__(self, wallet): self.active_orders = {} @@ -155,7 +160,12 @@ def privmsg_all_orders(self, target, orderlist=None): orderline = '' if len(orderline) > 0: self.privmsg(target, orderline) - + + def send_error(self, nick, errmsg): + debug('error<%s> : %s' % (nick, errmsg)) + self.privmsg(nick, command_prefix + 'error ' + errmsg) + raise CJMakerOrderError() + def on_welcome(self): self.privmsg_all_orders(CHANNEL) @@ -165,18 +175,32 @@ def on_privmsg(self, nick, message): return command_lines = message.split(command_prefix) for command_line in command_lines: + if len(command_line) == 0: + continue chunks = command_line.split(" ") - if chunks[0] == 'fill': - oid = int(chunks[1]) - amount = int(chunks[2]) #TODO make sure that nick doesnt already have an open order - self.active_orders[nick] = CoinJoinOrder(self, nick, oid, amount) - elif chunks[0] == 'txpart': - b64txpart = chunks[1] #TODO check nick appears in active_orders - self.active_orders[nick].recv_tx_part(b64txpart) - elif chunks[0] == 'tx': - b64txpart = chunks[1] - self.active_orders[nick].recv_tx(nick, b64txpart) - + try: + if len(chunks) < 2: + self.send_error(nick, 'Not enough arguments') + if chunks[0] == 'fill': + if nick in self.active_orders and self.active_orders[nick] != None: + self.send_error(nick, 'Already have partially-filled order') + try: + oid = int(chunks[1]) + amount = int(chunks[2]) + except (ValueError, IndexError) as e: + self.send_error(nick, str(e)) + self.active_orders[nick] = CoinJoinOrder(self, nick, oid, amount) + elif chunks[0] == 'txpart' or chunks[0] == 'tx': + if nick not in self.active_orders or self.active_orders[nick] == None: + self.send_error(nick, 'No open order from this nick') + b64txpart = chunks[1] + if chunks[0] == 'txpart': + self.active_orders[nick].recv_tx_part(b64txpart) + else: + self.active_orders[nick].recv_tx(nick, b64txpart) + except CJMakerOrderError: + self.active_orders[nick] = None + continue #each order has an id for referencing to and looking up # using the same id again overwrites it, they'll be plenty of times when an order @@ -185,14 +209,10 @@ def on_pubmsg(self, nick, message): debug("pubmsg nick=%s message=%s" % (nick, message)) if message[0] == command_prefix: chunks = message[1:].split(" ") - if chunks[0] == '%quit' or chunks[0] == '%makerquit': - self.shutdown() - elif chunks[0] == '%say': #% is a way to remind me its a testing cmd - self.pubmsg(message[6:]) - elif chunks[0] == '%rm': - self.pubmsg('!cancel ' + chunks[1]) - elif chunks[0] == 'orderbook': + if chunks[0] == 'orderbook': self.privmsg_all_orders(nick) + elif chunks[0] == '%quit' or chunks[0] == '%makerquit': + self.shutdown() def on_set_topic(self, newtopic): chunks = newtopic.split('|') @@ -284,6 +304,11 @@ def on_tx_confirmed(self, cjorder, confirmations, txid, balance, added_utxos): def main(): + from socket import gethostname + nickname = 'cj-maker-' + btc.sha256(gethostname())[:6] + import sys + seed = sys.argv[1] #btc.sha256('dont use brainwallets except for holding testnet coins') + print 'downloading wallet history' wallet = Wallet(seed) wallet.download_wallet_history() diff --git a/yield-generator.py b/yield-generator.py index 8f59e683..d9b4e591 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -92,6 +92,9 @@ def on_tx_confirmed(self, cjorder, confirmations, txid, balance, added_utxos): return ([], to_announce) def main(): + import sys + seed = sys.argv[1] #btc.sha256('dont use brainwallets except for holding testnet coins') + print 'downloading wallet history' wallet = Wallet(seed, max_mix_depth = mix_levels) wallet.download_wallet_history() From c7a39e7a4cfd80c71b29142c83244185baba42d9 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sat, 27 Dec 2014 21:13:56 +0000 Subject: [PATCH 024/409] slight protocol changes --- maker.py | 12 +++++------- taker.py | 6 +++--- yield-generator.py | 6 +++--- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/maker.py b/maker.py index 1c5b3b54..ebab6f71 100644 --- a/maker.py +++ b/maker.py @@ -17,18 +17,16 @@ def __init__(self, maker, nick, oid, amount): if amount <= order['minsize'] or amount >= order['maxsize']: self.maker.send_error(nick, 'amount out of range') #TODO return addresses, not mixing depths, so you can coinjoin to outside your own wallet - self.utxos, cj_mixing_depth, change_mixing_depth = maker.oid_to_order(oid, amount) + self.utxos, self.cj_addr, self.change_addr = maker.oid_to_order(oid, amount) self.ordertype = order['ordertype'] self.txfee = order['txfee'] self.cjfee = order['cjfee'] - self.cj_addr = maker.wallet.get_receive_addr(cj_mixing_depth) - self.change_addr = maker.wallet.get_change_addr(change_mixing_depth) self.b64txparts = [] debug('new cjorder nick=%s oid=%d amount=%d' % (nick, oid, amount)) #always a new address even if the order ends up never being # furfilled, you dont want someone pretending to fill all your # orders to find out which addresses you use - maker.privmsg(nick, command_prefix + 'addrs ' + ','.join(self.utxos) + ' ' + + maker.privmsg(nick, command_prefix + 'io ' + ','.join(self.utxos) + ' ' + self.cj_addr + ' ' + self.change_addr) def recv_tx_part(self, b64txpart): @@ -269,9 +267,9 @@ def oid_to_order(self, oid, amount): ''' order = [o for o in self.orderlist if o['oid'] == oid][0] - cj_mixing_depth = order['mixdepth'] + 1 - change_mixing_depth = order['mixdepth'] - return [order['utxo']], cj_mixing_depth, change_mixing_depth + cj_addr = self.wallet.get_receive_addr(order['mixdepth'] + 1) + change_addr = self.wallet.get_change_addr(order['mixdepth']) + return [order['utxo']], cj_addr, change_addr def get_next_oid(self): self.nextoid += 1 diff --git a/taker.py b/taker.py index 2769b593..db5574b4 100644 --- a/taker.py +++ b/taker.py @@ -28,7 +28,7 @@ def __init__(self, taker, cj_amount, counterparties, oids, my_utxos, my_cj_addr, for c, oid in zip(counterparties, oids): taker.privmsg(c, command_prefix + 'fill ' + str(oid) + ' ' + str(cj_amount)) - def recv_addrs(self, nick, utxo_list, cj_addr, change_addr): + def recv_txio(self, nick, utxo_list, cj_addr, change_addr): if nick not in self.nonrespondants: debug('nick(' + nick + ') not in nonrespondants ' + str(self.nonrespondants)) return @@ -202,11 +202,11 @@ def on_privmsg(self, nick, message): return for command in message[1:].split(command_prefix): chunks = command.split(" ") - if chunks[0] == 'addrs': + if chunks[0] == 'io': utxo_list = chunks[1].split(',') cj_addr = chunks[2] change_addr = chunks[3] - self.cjtx.recv_addrs(nick, utxo_list, cj_addr, change_addr) + self.cjtx.recv_txio(nick, utxo_list, cj_addr, change_addr) elif chunks[0] == 'sig': sig = chunks[1] self.cjtx.add_signature(sig) diff --git a/yield-generator.py b/yield-generator.py index d9b4e591..72db3d19 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -45,9 +45,9 @@ def oid_to_order(self, oid, amount): for utxo in order['utxos']] inputs = btc.select(unspent, amount) mixdepth = oid - cj_mixdepth = (mixdepth + 1) % self.wallet.max_mix_depth - change_mixdepth = mixdepth - return [i['utxo'] for i in inputs], cj_mixdepth, change_mixdepth + cj_addr = self.wallet.get_receive_addr((mixdepth + 1) % self.wallet.max_mix_depth) + change_addr = self.wallet.get_change_addr(mixdepth) + return [i['utxo'] for i in inputs], cj_addr, change_addr def on_tx_unconfirmed(self, cjorder, balance, removed_utxos): #want to replace the current relorders with the same From 63aac7f8fcca467e9c5c202d9ac3d3643e5acb9b Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 28 Dec 2014 00:56:37 +0000 Subject: [PATCH 025/409] restructured taker slightly --- taker.py | 39 ++++++++++++++++++++++----------------- yield-generator.py | 4 ++++ 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/taker.py b/taker.py index db5574b4..0d51a2b0 100644 --- a/taker.py +++ b/taker.py @@ -168,8 +168,6 @@ def on_pubmsg(self, nick, message): elif chunks[0] in ordername_list: self.add_order(nick, chunks) - #self.connection.quit("Using irc.client.py") - def on_welcome(self): self.pubmsg(command_prefix + 'orderbook') @@ -187,13 +185,14 @@ def on_leave(self, nick): def on_disconnect(self): self.db.execute('DELETE FROM orderbook;') -my_tx_fee = 10000 - +#assume this only has one open cj tx at a time class Taker(OrderbookWatch): - - def __init__(self, wallet): + def __init__(self): OrderbookWatch.__init__(self) - self.wallet = wallet + self.cjtx = None + #TODO have a list of maker's nick we're coinjoining with, so + # that some other guy doesnt send you confusing stuff + #maybe a start_cj_tx() method is needed def on_privmsg(self, nick, message): OrderbookWatch.on_privmsg(self, nick, message) @@ -202,18 +201,24 @@ def on_privmsg(self, nick, message): return for command in message[1:].split(command_prefix): chunks = command.split(" ") - if chunks[0] == 'io': - utxo_list = chunks[1].split(',') - cj_addr = chunks[2] - change_addr = chunks[3] - self.cjtx.recv_txio(nick, utxo_list, cj_addr, change_addr) - elif chunks[0] == 'sig': - sig = chunks[1] - self.cjtx.add_signature(sig) + if chunks[0] == 'io': + utxo_list = chunks[1].split(',') + cj_addr = chunks[2] + change_addr = chunks[3] + self.cjtx.recv_txio(nick, utxo_list, cj_addr, change_addr) + elif chunks[0] == 'sig': + sig = chunks[1] + self.cjtx.add_signature(sig) + +my_tx_fee = 10000 +class TestTaker(Taker): + def __init__(self, wallet): + Taker.__init__(self) + self.wallet = wallet def on_pubmsg(self, nick, message): - OrderbookWatch.on_pubmsg(self, nick, message) + Taker.on_pubmsg(self, nick, message) if message[0] != command_prefix: return for command in message[1:].split(command_prefix): @@ -263,7 +268,7 @@ def main(): wallet.find_unspent_addresses() print 'starting irc' - taker = Taker(wallet) + taker = TestTaker(wallet) taker.run(HOST, PORT, nickname, CHANNEL) if __name__ == "__main__": diff --git a/yield-generator.py b/yield-generator.py index 72db3d19..ff3ea589 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -13,6 +13,10 @@ #is a maker for the purposes of generating a yield from held bitcoins #for each mixing level, adds up the balance of all the addresses and put up # a relative fee order, oid=mix depth +#TODO theres no need for seperate orders, just announce the order with the highest size +# and when it arrives choose which mixdepth you want, best algorithm is probably +# to try to keep coins concentrated into one depth so you can join larger amounts +#TODO when a nick with an open order quits, is kicked or dies, remove him from open orders class YieldGenerator(Maker): def __init__(self, wallet): Maker.__init__(self, wallet) From 19d120d809b11c11347fc6deb03aa5556b78f487 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 28 Dec 2014 02:44:10 +0000 Subject: [PATCH 026/409] slight wallet code edit --- common.py | 12 ++++++++++++ yield-generator.py | 11 +---------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/common.py b/common.py index ccc614c9..05347303 100644 --- a/common.py +++ b/common.py @@ -96,6 +96,18 @@ def add_new_utxos(self, tx, txid): self.unspent[utxo] = addrdict return added_utxos + def get_mix_utxo_list(self): + ''' + returns a list of utxos sorted by different mix levels + ''' + mix_utxo_list = {} + for utxo, addrvalue in self.unspent.iteritems(): + mixdepth = self.addr_cache[addrvalue['address']][0] + if mixdepth not in mix_utxo_list: + mix_utxo_list[mixdepth] = [] + mix_utxo_list[mixdepth].append(utxo) + return mix_utxo_list + def download_wallet_history(self, gaplimit=6): ''' sets Wallet internal indexes to be at the next unused address diff --git a/yield-generator.py b/yield-generator.py index ff3ea589..b925a463 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -21,17 +21,8 @@ class YieldGenerator(Maker): def __init__(self, wallet): Maker.__init__(self, wallet) - def get_mix_utxo_list(self): - mix_utxo_list = {} - for utxo, addrvalue in self.wallet.unspent.iteritems(): - mixdepth = self.wallet.addr_cache[addrvalue['address']][0] - if mixdepth not in mix_utxo_list: - mix_utxo_list[mixdepth] = [] - mix_utxo_list[mixdepth].append(utxo) - return mix_utxo_list - def create_my_orders(self): - mix_utxo_list = self.get_mix_utxo_list() + mix_utxo_list = self.wallet.get_mix_utxo_list() orderlist = [] for mixdepth, utxo_list in mix_utxo_list.iteritems(): total_value = 0 From 71fbd754d1c426b2d9fc88caf6ea975404385c61 Mon Sep 17 00:00:00 2001 From: Belcher Date: Mon, 29 Dec 2014 03:37:23 +0000 Subject: [PATCH 027/409] added send-payment.py for 2-party coinjoin --- README.txt | 7 +++ send-payment.py | 140 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 send-payment.py diff --git a/README.txt b/README.txt index afe5918d..32095650 100644 --- a/README.txt +++ b/README.txt @@ -68,6 +68,10 @@ fees proportional to how many utxos used, since the marginal cost is unrelated t #TODO option for how many blocks deep to wait before using a utxo for more mixing # 1 confirm is probably enough +TODO +have the taker enforce this, look up the txhash of the maker's utxo and make sure + it is already in a block + TODO implement rate limiting for irc.privmsg to stop the bot being killed due to flood i suggest creating a thread that only dispatches/writes to the irc socket @@ -103,6 +107,9 @@ e.g. patient-single-tx.py which does the above but doesnt mind waiting up to a l e.g. gui-taker.py has a gui which shows the user the orderbook and they can easily fill and order and see other statistics, could be easily done by opening a http port and sending a html form and graphics +TODO +extend send-payment.py to be able to do more than 2-party coinjoin + TODO implement this the thing that gmaxwell wrote about in the original coinjoin post, as a kind of tumbler "Isn't the anonymity set size limited by how many parties you can get in a single transaction?" diff --git a/send-payment.py b/send-payment.py new file mode 100644 index 00000000..028f6785 --- /dev/null +++ b/send-payment.py @@ -0,0 +1,140 @@ +#! /usr/bin/env python + +from common import * +import taker as takermodule +import bitcoin as btc + +from optparse import OptionParser +import threading + + +def choose_order(db, cj_amount): + + sqlorders = db.execute('SELECT * FROM orderbook;').fetchall() + orders = [(o['counterparty'], o['oid'], calc_cj_fee(o['ordertype'], o['cjfee'], cj_amount)) + for o in sqlorders if cj_amount >= o['minsize'] or cj_amount <= o['maxsize']] + orders = sorted(orders, key=lambda k: k[2]) + print 'orders = ' + str(orders) + return orders[0][:2] #choose the cheapest, later this will be chosen differently + +def choose_sweep_order(db, my_total_input, my_tx_fee): + ''' + choose an order given that we want to be left with no change + i.e. sweep an entire group of utxos + + solve for cjamount when mychange = 0 + ABS FEE + mychange = totalin - cjamount - mytxfee - absfee = 0 + => cjamount = totalin - mytxfee - absfee + REL FEE + mychange = totalin - cjamount - mytxfee - relfee*cjamount + => 0 = totalin - mytxfee - cjamount*(1 + relfee) + => cjamount = (totalin - mytxfee) / (1 + relfee) + ''' + def calc_zero_change_cj_amount(ordertype, cjfee): + cj_amount = None + if ordertype == 'absorder': + cj_amount = my_total_input - my_tx_fee - cjfee + elif ordertype == 'relorder': + cj_amount = (my_total_input - my_tx_fee) / (Decimal(cjfee) + 1) + cj_amount = int(cj_amount.quantize(Decimal(1))) + else: + raise RuntimeError('unknown order type: ' + str(ordertype)) + return cj_amount + + sqlorders = db.execute('SELECT * FROM orderbook;').fetchall() + orders = [(o['counterparty'], o['oid'], calc_zero_change_cj_amount(o['ordertype'], o['cjfee']), + o['minsize'], o['maxsize']) for o in sqlorders] + #filter cj_amounts that are not in range + orders = [o[:3] for o in orders if o[2] >= o[3] and o[2] <= o[4]] + orders = sorted(orders, key=lambda k: k[2]) + print 'sweep orders = ' + str(orders) + return orders[-1] #choose one with the highest cj_amount, most left over after paying everything else + +#thread which does the buy-side algorithm +# chooses which coinjoins to initiate and when +class PaymentThread(threading.Thread): + def __init__(self, taker): + threading.Thread.__init__(self) + self.daemon = True + self.taker = taker + + def finishcallback(self): + self.taker.shutdown() + + def run(self): + print 'waiting for all orders to certainly arrive' + time.sleep(self.taker.waittime) + counterparty, oid = choose_order(self.taker.db, self.taker.amount) + + utxo_list = self.taker.wallet.get_mix_utxo_list()[0] #only spend from the unmixed funds + + unspent = [{'utxo': utxo, 'value': self.taker.wallet.unspent[utxo]['value']} + for utxo in utxo_list] + inputs = btc.select(unspent, self.taker.amount) + utxos = [i['utxo'] for i in inputs] + print 'will spend ' + str(inputs) + + self.taker.cjtx = takermodule.CoinJoinTX(self.taker, self.taker.amount, + [counterparty], [oid], utxos, self.taker.destaddr, + self.taker.wallet.get_change_addr(0), self.taker.txfee, + self.finishcallback) + + ''' + counterparty, oid, cj_amount = choose_sweep_order(addrvalue['value'], my_tx_fee) + cjtx = CoinJoinTX(self.taker, cj_amount, [counterparty], [int(oid)], + [utxo], self.taker.wallet.get_receive_addr(mixing_depth=1), None, + my_tx_fee, self.finished_cj_callback) + ''' + +class SendPayment(takermodule.Taker): + def __init__(self, wallet, destaddr, amount, txfee, waittime): + takermodule.Taker.__init__(self) + self.wallet = wallet + self.destaddr = destaddr + self.amount = amount + self.txfee = txfee + self.waittime = waittime + + def on_welcome(self): + takermodule.Taker.on_welcome(self) + PaymentThread(self).start() + +#how long to wait for all the orders to arrive before starting to do coinjoins +ORDER_ARRIVAL_WAIT_TIME = 5 +def main(): + parser = OptionParser(usage='usage: %prog [options] [seed] [amount] [destaddr]', + description='Sends a single payment from your wallet to an given address' + + ' using coinjoin and then switches off.') + parser.add_option('-t', '--txfee', action='store', type='int', dest='txfee', + default=5000, help='miner fee contribution') + parser.add_option('-w', '--wait-time', action='store', type='float', dest='waittime', + help='wait time in seconds to allow orders to arrive', default=10) + (options, args) = parser.parse_args() + + if len(args) < 3: + parser.error('Needs a seed, amount and destination address') + sys.exit(0) + seed = args[0] + amount = int(args[1]) + destaddr = args[2] + + + #python send-payment.py -w 2 67286d672a2980ca30f7465084e90447 20000000 moovynzW3fioyQZEACSg9L27HjefZ3F5m2 + + from socket import gethostname + nickname = 'payer-' + btc.sha256(gethostname())[:6] + + print 'downloading wallet history' + wallet = Wallet(seed) + wallet.download_wallet_history() + wallet.find_unspent_addresses() + + #TODO options number of parties in the coinjoin + print 'starting irc' + taker = SendPayment(wallet, destaddr, amount, options.txfee, options.waittime) + taker.run(HOST, PORT, nickname, CHANNEL) + +if __name__ == "__main__": + main() + print('done') From 850bc98951ecc3d5735fecc52d3b8a5d1f073c87 Mon Sep 17 00:00:00 2001 From: Belcher Date: Mon, 29 Dec 2014 03:43:27 +0000 Subject: [PATCH 028/409] fixed bug in maker error checking code --- maker.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/maker.py b/maker.py index ebab6f71..f164cd6e 100644 --- a/maker.py +++ b/maker.py @@ -50,7 +50,6 @@ def recv_tx(self, nick, b64txpart): if not goodtx: debug('not a good tx, reason=' + errmsg) self.maker.send_error(nick, errmsg) - return False debug('goodtx') sigs = [] for index, ins in enumerate(self.tx['ins']): @@ -74,7 +73,6 @@ def recv_tx(self, nick, b64txpart): sigline = command_prefix + 'sig ' + sig if len(sigline) > 0: self.maker.privmsg(nick, sigline) - return True def unconfirm_callback(self, balance): removed_utxos = self.maker.wallet.remove_old_utxos(self.tx) @@ -196,6 +194,7 @@ def on_privmsg(self, nick, message): self.active_orders[nick].recv_tx_part(b64txpart) else: self.active_orders[nick].recv_tx(nick, b64txpart) + self.active_orders[nick] = None except CJMakerOrderError: self.active_orders[nick] = None continue @@ -220,6 +219,9 @@ def on_set_topic(self, newtopic): except IndexError: pass + def on_leave(self, nick): + self.active_orders[nick] = None + #these functions # create_my_orders() # oid_to_uxto() From a74dd770bcdb40430e26e484b110788380cddcc4 Mon Sep 17 00:00:00 2001 From: Belcher Date: Mon, 29 Dec 2014 15:16:21 +0000 Subject: [PATCH 029/409] arg to CoinJoinTX() is now a dict not two lists --- taker.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/taker.py b/taker.py index 0d51a2b0..45ba2c28 100644 --- a/taker.py +++ b/taker.py @@ -7,16 +7,17 @@ import sqlite3, base64, threading, time, random class CoinJoinTX(object): - def __init__(self, taker, cj_amount, counterparties, oids, my_utxos, my_cj_addr, + def __init__(self, taker, cj_amount, orders, my_utxos, my_cj_addr, my_change_addr, my_txfee, finishcallback=None): ''' if my_change is None then there wont be a change address thats used if you want to entirely coinjoin one utxo with no change left over + orders is the orders you want to fill {'counterpartynick': oid, 'cp2': oid2} ''' self.taker = taker self.cj_amount = cj_amount - self.active_orders = dict(zip(counterparties, oids)) - self.nonrespondants = list(counterparties) + self.active_orders = dict(orders) + self.nonrespondants = list(orders.keys()) self.my_utxos = my_utxos self.utxos = {taker.nick: my_utxos} self.finishcallback = finishcallback @@ -25,7 +26,7 @@ def __init__(self, taker, cj_amount, counterparties, oids, my_utxos, my_cj_addr, self.my_change_addr = my_change_addr self.cjfee_total = 0 self.latest_tx = None - for c, oid in zip(counterparties, oids): + for c, oid in orders.iteritems(): taker.privmsg(c, command_prefix + 'fill ' + str(oid) + ' ' + str(cj_amount)) def recv_txio(self, nick, utxo_list, cj_addr, change_addr): @@ -236,11 +237,11 @@ def on_pubmsg(self, nick, message): elif chunks[0] == '%fill': #!fill [counterparty] [oid] [amount] [utxo] counterparty = chunks[1] - oid = chunks[2] + oid = int(chunks[2]) amount = chunks[3] my_utxo = chunks[4] print 'making cjtx' - self.cjtx = CoinJoinTX(self, int(amount), [counterparty], [int(oid)], + self.cjtx = CoinJoinTX(self, int(amount), {counterparty: oid}, [my_utxo], self.wallet.get_receive_addr(mixing_depth=1), self.wallet.get_change_addr(mixing_depth=0), my_tx_fee) elif chunks[0] == '%2fill': @@ -252,7 +253,7 @@ def on_pubmsg(self, nick, message): cp2 = chunks[5] oid2 = int(chunks[6]) print 'creating cjtx' - self.cjtx = CoinJoinTX(self, amount, [cp1, cp2], [oid1, oid2], + self.cjtx = CoinJoinTX(self, amount, {cp1: oid1, cp2: oid2}, [my_utxo], self.wallet.get_receive_addr(mixing_depth=1), self.wallet.get_change_addr(mixing_depth=0), my_tx_fee) From c5ffe2e30e9112f65e567e1563cd2dfc669ac29c Mon Sep 17 00:00:00 2001 From: Belcher Date: Mon, 29 Dec 2014 16:47:43 +0000 Subject: [PATCH 030/409] send-payment.py now works for N>1 makers --- send-payment.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/send-payment.py b/send-payment.py index 028f6785..aef73c54 100644 --- a/send-payment.py +++ b/send-payment.py @@ -8,14 +8,20 @@ import threading -def choose_order(db, cj_amount): +def choose_order(db, cj_amount, n): sqlorders = db.execute('SELECT * FROM orderbook;').fetchall() orders = [(o['counterparty'], o['oid'], calc_cj_fee(o['ordertype'], o['cjfee'], cj_amount)) for o in sqlorders if cj_amount >= o['minsize'] or cj_amount <= o['maxsize']] orders = sorted(orders, key=lambda k: k[2]) - print 'orders = ' + str(orders) - return orders[0][:2] #choose the cheapest, later this will be chosen differently + print 'considered orders = ' + str(orders) + chosen_orders = [] + for i in range(n): + chosen_order = orders[0] #choose the cheapest, later this will be chosen differently + orders = [o for o in orders if o[0] != chosen_order[0]] + chosen_orders.append(chosen_order) + chosen_orders = [o[:2] for o in chosen_orders] + return dict(chosen_orders) def choose_sweep_order(db, my_total_input, my_tx_fee): ''' @@ -65,7 +71,8 @@ def finishcallback(self): def run(self): print 'waiting for all orders to certainly arrive' time.sleep(self.taker.waittime) - counterparty, oid = choose_order(self.taker.db, self.taker.amount) + orders = choose_order(self.taker.db, self.taker.amount, self.taker.makercount) + print 'chosen orders to fill ' + str(orders) utxo_list = self.taker.wallet.get_mix_utxo_list()[0] #only spend from the unmixed funds @@ -76,7 +83,7 @@ def run(self): print 'will spend ' + str(inputs) self.taker.cjtx = takermodule.CoinJoinTX(self.taker, self.taker.amount, - [counterparty], [oid], utxos, self.taker.destaddr, + orders, utxos, self.taker.destaddr, self.taker.wallet.get_change_addr(0), self.taker.txfee, self.finishcallback) @@ -88,11 +95,12 @@ def run(self): ''' class SendPayment(takermodule.Taker): - def __init__(self, wallet, destaddr, amount, txfee, waittime): + def __init__(self, wallet, destaddr, amount, makercount, txfee, waittime): takermodule.Taker.__init__(self) self.wallet = wallet self.destaddr = destaddr self.amount = amount + self.makercount = makercount self.txfee = txfee self.waittime = waittime @@ -100,8 +108,6 @@ def on_welcome(self): takermodule.Taker.on_welcome(self) PaymentThread(self).start() -#how long to wait for all the orders to arrive before starting to do coinjoins -ORDER_ARRIVAL_WAIT_TIME = 5 def main(): parser = OptionParser(usage='usage: %prog [options] [seed] [amount] [destaddr]', description='Sends a single payment from your wallet to an given address' + @@ -110,6 +116,8 @@ def main(): default=5000, help='miner fee contribution') parser.add_option('-w', '--wait-time', action='store', type='float', dest='waittime', help='wait time in seconds to allow orders to arrive', default=10) + parser.add_option('-m', '--makercount', action='store', type='int', dest='makercount', + help='how many makers to coinjoin with', default=2) (options, args) = parser.parse_args() if len(args) < 3: @@ -130,9 +138,8 @@ def main(): wallet.download_wallet_history() wallet.find_unspent_addresses() - #TODO options number of parties in the coinjoin print 'starting irc' - taker = SendPayment(wallet, destaddr, amount, options.txfee, options.waittime) + taker = SendPayment(wallet, destaddr, amount, options.makercount, options.txfee, options.waittime) taker.run(HOST, PORT, nickname, CHANNEL) if __name__ == "__main__": From 6f7526a02a4c0f90ce3060b14cacf3822e294335 Mon Sep 17 00:00:00 2001 From: Belcher Date: Tue, 30 Dec 2014 17:06:43 +0000 Subject: [PATCH 031/409] slightly more detail on the protocol --- README.txt | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/README.txt b/README.txt index 32095650..fe597a09 100644 --- a/README.txt +++ b/README.txt @@ -30,19 +30,33 @@ theres lots that needs to be done some other notes below.. #COINJOIN PROTOCOL -#when a maker joins the channel it says out all its orders -# an order contains an order id, max size, min size, fee, whether the fee is absolute or -# as a proportion of the coinjoin amount -#when a taker joins the channel, it asks for orders to be pmed to him -#taker initiates coinjoin -#tells maker(s) by pm which order it wants to fill, sends the order id and the coinjoin amount -#maker(s) pm back the utxos they will input, and exactly two addresses, the coinjoin output and the change address -#taker collects all the utxos and outputs and makes a transaction -# pms them to the maker(s) who check everything is ok -# that the miner fee is right, that the cj fee is right -# and pm back signatures -# iv checked, it easily possible to put the signatures back into a tx -#taker then signs his own and pushtx() +when a maker joins the channel it says out all its orders + an order contains an order id, max size, min size, fee, whether the fee is absolute or + as a proportion of the coinjoin amount +when a taker joins the channel, it asks for orders to be pmed to him +taker initiates coinjoin +tells maker(s) by pm which order it wants to fill, sends the order id and the coinjoin amount +maker(s) pm back the utxos they will input, and exactly two addresses, the coinjoin output and the change address +taker collects all the utxos and outputs and makes a transaction + pms them to the maker(s) who check everything is ok + that the miner fee is right, that the cj fee is right + and pm back signatures + iv checked, it easily possible to put the signatures back into a tx +taker then signs his own and pushtx() + +IRC commands used when starting a coinjoin, everything in pm + !fill [order id] [coinjoin amount] + !io [comma seperated list of utxos] [coinjoin address] [change address] +when taker collects inputs and outputs of all the makers it's contacted, it creates a tx out of them + !txpart [base64 encoded tx part] +... + !tx [base64 encoded tx part] +maker concatenates all the !txpart and !tx commands and obtains unsigned tx +it signs its own utxos and extracts just the script from it which contains signature and pubkey + !sig [base64 encoded script] +taker collects all scripts and places them into the tx +taker pushes tx when all the scripts have arrived + #TODO #ask people on the testnet stuff to code up a few trading algos to see if the interface/protocol that From 52dfe4adcc47d9c8378bdf78bd47a960f59d50b8 Mon Sep 17 00:00:00 2001 From: Belcher Date: Tue, 30 Dec 2014 18:59:30 +0000 Subject: [PATCH 032/409] rewrite algo for yield-generator --- send-payment.py | 9 +++- yield-generator.py | 107 +++++++++++++++++++-------------------------- 2 files changed, 53 insertions(+), 63 deletions(-) diff --git a/send-payment.py b/send-payment.py index aef73c54..a83cf3f8 100644 --- a/send-payment.py +++ b/send-payment.py @@ -71,6 +71,13 @@ def finishcallback(self): def run(self): print 'waiting for all orders to certainly arrive' time.sleep(self.taker.waittime) + + crow = self.taker.db.execute('SELECT COUNT(DISTINCT counterparty) FROM orderbook;').fetchone() + counterparty_count = crow['COUNT(DISTINCT counterparty)'] + if counterparty_count < self.taker.makercount: + print 'not enough counterparties to fill order, ending' + return + orders = choose_order(self.taker.db, self.taker.amount, self.taker.makercount) print 'chosen orders to fill ' + str(orders) @@ -115,7 +122,7 @@ def main(): parser.add_option('-t', '--txfee', action='store', type='int', dest='txfee', default=5000, help='miner fee contribution') parser.add_option('-w', '--wait-time', action='store', type='float', dest='waittime', - help='wait time in seconds to allow orders to arrive', default=10) + help='wait time in seconds to allow orders to arrive', default=5) parser.add_option('-m', '--makercount', action='store', type='int', dest='makercount', help='how many makers to coinjoin with', default=2) (options, args) = parser.parse_args() diff --git a/yield-generator.py b/yield-generator.py index b925a463..be4db4ee 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -4,101 +4,84 @@ import bitcoin as btc import pprint -import pdb txfee = 1000 cjfee = '0.01' # 1% fee -mix_levels = 4 - -#is a maker for the purposes of generating a yield from held bitcoins -#for each mixing level, adds up the balance of all the addresses and put up -# a relative fee order, oid=mix depth -#TODO theres no need for seperate orders, just announce the order with the highest size -# and when it arrives choose which mixdepth you want, best algorithm is probably -# to try to keep coins concentrated into one depth so you can join larger amounts -#TODO when a nick with an open order quits, is kicked or dies, remove him from open orders +mix_levels = 5 + +#is a maker for the purposes of generating a yield from held +# bitcoins without ruining privacy for the taker, the taker could easily check +# the history of the utxos this bot sends, so theres not much incentive +# to ruin the privacy for barely any more yield +#sell-side algorithm: +#add up the value of each utxo for each mixing depth, +# announce a relative-fee order of the highest balance +#spent from utxos that try to make the highest balance even higher +# so try to keep coins concentrated in one mixing depth class YieldGenerator(Maker): def __init__(self, wallet): Maker.__init__(self, wallet) - + def create_my_orders(self): mix_utxo_list = self.wallet.get_mix_utxo_list() - orderlist = [] + mix_balance = {} for mixdepth, utxo_list in mix_utxo_list.iteritems(): total_value = 0 for utxo in utxo_list: total_value += self.wallet.unspent[utxo]['value'] - order = {'oid': mixdepth, 'ordertype': 'relorder', 'minsize': 0, - 'maxsize': total_value, 'txfee': txfee, 'cjfee': cjfee, - 'utxos': utxo_list} - orderlist.append(order) - return orderlist + mix_balance[mixdepth] = total_value + + #print mix_balance + max_mix = max(mix_balance, key=mix_balance.get) + order = {'oid': 0, 'ordertype': 'relorder', 'minsize': 0, + 'maxsize': mix_balance[max_mix], 'txfee': txfee, 'cjfee': cjfee, + 'mix_balance': mix_balance} + return [order] def oid_to_order(self, oid, amount): - order = [o for o in self.orderlist if o['oid'] == oid][0] + mix_balance = self.orderlist[0]['mix_balance'] + max_mix = max(mix_balance, key=mix_balance.get) + + #algo attempts to make the largest-balance mixing depth get an even larger balance + mixdepth = (max_mix - 1) % self.wallet.max_mix_depth + while True: + if mixdepth in mix_balance and mix_balance[mixdepth] > amount: + break + mixdepth = (mixdepth - 1) % self.wallet.max_mix_depth + #mixdepth is the chosen depth we'll be spending from + + mix_utxo_list = self.wallet.get_mix_utxo_list() unspent = [{'utxo': utxo, 'value': self.wallet.unspent[utxo]['value']} - for utxo in order['utxos']] + for utxo in mix_utxo_list[mixdepth]] inputs = btc.select(unspent, amount) - mixdepth = oid cj_addr = self.wallet.get_receive_addr((mixdepth + 1) % self.wallet.max_mix_depth) change_addr = self.wallet.get_change_addr(mixdepth) return [i['utxo'] for i in inputs], cj_addr, change_addr def on_tx_unconfirmed(self, cjorder, balance, removed_utxos): - #want to replace the current relorders with the same - # thing except reduced maxvalue to take into account the use - source_mixdepth = self.wallet.addr_cache[removed_utxos.values()[0]['address']][0] - debug('source mixdepth = %d' % (source_mixdepth)) - removed_utxos_balance = sum([addrvalue['value'] for addrvalue in removed_utxos.values()]) - debug('removed_utxos_balance = %d' % (removed_utxos_balance)) - - oldorder = [order for order in self.orderlist if order['oid'] == source_mixdepth][0] - neworder = oldorder.copy() - neworder['maxsize'] = oldorder['maxsize'] - removed_utxos_balance - [neworder['utxos'].remove(u) for u in removed_utxos.keys()] - #TODO if the maxsize left is zero or below the dust limit, just cancel the order - debug('neworder\n' + pprint.pformat(neworder)) - - #to_announce = self.create_my_orders() + #if the balance of the highest-balance mixing depth change then reannounce it + oldorder = self.orderlist[0] + neworder = self.create_my_orders()[0] + if oldorder['maxsize'] == neworder['maxsize']: + return ([], []) + else: + return ([], [neworder]) return ([], [neworder]) def on_tx_confirmed(self, cjorder, confirmations, txid, balance, added_utxos): - #add the new available utxos to the maxsize and announce it - # if we dont have a mixdepth of that level, make one - - to_announce = [] - for utxo, addrvalue in added_utxos.iteritems(): - mixdepth = self.wallet.addr_cache[addrvalue['address']][0] - debug('mixdepth=%d' % (mixdepth)) - oldorder_search = [order for order in self.orderlist if order['oid'] == mixdepth] - debug('len=' + str(len(oldorder_search)) + ' oldorder_search=\n' + pprint.pformat(oldorder_search)) - if len(oldorder_search) == 0: - #there were no existing orders at that mixing depth - neworder = {'oid': mixdepth, 'ordertype': 'relorder', 'minsize': 0, - 'maxsize': addrvalue['value'], 'txfee': txfee, 'cjfee': cjfee, - 'utxos': [utxo]} - else: - #assert len(oldorder_search) == 1 - oldorder = oldorder_search[0] - neworder = oldorder.copy() - neworder['maxsize'] = oldorder['maxsize'] + addrvalue['value'] - neworder['utxos'].append(utxo) - to_announce.append(neworder) - return ([], to_announce) + return self.on_tx_unconfirmed(None, None, None) def main(): import sys seed = sys.argv[1] #btc.sha256('dont use brainwallets except for holding testnet coins') + from socket import gethostname + nickname = 'yield-gen-' + btc.sha256(gethostname())[:6] print 'downloading wallet history' wallet = Wallet(seed, max_mix_depth = mix_levels) wallet.download_wallet_history() wallet.find_unspent_addresses() - - - from socket import gethostname - nickname = 'yield-gen-' + btc.sha256(gethostname())[:6] - + maker = YieldGenerator(wallet) print 'connecting to irc' maker.run(HOST, PORT, nickname, CHANNEL) From 1265c4bf6f47dabb50d8729c9ce26e216eeddab5 Mon Sep 17 00:00:00 2001 From: Belcher Date: Tue, 30 Dec 2014 22:16:04 +0000 Subject: [PATCH 033/409] edited options slightly --- README.txt | 3 +++ send-payment.py | 9 +++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.txt b/README.txt index fe597a09..d51b510f 100644 --- a/README.txt +++ b/README.txt @@ -90,6 +90,9 @@ have the taker enforce this, look up the txhash of the maker's utxo and make sur TODO implement rate limiting for irc.privmsg to stop the bot being killed due to flood i suggest creating a thread that only dispatches/writes to the irc socket +TODO sort out the nick = nick + '_' stuff in irclib +its not a good way of doing it + #TODO encrypt messages between taker and maker, to stop trivial server eavesdropping # but that wont stop mitm # after chats on irc, easiest is to do Trust On First Use, maker sends a pubkey over diff --git a/send-payment.py b/send-payment.py index a83cf3f8..2ccf7ec1 100644 --- a/send-payment.py +++ b/send-payment.py @@ -119,11 +119,11 @@ def main(): parser = OptionParser(usage='usage: %prog [options] [seed] [amount] [destaddr]', description='Sends a single payment from your wallet to an given address' + ' using coinjoin and then switches off.') - parser.add_option('-t', '--txfee', action='store', type='int', dest='txfee', - default=5000, help='miner fee contribution') + parser.add_option('-f', '--txfee', action='store', type='int', dest='txfee', + default=10000, help='miner fee contribution') parser.add_option('-w', '--wait-time', action='store', type='float', dest='waittime', help='wait time in seconds to allow orders to arrive', default=5) - parser.add_option('-m', '--makercount', action='store', type='int', dest='makercount', + parser.add_option('-N', '--makercount', action='store', type='int', dest='makercount', help='how many makers to coinjoin with', default=2) (options, args) = parser.parse_args() @@ -134,9 +134,6 @@ def main(): amount = int(args[1]) destaddr = args[2] - - #python send-payment.py -w 2 67286d672a2980ca30f7465084e90447 20000000 moovynzW3fioyQZEACSg9L27HjefZ3F5m2 - from socket import gethostname nickname = 'payer-' + btc.sha256(gethostname())[:6] From 55311e45f9232a28a60400ec75098ef28c31b59c Mon Sep 17 00:00:00 2001 From: Belcher Date: Tue, 30 Dec 2014 23:10:58 +0000 Subject: [PATCH 034/409] added nickserv support --- irclib.py | 2 ++ yield-generator.py | 9 +++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/irclib.py b/irclib.py index 4ba26762..8ab2999a 100644 --- a/irclib.py +++ b/irclib.py @@ -43,6 +43,7 @@ def on_welcome(self): pass def on_set_topic(self, newtopic): pass def on_leave(self, nick): pass def on_disconnect(self): pass + def on_connect(self): pass #TODO implement on_nick_change def close(self): @@ -95,6 +96,7 @@ def __handle_line(self, line): self.lockcond.notify() self.lockcond.release() elif chunks[1] == '376': #end of motd + self.on_connect() self.send_raw('JOIN ' + self.channel) elif chunks[1] == '433': #nick in use self.nick += '_' diff --git a/yield-generator.py b/yield-generator.py index be4db4ee..5a152468 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -2,12 +2,15 @@ from maker import * import bitcoin as btc +import time import pprint txfee = 1000 cjfee = '0.01' # 1% fee mix_levels = 5 +nickname = 'yield-generate' +nickserv_password = '' #is a maker for the purposes of generating a yield from held # bitcoins without ruining privacy for the taker, the taker could easily check @@ -22,6 +25,10 @@ class YieldGenerator(Maker): def __init__(self, wallet): Maker.__init__(self, wallet) + def on_connect(self): + if len(nickserv_password) > 0: + self.privmsg('NickServ', 'identify ' + nickserv_password) + def create_my_orders(self): mix_utxo_list = self.wallet.get_mix_utxo_list() mix_balance = {} @@ -74,8 +81,6 @@ def on_tx_confirmed(self, cjorder, confirmations, txid, balance, added_utxos): def main(): import sys seed = sys.argv[1] #btc.sha256('dont use brainwallets except for holding testnet coins') - from socket import gethostname - nickname = 'yield-gen-' + btc.sha256(gethostname())[:6] print 'downloading wallet history' wallet = Wallet(seed, max_mix_depth = mix_levels) From 0d15de13aa6f6d0034e6b00690dd5494b3dbf10f Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Wed, 31 Dec 2014 17:06:24 +0200 Subject: [PATCH 035/409] added libnacl and tests --- enc_wrapper.py | 60 +++++ enc_wrapper_tester.py | 38 +++ libnacl/__init__.py | 582 ++++++++++++++++++++++++++++++++++++++++++ libnacl/base.py | 64 +++++ libnacl/blake.py | 44 ++++ libnacl/dual.py | 34 +++ libnacl/encode.py | 63 +++++ libnacl/public.py | 81 ++++++ libnacl/secret.py | 44 ++++ libnacl/sign.py | 55 ++++ libnacl/utils.py | 68 +++++ libnacl/version.py | 1 + 12 files changed, 1134 insertions(+) create mode 100644 enc_wrapper.py create mode 100644 enc_wrapper_tester.py create mode 100644 libnacl/__init__.py create mode 100644 libnacl/base.py create mode 100644 libnacl/blake.py create mode 100644 libnacl/dual.py create mode 100644 libnacl/encode.py create mode 100644 libnacl/public.py create mode 100644 libnacl/secret.py create mode 100644 libnacl/sign.py create mode 100644 libnacl/utils.py create mode 100644 libnacl/version.py diff --git a/enc_wrapper.py b/enc_wrapper.py new file mode 100644 index 00000000..7e761720 --- /dev/null +++ b/enc_wrapper.py @@ -0,0 +1,60 @@ +#A wrapper for public key +#authenticated encryption +#using Diffie Hellman key +#exchange to set up a +#symmetric encryption. + +import libnacl.public +import binascii + +def init_keypair(fname = None): + '''Create a new encryption + keypair; stored in file fname + if provided. The keypair object + is returned. + ''' + kp = libnacl.public.SecretKey() + if fname: + #Note: handles correct file permissions + kp.save(fname) + return kp + +#the next two functions are useful +#for exchaging pubkeys with counterparty +def get_pubkey(kp, as_hex=False): + '''Given a keypair object, + return its public key, + optionally in hex.''' + return kp.hex_pk() if as_hex else kp.pk + +def init_pubkey(hexpk, fname = None): + '''Create a pubkey object from a + hex formatted string. + Save to file fname if specified. + ''' + pk = libnacl.public.PublicKey(binascii.unhexlify(hexpk)) + if fname: + pk.save(fname) + return pk + +def as_init_encryption(kp, c_pk): + '''Given an initialised + keypair kp and a counterparty + pubkey c_pk, create a Box + ready for encryption/decryption. + ''' + return libnacl.public.Box(kp.sk,c_pk) +''' +After initialisation, it's possible +to use the box object returned from +as_init_encryption to directly change +from plaintext to ciphertext: + ciphertext = box.encrypt(plaintext) + plaintext = box.decrypt(ciphertext) +Notes: + 1. use binary format for ctext/ptext + 2. Nonce is handled at the implementation layer. +''' + +#TODO: Sign, verify. + diff --git a/enc_wrapper_tester.py b/enc_wrapper_tester.py new file mode 100644 index 00000000..2115e337 --- /dev/null +++ b/enc_wrapper_tester.py @@ -0,0 +1,38 @@ +import enc_wrapper as e +import binascii + +alice_kp = e.init_keypair(fname='alice1.txt') +bob_kp = e.init_keypair(fname='bob1.txt') + +#this is the DH key exchange part +bob_otwpk = e.get_pubkey(bob_kp,True) +print "sending pubkey from bob to alice: "+bob_otwpk +alice_otwpk = e.get_pubkey(alice_kp,True) +print "sending pubkey from bob to alice: "+alice_otwpk + +bob_pk = e.init_pubkey(bob_otwpk) +alice_box = e.as_init_encryption(alice_kp,bob_pk) +alice_pk = e.init_pubkey(alice_otwpk) +bob_box = e.as_init_encryption(bob_kp,alice_pk) + +#now Alice and Bob can use their 'box' +#constructs (both of which utilise the same +#shared secret) to perform encryption/decryption +for i in range(8): + alice_message = 'Attack at dawn ! \n\n x'+str(i) + + otw_amsg = alice_box.encrypt(alice_message) + print "Sending from alice to bob: " + otw_amsg + + bob_ptext = bob_box.decrypt(otw_amsg) + print "Bob received: " + bob_ptext + + bob_message = 'Not tonight Josephine.' + str(i) * 45 + otw_bmsg = bob_box.encrypt(bob_message) + print "Sending from bob to alice: " + otw_bmsg + + alice_ptext = alice_box.decrypt(otw_bmsg) + print "Alice received: " + alice_ptext + + + diff --git a/libnacl/__init__.py b/libnacl/__init__.py new file mode 100644 index 00000000..df266a2f --- /dev/null +++ b/libnacl/__init__.py @@ -0,0 +1,582 @@ +# -*- coding: utf-8 -*- +''' +Wrap libsodium routines +''' +# pylint: disable=C0103 +# Import libnacl libs +from libnacl.version import __version__ +# Import python libs +import ctypes +import sys + +__SONAMES = (13, 10, 5, 4) + + +def _get_nacl(): + ''' + Locate the nacl c libs to use + ''' + # Import libsodium + if sys.platform.startswith('win'): + try: + return ctypes.cdll.LoadLibrary('libsodium') + except OSError: + pass + for soname_ver in __SONAMES: + try: + return ctypes.cdll.LoadLibrary( + 'libsodium-{0}'.format(soname_ver) + ) + except OSError: + pass + try: + return ctypes.cdll.LoadLibrary('tweetnacl') + except OSError: + msg = ('Could not locate nacl lib, searched for libsodium, ' + 'tweetnacl') + raise OSError(msg) + elif sys.platform.startswith('darwin'): + try: + return ctypes.cdll.LoadLibrary('libsodium.dylib') + except OSError: + pass + try: + return ctypes.cdll.LoadLibrary('tweetnacl.dylib') + except OSError: + msg = ('Could not locate nacl lib, searched for libsodium, ' + 'tweetnacl') + raise OSError(msg) + else: + try: + return ctypes.cdll.LoadLibrary('libsodium.so') + except OSError: + pass + try: + return ctypes.cdll.LoadLibrary('/usr/local/lib/libsodium.so') + except OSError: + pass + + for soname_ver in __SONAMES: + try: + return ctypes.cdll.LoadLibrary( + 'libsodium.so.{0}'.format(soname_ver) + ) + except OSError: + pass + try: + return ctypes.cdll.LoadLibrary('tweetnacl.so') + except OSError: + msg = 'Could not locate nacl lib, searched for libsodium.so, ' + for soname_ver in __SONAMES: + msg += 'libsodium.so.{0}, '.format(soname_ver) + msg += ' and tweetnacl.so' + raise OSError(msg) + +nacl = _get_nacl() + +# Define constants +crypto_box_SECRETKEYBYTES = nacl.crypto_box_secretkeybytes() +crypto_box_PUBLICKEYBYTES = nacl.crypto_box_publickeybytes() +crypto_box_NONCEBYTES = nacl.crypto_box_noncebytes() +crypto_box_ZEROBYTES = nacl.crypto_box_zerobytes() +crypto_box_BOXZEROBYTES = nacl.crypto_box_boxzerobytes() +crypto_box_BEFORENMBYTES = nacl.crypto_box_beforenmbytes() +crypto_scalarmult_BYTES = nacl.crypto_scalarmult_bytes() +crypto_scalarmult_SCALARBYTES = nacl.crypto_scalarmult_scalarbytes() +crypto_sign_BYTES = nacl.crypto_sign_bytes() +crypto_sign_SEEDBYTES = nacl.crypto_sign_secretkeybytes() // 2 +crypto_sign_PUBLICKEYBYTES = nacl.crypto_sign_publickeybytes() +crypto_sign_SECRETKEYBYTES = nacl.crypto_sign_secretkeybytes() +crypto_box_MACBYTES = crypto_box_ZEROBYTES - crypto_box_BOXZEROBYTES +crypto_secretbox_KEYBYTES = nacl.crypto_secretbox_keybytes() +crypto_secretbox_NONCEBYTES = nacl.crypto_secretbox_noncebytes() +crypto_secretbox_ZEROBYTES = nacl.crypto_secretbox_zerobytes() +crypto_secretbox_BOXZEROBYTES = nacl.crypto_secretbox_boxzerobytes() +crypto_secretbox_MACBYTES = crypto_secretbox_ZEROBYTES - crypto_secretbox_BOXZEROBYTES +crypto_stream_KEYBYTES = nacl.crypto_stream_keybytes() +crypto_stream_NONCEBYTES = nacl.crypto_stream_noncebytes() +crypto_auth_BYTES = nacl.crypto_auth_bytes() +crypto_auth_KEYBYTES = nacl.crypto_auth_keybytes() +crypto_onetimeauth_BYTES = nacl.crypto_onetimeauth_bytes() +crypto_onetimeauth_KEYBYTES = nacl.crypto_onetimeauth_keybytes() +crypto_generichash_BYTES = nacl.crypto_generichash_bytes() +crypto_generichash_BYTES_MIN = nacl.crypto_generichash_bytes_min() +crypto_generichash_BYTES_MAX = nacl.crypto_generichash_bytes_max() +crypto_generichash_KEYBYTES = nacl.crypto_generichash_keybytes() +crypto_generichash_KEYBYTES_MIN = nacl.crypto_generichash_keybytes_min() +crypto_generichash_KEYBYTES_MAX = nacl.crypto_generichash_keybytes_max() +crypto_scalarmult_curve25519_BYTES = nacl.crypto_scalarmult_curve25519_bytes() +crypto_hash_BYTES = nacl.crypto_hash_sha512_bytes() +crypto_hash_sha256_BYTES = nacl.crypto_hash_sha256_bytes() +crypto_hash_sha512_BYTES = nacl.crypto_hash_sha512_bytes() +# pylint: enable=C0103 + + +# Define exceptions +class CryptError(Exception): + ''' + Base Exception for cryptographic errors + ''' + +# Pubkey defs + + +def crypto_box_keypair(): + ''' + Generate and return a new keypair + + pk, sk = nacl.crypto_box_keypair() + ''' + pk = ctypes.create_string_buffer(crypto_box_PUBLICKEYBYTES) + sk = ctypes.create_string_buffer(crypto_box_SECRETKEYBYTES) + nacl.crypto_box_keypair(pk, sk) + return pk.raw, sk.raw + + +def crypto_box(msg, nonce, pk, sk): + ''' + Using a public key and a secret key encrypt the given message. A nonce + must also be passed in, never reuse the nonce + + enc_msg = nacl.crypto_box('secret message', , , ) + ''' + if len(pk) != crypto_box_PUBLICKEYBYTES: + raise ValueError('Invalid public key') + if len(sk) != crypto_box_SECRETKEYBYTES: + raise ValueError('Invalid secret key') + if len(nonce) != crypto_box_NONCEBYTES: + raise ValueError('Invalid nonce') + pad = b'\x00' * crypto_box_ZEROBYTES + msg + c = ctypes.create_string_buffer(len(pad)) + ret = nacl.crypto_box(c, pad, ctypes.c_ulonglong(len(pad)), nonce, pk, sk) + if ret: + raise CryptError('Unable to encrypt message') + return c.raw[crypto_box_BOXZEROBYTES:] + + +def crypto_box_open(ctxt, nonce, pk, sk): + ''' + Decrypts a message given the receivers private key, and senders public key + ''' + if len(pk) != crypto_box_PUBLICKEYBYTES: + raise ValueError('Invalid public key') + if len(sk) != crypto_box_SECRETKEYBYTES: + raise ValueError('Invalid secret key') + if len(nonce) != crypto_box_NONCEBYTES: + raise ValueError('Invalid nonce') + pad = b'\x00' * crypto_box_BOXZEROBYTES + ctxt + msg = ctypes.create_string_buffer(len(pad)) + ret = nacl.crypto_box_open( + msg, + pad, + ctypes.c_ulonglong(len(pad)), + nonce, + pk, + sk) + if ret: + raise CryptError('Unable to decrypt ciphertext') + return msg.raw[crypto_box_ZEROBYTES:] + + +def crypto_box_beforenm(pk, sk): + ''' + Partially performs the computation required for both encryption and decryption of data + ''' + if len(pk) != crypto_box_PUBLICKEYBYTES: + raise ValueError('Invalid public key') + if len(sk) != crypto_box_SECRETKEYBYTES: + raise ValueError('Invalid secret key') + k = ctypes.create_string_buffer(crypto_box_BEFORENMBYTES) + ret = nacl.crypto_box_beforenm(k, pk, sk) + if ret: + raise CryptError('Unable to compute shared key') + return k.raw + + +def crypto_box_afternm(msg, nonce, k): + ''' + Encrypts a given a message, using partial computed data + ''' + if len(k) != crypto_box_BEFORENMBYTES: + raise ValueError('Invalid shared key') + if len(nonce) != crypto_box_NONCEBYTES: + raise ValueError('Invalid nonce') + pad = b'\x00' * crypto_box_ZEROBYTES + msg + ctxt = ctypes.create_string_buffer(len(pad)) + ret = nacl.crypto_box_afternm(ctxt, pad, ctypes.c_ulonglong(len(pad)), nonce, k) + if ret: + raise ValueError('Unable to encrypt messsage') + return ctxt.raw[crypto_box_BOXZEROBYTES:] + + +def crypto_box_open_afternm(ctxt, nonce, k): + ''' + Decrypts a ciphertext ctxt given k + ''' + if len(k) != crypto_box_BEFORENMBYTES: + raise ValueError('Invalid shared key') + if len(nonce) != crypto_box_NONCEBYTES: + raise ValueError('Invalid nonce') + pad = b'\x00' * crypto_box_BOXZEROBYTES + ctxt + msg = ctypes.create_string_buffer(len(pad)) + ret = nacl.crypto_box_open_afternm( + msg, + pad, + ctypes.c_ulonglong(len(pad)), + nonce, + k) + if ret: + raise ValueError('unable to decrypt message') + return msg.raw[crypto_box_ZEROBYTES:] + +# Signing functions + + +def crypto_sign_keypair(): + ''' + Generates a signing/verification key pair + ''' + vk = ctypes.create_string_buffer(crypto_sign_PUBLICKEYBYTES) + sk = ctypes.create_string_buffer(crypto_sign_SECRETKEYBYTES) + ret = nacl.crypto_sign_keypair(vk, sk) + if ret: + raise ValueError('Failed to generate keypair') + return vk.raw, sk.raw + + +def crypto_sign(msg, sk): + ''' + Sign the given message witht he given signing key + ''' + sig = ctypes.create_string_buffer(len(msg) + crypto_sign_BYTES) + slen = ctypes.pointer(ctypes.c_ulonglong()) + ret = nacl.crypto_sign( + sig, + slen, + msg, + ctypes.c_ulonglong(len(msg)), + sk) + if ret: + raise ValueError('Failed to sign message') + return sig.raw + + +def crypto_sign_seed_keypair(seed): + ''' + Computes and returns the secret adn verify keys from the given seed + ''' + if len(seed) != crypto_sign_SEEDBYTES: + raise ValueError('Invalid Seed') + sk = ctypes.create_string_buffer(crypto_sign_SECRETKEYBYTES) + vk = ctypes.create_string_buffer(crypto_sign_PUBLICKEYBYTES) + + ret = nacl.crypto_sign_seed_keypair(vk, sk, seed) + if ret: + raise CryptError('Failed to generate keypair from seed') + return (vk.raw, sk.raw) + + +def crypto_sign_open(sig, vk): + ''' + Verifies the signed message sig using the signer's verification key + ''' + msg = ctypes.create_string_buffer(len(sig)) + msglen = ctypes.c_ulonglong() + msglenp = ctypes.pointer(msglen) + ret = nacl.crypto_sign_open( + msg, + msglenp, + sig, + ctypes.c_ulonglong(len(sig)), + vk) + if ret: + raise ValueError('Failed to validate message') + return msg.raw[:msglen.value] # pylint: disable=invalid-slice-index + +# Authenticated Symmetric Encryption + + +def crypto_secretbox(msg, nonce, key): + ''' + Encrypts and authenticates a message using the given secret key, and nonce + ''' + pad = b'\x00' * crypto_secretbox_ZEROBYTES + msg + ctxt = ctypes.create_string_buffer(len(pad)) + ret = nacl.crypto_secretbox(ctxt, pad, ctypes.c_ulonglong(len(pad)), nonce, key) + if ret: + raise ValueError('Failed to encrypt message') + return ctxt.raw[crypto_secretbox_BOXZEROBYTES:] + + +def crypto_secretbox_open(ctxt, nonce, key): + ''' + Decrypts a ciphertext ctxt given the receivers private key, and senders + public key + ''' + pad = b'\x00' * crypto_secretbox_BOXZEROBYTES + ctxt + msg = ctypes.create_string_buffer(len(pad)) + ret = nacl.crypto_secretbox_open( + msg, + pad, + ctypes.c_ulonglong(len(pad)), + nonce, + key) + if ret: + raise ValueError('Failed to decrypt message') + return msg.raw[crypto_secretbox_ZEROBYTES:] + +# Symmetric Encryption + + +def crypto_stream(slen, nonce, key): + ''' + Generates a stream using the given secret key and nonce + ''' + stream = ctypes.create_string_buffer(slen) + ret = nacl.crypto_stream(stream, ctypes.c_ulonglong(slen), nonce, key) + if ret: + raise ValueError('Failed to init stream') + return stream.raw + + +def crypto_stream_xor(msg, nonce, key): + ''' + Encrypts the given message using the given secret key and nonce + + The crypto_stream_xor function guarantees that the ciphertext is the + plaintext (xor) the output of crypto_stream. Consequently + crypto_stream_xor can also be used to decrypt + ''' + stream = ctypes.create_string_buffer(len(msg)) + ret = nacl.crypto_stream_xor( + stream, + msg, + ctypes.c_ulonglong(len(msg)), + nonce, + key) + if ret: + raise ValueError('Failed to init stream') + return stream.raw + + +# Authentication + + +def crypto_auth(msg, key): + ''' + Constructs a one time authentication token for the given message msg + using a given secret key + ''' + tok = ctypes.create_string_buffer(crypto_auth_BYTES) + ret = nacl.crypto_auth(tok, msg, ctypes.c_ulonglong(len(msg)), key) + if ret: + raise ValueError('Failed to auth msg') + return tok.raw[:crypto_auth_BYTES] + + +def crypto_auth_verify(msg, key): + ''' + Verifies that the given authentication token is correct for the given + message and key + ''' + tok = ctypes.create_string_buffer(crypto_auth_BYTES) + ret = nacl.crypto_auth_verify(tok, msg, ctypes.c_ulonglong(len(msg)), key) + if ret: + raise ValueError('Failed to auth msg') + return tok.raw[:crypto_auth_BYTES] + +# One time authentication + + +def crypto_onetimeauth(msg, key): + ''' + Constructs a one time authentication token for the given message msg using + a given secret key + ''' + tok = ctypes.create_string_buffer(crypto_onetimeauth_BYTES) + ret = nacl.crypto_onetimeauth(tok, msg, ctypes.c_ulonglong(len(msg)), key) + if ret: + raise ValueError('Failed to auth msg') + return tok.raw[:crypto_onetimeauth_BYTES] + + +def crypto_onetimeauth_verify(msg, key): + ''' + Verifies that the given authentication token is correct for the given + message and key + ''' + tok = ctypes.create_string_buffer(crypto_onetimeauth_BYTES) + ret = nacl.crypto_onetimeauth(tok, msg, ctypes.c_ulonglong(len(msg)), key) + if ret: + raise ValueError('Failed to auth msg') + return tok.raw[:crypto_onetimeauth_BYTES] + +# Hashing + + +def crypto_hash(msg): + ''' + Compute a hash of the given message + ''' + hbuf = ctypes.create_string_buffer(crypto_hash_BYTES) + nacl.crypto_hash(hbuf, msg, ctypes.c_ulonglong(len(msg))) + return hbuf.raw + + +def crypto_hash_sha256(msg): + ''' + Compute the sha256 hash of the given message + ''' + hbuf = ctypes.create_string_buffer(crypto_hash_sha256_BYTES) + nacl.crypto_hash_sha256(hbuf, msg, ctypes.c_ulonglong(len(msg))) + return hbuf.raw + + +def crypto_hash_sha512(msg): + ''' + Compute the sha512 hash of the given message + ''' + hbuf = ctypes.create_string_buffer(crypto_hash_sha512_BYTES) + nacl.crypto_hash_sha512(hbuf, msg, ctypes.c_ulonglong(len(msg))) + return hbuf.raw + +# Generic Hash + + +def crypto_generichash(msg, key=None): + ''' + Compute the blake2 hash of the given message with a given key + ''' + hbuf = ctypes.create_string_buffer(crypto_generichash_BYTES) + if key: + key_len = len(key) + else: + key_len = 0 + nacl.crypto_generichash( + hbuf, + ctypes.c_ulonglong(len(hbuf)), + msg, + ctypes.c_ulonglong(len(msg)), + key, + ctypes.c_ulonglong(key_len)) + return hbuf.raw + +# scalarmult + + +def crypto_scalarmult_base(n): + ''' + Computes and returns the scalar product of a standard group element and an + integer "n". + ''' + buf = ctypes.create_string_buffer(crypto_scalarmult_BYTES) + ret = nacl.crypto_scalarmult_base(buf, n) + if ret: + raise CryptError('Failed to compute scalar product') + return buf.raw + +# String cmp + + +def crypto_verify_16(string1, string2): + ''' + Compares the first crypto_verify_16_BYTES of the given strings + + The time taken by the function is independent of the contents of string1 + and string2. In contrast, the standard C comparison function + memcmp(string1,string2,16) takes time that is dependent on the longest + matching prefix of string1 and string2. This often allows for easy + timing attacks. + ''' + return not nacl.crypto_verify_16(string1, string2) + + +def crypto_verify_32(string1, string2): + ''' + Compares the first crypto_verify_32_BYTES of the given strings + + The time taken by the function is independent of the contents of string1 + and string2. In contrast, the standard C comparison function + memcmp(string1,string2,16) takes time that is dependent on the longest + matching prefix of string1 and string2. This often allows for easy + timing attacks. + ''' + return not nacl.crypto_verify_32(string1, string2) + + +# Random byte generation + +def randombytes(size): + ''' + Return a string of random bytes of the given size + ''' + buf = ctypes.create_string_buffer(size) + nacl.randombytes(buf, ctypes.c_ulonglong(size)) + return buf.raw + + +def randombytes_buf(size): + ''' + Return a string of random bytes of the given size + ''' + size = int(size) + buf = ctypes.create_string_buffer(size) + nacl.randombytes_buf(buf, size) + return buf.raw + + +def randombytes_close(): + ''' + Close the file descriptor or the handle for the cryptographic service + provider + ''' + nacl.randombytes_close() + + +def randombytes_random(): + ''' + Return a random 32-bit unsigned value + ''' + return nacl.randombytes_random() + + +def randombytes_stir(): + ''' + Generate a new key for the pseudorandom number generator + + The file descriptor for the entropy source is kept open, so that the + generator can be reseeded even in a chroot() jail. + ''' + nacl.randombytes_stir() + + +def randombytes_uniform(upper_bound): + ''' + Return a value between 0 and upper_bound using a uniform distribution + ''' + return nacl.randombytes_uniform(upper_bound) + + +# Utility functions + +def sodium_library_version_major(): + ''' + Return the major version number + ''' + return nacl.sodium_library_version_major() + + +def sodium_library_version_minor(): + ''' + Return the minor version number + ''' + return nacl.sodium_library_version_minor() + + +def sodium_version_string(): + ''' + Return the version string + ''' + func = nacl.sodium_version_string + func.restype = ctypes.c_char_p + return func() diff --git a/libnacl/base.py b/libnacl/base.py new file mode 100644 index 00000000..6885b548 --- /dev/null +++ b/libnacl/base.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +''' +Implement the base key object for other keys to inherit convenience functions +''' +# Import libnacl libs +import libnacl.encode + +# Import python libs +import os +import stat + +class BaseKey(object): + ''' + Include methods for key management convenience + ''' + def hex_sk(self): + if hasattr(self, 'sk'): + return libnacl.encode.hex_encode(self.sk) + else: + return '' + + def hex_pk(self): + if hasattr(self, 'pk'): + return libnacl.encode.hex_encode(self.pk) + + def hex_vk(self): + if hasattr(self, 'vk'): + return libnacl.encode.hex_encode(self.vk) + + def hex_seed(self): + if hasattr(self, 'seed'): + return libnacl.encode.hex_encode(self.seed) + + def save(self, path, serial='json'): + ''' + Safely save keys with perms of 0400 + ''' + pre = {} + sk = self.hex_sk() + pk = self.hex_pk() + vk = self.hex_vk() + seed = self.hex_seed() + if sk and pk: + pre['priv'] = sk.decode('utf-8') + if pk: + pre['pub'] = pk.decode('utf-8') + if vk: + pre['verify'] = vk.decode('utf-8') + if seed: + pre['sign'] = seed.decode('utf-8') + if serial == 'msgpack': + import msgpack + packaged = msgpack.dumps(pre) + elif serial == 'json': + import json + packaged = json.dumps(pre) + + perm_other = stat.S_IWOTH | stat.S_IXOTH | stat.S_IWOTH + perm_group = stat.S_IXGRP | stat.S_IWGRP | stat.S_IRWXG + + cumask = os.umask(perm_other | perm_group) + with open(path, 'w+') as fp_: + fp_.write(packaged) + os.umask(cumask) diff --git a/libnacl/blake.py b/libnacl/blake.py new file mode 100644 index 00000000..8da7d697 --- /dev/null +++ b/libnacl/blake.py @@ -0,0 +1,44 @@ +''' +Mimic very closely the python hashlib classes for blake2b + +NOTE: + This class does not yet implement streaming the msg into the + hash function via the update method +''' + +# Import python libs +import binascii + +# Import libnacl libs +import libnacl + + +class Blake2b(object): + ''' + Manage a Blake2b hash + ''' + def __init__(self, msg, key=None): + self.msg = msg + self.key = key + self.raw_digest = libnacl.crypto_generichash(msg, key) + self.digest_size = len(self.raw_digest) + + def digest(self): + ''' + Return the digest of the string + ''' + return self.raw_digest + + def hexdigest(self): + ''' + Return the hex digest of the string + ''' + return binascii.hexlify(self.raw_digest) + + +def blake2b(msg, key=None): + ''' + Create and return a Blake2b object to mimic the behavior of the python + hashlib functions + ''' + return Blake2b(msg, key) diff --git a/libnacl/dual.py b/libnacl/dual.py new file mode 100644 index 00000000..c48fb673 --- /dev/null +++ b/libnacl/dual.py @@ -0,0 +1,34 @@ +''' +The dual key system allows for the creation of keypairs that contain both +cryptographic and signing keys +''' +# import libnacl libs +import libnacl +import libnacl.base +import libnacl.public +import libnacl.sign + + +class DualSecret(libnacl.base.BaseKey): + ''' + Manage crypt and sign keys in one object + ''' + def __init__(self, crypt=None, sign=None): + self.crypt = libnacl.public.SecretKey(crypt) + self.signer = libnacl.sign.Signer(sign) + self.sk = self.crypt.sk + self.seed = self.signer.seed + self.pk = self.crypt.pk + self.vk = self.signer.vk + + def sign(self, msg): + ''' + Sign the given message + ''' + return self.signer.sign(msg) + + def signature(self, msg): + ''' + Return just the signature for the message + ''' + return self.signer.signature(msg) diff --git a/libnacl/encode.py b/libnacl/encode.py new file mode 100644 index 00000000..efbfe998 --- /dev/null +++ b/libnacl/encode.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +''' +Build in routines and classes to simplify encoding routines +''' +# Import python libs +import base64 +import binascii + + +def hex_encode(data): + ''' + Hex encode data + ''' + return binascii.hexlify(data) + + +def hex_decode(data): + ''' + Hex decode data + ''' + return binascii.unhexlify(data) + + +def base16_encode(data): + ''' + Base32 encode data + ''' + return base64.b16encode(data) + + +def base16_decode(data): + ''' + Base16 decode data + ''' + return base64.b16decode(data) + + +def base32_encode(data): + ''' + Base16 encode data + ''' + return base64.b32encode(data) + + +def base32_decode(data): + ''' + Base32 decode data + ''' + return base64.b32decode(data) + + +def base64_encode(data): + ''' + Base16 encode data + ''' + return base64.b64encode(data) + + +def base64_decode(data): + ''' + Base32 decode data + ''' + return base64.b64decode(data) diff --git a/libnacl/public.py b/libnacl/public.py new file mode 100644 index 00000000..81f5b342 --- /dev/null +++ b/libnacl/public.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +''' +High level classes and routines around public key encryption and decryption +''' +# import libnacl libs +import libnacl +import libnacl.utils +import libnacl.encode +import libnacl.dual +import libnacl.base + + +class PublicKey(libnacl.base.BaseKey): + ''' + This class is used to manage public keys + ''' + def __init__(self, pk): + self.pk = pk + + +class SecretKey(libnacl.base.BaseKey): + ''' + This class is used to manage keypairs + ''' + def __init__(self, sk=None): + ''' + If a secret key is not passed in then it will be generated + ''' + if sk is None: + self.pk, self.sk = libnacl.crypto_box_keypair() + elif len(sk) == libnacl.crypto_box_SECRETKEYBYTES: + self.sk = sk + self.pk = libnacl.crypto_scalarmult_base(sk) + else: + raise ValueError('Passed in invalid secret key') + + +class Box(object): + ''' + TheBox class is used to create cryptographic boxes and unpack + cryptographic boxes + ''' + def __init__(self, sk, pk): + if isinstance(sk, (SecretKey, libnacl.dual.DualSecret)): + sk = sk.sk + if isinstance(pk, (SecretKey, libnacl.dual.DualSecret)): + raise ValueError('Passed in secret key as public key') + if isinstance(pk, PublicKey): + pk = pk.pk + if pk and sk: + self._k = libnacl.crypto_box_beforenm(pk, sk) + + def encrypt(self, msg, nonce=None, pack_nonce=True): + ''' + Encrypt the given message with the given nonce, if the nonce is not + provided it will be generated from the libnacl.utils.rand_nonce + function + ''' + if nonce is None: + nonce = libnacl.utils.rand_nonce() + elif len(nonce) != libnacl.crypto_box_NONCEBYTES: + raise ValueError('Invalid nonce size') + ctxt = libnacl.crypto_box_afternm(msg, nonce, self._k) + if pack_nonce: + return nonce + ctxt + else: + return nonce, ctxt + + def decrypt(self, ctxt, nonce=None): + ''' + Decrypt the given message, if a nonce is passed in attempt to decrypt + it with the given nonce, otherwise assum that the nonce is attached + to the message + ''' + if nonce is None: + nonce = ctxt[:libnacl.crypto_box_NONCEBYTES] + ctxt = ctxt[libnacl.crypto_box_NONCEBYTES:] + elif len(nonce) != libnacl.crypto_box_NONCEBYTES: + raise ValueError('Invalid nonce') + msg = libnacl.crypto_box_open_afternm(ctxt, nonce, self._k) + return msg diff --git a/libnacl/secret.py b/libnacl/secret.py new file mode 100644 index 00000000..12a2c235 --- /dev/null +++ b/libnacl/secret.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +''' +Utilities to make secret box encryption simple +''' +# Import libnacl +import libnacl +import libnacl.utils +import libnacl.base + + +class SecretBox(libnacl.base.BaseKey): + ''' + Manage symetric encryption using the salsa20 algorithm + ''' + def __init__(self, key=None): + if key is None: + key = libnacl.utils.salsa_key() + if len(key) != libnacl.crypto_secretbox_KEYBYTES: + raise ValueError('Invalid key') + self.sk = key + + def encrypt(self, msg, nonce=None): + ''' + Encrypt the given message. If a nonce is not given it will be + generated via the rand_nonce function + ''' + if nonce is None: + nonce = libnacl.utils.rand_nonce() + if len(nonce) != libnacl.crypto_secretbox_NONCEBYTES: + raise ValueError('Invalid Nonce') + ctxt = libnacl.crypto_secretbox(msg, nonce, self.sk) + return nonce + ctxt + + def decrypt(self, ctxt, nonce=None): + ''' + Decrypt the given message, if no nonce is given the nonce will be + extracted from the message + ''' + if nonce is None: + nonce = ctxt[:libnacl.crypto_secretbox_NONCEBYTES] + ctxt = ctxt[libnacl.crypto_secretbox_NONCEBYTES:] + if len(nonce) != libnacl.crypto_secretbox_NONCEBYTES: + raise ValueError('Invalid nonce') + return libnacl.crypto_secretbox_open(ctxt, nonce, self.sk) diff --git a/libnacl/sign.py b/libnacl/sign.py new file mode 100644 index 00000000..9dbd83f5 --- /dev/null +++ b/libnacl/sign.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +''' +High level routines to maintain signing keys and to sign and verify messages +''' +# Import libancl libs +import libnacl +import libnacl.base +import libnacl.encode + + +class Signer(libnacl.base.BaseKey): + ''' + The tools needed to sign messages + ''' + def __init__(self, seed=None): + ''' + Create a signing key, if not seed it supplied a keypair is generated + ''' + if seed: + if len(seed) != libnacl.crypto_sign_SEEDBYTES: + raise ValueError('Invalid seed bytes') + self.vk, self.sk = libnacl.crypto_sign_seed_keypair(seed) + else: + seed = libnacl.randombytes(libnacl.crypto_sign_SEEDBYTES) + self.vk, self.sk = libnacl.crypto_sign_seed_keypair(seed) + self.seed = seed + + def sign(self, msg): + ''' + Sign the given message with this key + ''' + return libnacl.crypto_sign(msg, self.sk) + + def signature(self, msg): + ''' + Return just the signature for the message + ''' + return libnacl.crypto_sign(msg, self.sk)[:libnacl.crypto_sign_BYTES] + + +class Verifier(libnacl.base.BaseKey): + ''' + Verify signed messages + ''' + def __init__(self, vk_hex): + ''' + Create a verification key from a hex encoded vkey + ''' + self.vk = libnacl.encode.hex_decode(vk_hex) + + def verify(self, msg): + ''' + Verify the message with tis key + ''' + return libnacl.crypto_sign_open(msg, self.vk) diff --git a/libnacl/utils.py b/libnacl/utils.py new file mode 100644 index 00000000..c8429d87 --- /dev/null +++ b/libnacl/utils.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + +import struct +import time + +# Import nacl libs +import libnacl +import libnacl.encode +import libnacl.public +import libnacl.sign +import libnacl.dual + + +def load_key(path, serial='json'): + ''' + Read in a key from a file and return the applicable key object based on + the contents of the file + ''' + with open(path, 'rb') as fp_: + packaged = fp_.read() + if serial == 'msgpack': + import msgpack + key_data = msgpack.loads(packaged) + elif serial == 'json': + import json + key_data = json.loads(packaged.decode(encoding='UTF-8')) + if 'priv' in key_data and 'sign' in key_data: + return libnacl.dual.DualSecret( + libnacl.encode.hex_decode(key_data['priv']), + libnacl.encode.hex_decode(key_data['sign'])) + elif 'priv' in key_data: + return libnacl.public.SecretKey( + libnacl.encode.hex_decode(key_data['priv'])) + elif 'sign' in key_data: + return libnacl.sign.Signer( + libnacl.encode.hex_decode(key_data['sign'])) + elif 'pub' in key_data: + return libnacl.public.PublicKey( + libnacl.encode.hex_decode(key_data['pub'])) + elif 'verify' in key_data: + return libnacl.sign.Verifier(key_data['verify']) + raise ValueError('Found no key data') + + +def salsa_key(): + ''' + Generates a salsa2020 key + ''' + return libnacl.randombytes(libnacl.crypto_secretbox_KEYBYTES) + + +def rand_nonce(): + ''' + Generates and returns a random bytestring of the size defined in libsodium + as crypto_box_NONCEBYTES + ''' + return libnacl.randombytes(libnacl.crypto_box_NONCEBYTES) + + +def time_nonce(): + ''' + Generates and returns a nonce as in rand_nonce() but using a timestamp for the first 8 bytes. + + This function now exists mostly for backwards compatibility, as rand_nonce() is usually preferred. + ''' + nonce = rand_nonce() + return (struct.pack('=d', time.time()) + nonce)[:len(nonce)] + diff --git a/libnacl/version.py b/libnacl/version.py new file mode 100644 index 00000000..96e3ce8d --- /dev/null +++ b/libnacl/version.py @@ -0,0 +1 @@ +__version__ = '1.4.0' From 6956e94c08a8e1f5ae660ca018967374b8de7fb6 Mon Sep 17 00:00:00 2001 From: Belcher Date: Wed, 31 Dec 2014 15:57:44 +0000 Subject: [PATCH 036/409] moved a function, renamed --- maker.py | 36 +++++++++++++++---------------- send-payment.py => sendpayment.py | 19 +++++++++------- 2 files changed, 29 insertions(+), 26 deletions(-) rename send-payment.py => sendpayment.py (89%) diff --git a/maker.py b/maker.py index f164cd6e..0d835c9a 100644 --- a/maker.py +++ b/maker.py @@ -14,7 +14,7 @@ def __init__(self, maker, nick, oid, amount): if len(order_s) == 0: self.maker.send_error(nick, 'oid not found') order = order_s[0] - if amount <= order['minsize'] or amount >= order['maxsize']: + if amount < order['minsize'] or amount > order['maxsize']: self.maker.send_error(nick, 'amount out of range') #TODO return addresses, not mixing depths, so you can coinjoin to outside your own wallet self.utxos, self.cj_addr, self.change_addr = maker.oid_to_order(oid, amount) @@ -78,29 +78,14 @@ def unconfirm_callback(self, balance): removed_utxos = self.maker.wallet.remove_old_utxos(self.tx) debug('saw tx on network, removed_utxos=\n' + pprint.pformat(removed_utxos)) to_cancel, to_announce = self.maker.on_tx_unconfirmed(self, balance, removed_utxos) - self.handle_modified_orders(to_cancel, to_announce) + self.maker.modify_orders(to_cancel, to_announce) def confirm_callback(self, confirmations, txid, balance): added_utxos = self.maker.wallet.add_new_utxos(self.tx, txid) debug('tx in a block, added_utxos=\n' + pprint.pformat(added_utxos)) to_cancel, to_announce = self.maker.on_tx_confirmed(self, confirmations, txid, balance, added_utxos) - self.handle_modified_orders(to_cancel, to_announce) - - def handle_modified_orders(self, to_cancel, to_announce): - for oid in to_cancel: - order = [o for o in self.maker.orderlist if o['oid'] == oid][0] - self.maker.orderlist.remove(order) - if len(to_cancel) > 0: - clines = [command_prefix + 'cancel ' + str(oid) for oid in to_cancel] - self.maker.pubmsg(''.join(clines)) - if len(to_announce) > 0: - self.maker.privmsg_all_orders(CHANNEL, to_announce) - for ann in to_announce: - oldorder_s = [order for order in self.maker.orderlist if order['oid'] == ann['oid']] - if len(oldorder_s) > 0: - self.maker.orderlist.remove(oldorder_s[0]) - self.maker.orderlist += to_announce + self.maker.modify_orders(to_cancel, to_announce) def verify_unsigned_tx(self, txd): tx_utxos = set([ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) for ins in txd['ins']]) @@ -222,6 +207,21 @@ def on_set_topic(self, newtopic): def on_leave(self, nick): self.active_orders[nick] = None + def modify_orders(self, to_cancel, to_announce): + for oid in to_cancel: + order = [o for o in self.orderlist if o['oid'] == oid][0] + self.orderlist.remove(order) + if len(to_cancel) > 0: + clines = [command_prefix + 'cancel ' + str(oid) for oid in to_cancel] + self.pubmsg(''.join(clines)) + if len(to_announce) > 0: + self.privmsg_all_orders(CHANNEL, to_announce) + for ann in to_announce: + oldorder_s = [order for order in self.orderlist if order['oid'] == ann['oid']] + if len(oldorder_s) > 0: + self.orderlist.remove(oldorder_s[0]) + self.orderlist += to_announce + #these functions # create_my_orders() # oid_to_uxto() diff --git a/send-payment.py b/sendpayment.py similarity index 89% rename from send-payment.py rename to sendpayment.py index 2ccf7ec1..09408478 100644 --- a/send-payment.py +++ b/sendpayment.py @@ -81,8 +81,7 @@ def run(self): orders = choose_order(self.taker.db, self.taker.amount, self.taker.makercount) print 'chosen orders to fill ' + str(orders) - utxo_list = self.taker.wallet.get_mix_utxo_list()[0] #only spend from the unmixed funds - + utxo_list = self.taker.wallet.get_mix_utxo_list()[self.taker.mixdepth] unspent = [{'utxo': utxo, 'value': self.taker.wallet.unspent[utxo]['value']} for utxo in utxo_list] inputs = btc.select(unspent, self.taker.amount) @@ -91,7 +90,7 @@ def run(self): self.taker.cjtx = takermodule.CoinJoinTX(self.taker, self.taker.amount, orders, utxos, self.taker.destaddr, - self.taker.wallet.get_change_addr(0), self.taker.txfee, + self.taker.wallet.get_change_addr(self.taker.mixdepth), self.taker.txfee, self.finishcallback) ''' @@ -102,7 +101,7 @@ def run(self): ''' class SendPayment(takermodule.Taker): - def __init__(self, wallet, destaddr, amount, makercount, txfee, waittime): + def __init__(self, wallet, destaddr, amount, makercount, txfee, waittime, mixdepth): takermodule.Taker.__init__(self) self.wallet = wallet self.destaddr = destaddr @@ -110,6 +109,7 @@ def __init__(self, wallet, destaddr, amount, makercount, txfee, waittime): self.makercount = makercount self.txfee = txfee self.waittime = waittime + self.mixdepth = mixdepth def on_welcome(self): takermodule.Taker.on_welcome(self) @@ -117,14 +117,16 @@ def on_welcome(self): def main(): parser = OptionParser(usage='usage: %prog [options] [seed] [amount] [destaddr]', - description='Sends a single payment from your wallet to an given address' + - ' using coinjoin and then switches off.') + description='Sends a single payment from the zero mixing depth of your ' + + ' wallet to an given address using coinjoin and then switches off.') parser.add_option('-f', '--txfee', action='store', type='int', dest='txfee', - default=10000, help='miner fee contribution') + default=10000, help='miner fee contribution, in satoshis') parser.add_option('-w', '--wait-time', action='store', type='float', dest='waittime', help='wait time in seconds to allow orders to arrive', default=5) parser.add_option('-N', '--makercount', action='store', type='int', dest='makercount', help='how many makers to coinjoin with', default=2) + parser.add_option('-m', '--mixdepth', action='store', type='int', dest='mixdepth', + help='mixing depth to spend from', default=0) (options, args) = parser.parse_args() if len(args) < 3: @@ -143,7 +145,8 @@ def main(): wallet.find_unspent_addresses() print 'starting irc' - taker = SendPayment(wallet, destaddr, amount, options.makercount, options.txfee, options.waittime) + taker = SendPayment(wallet, destaddr, amount, options.makercount, options.txfee, + options.waittime, options.mixdepth) taker.run(HOST, PORT, nickname, CHANNEL) if __name__ == "__main__": From b185964026e19679c809c2fdc36e89a2dc7bda7e Mon Sep 17 00:00:00 2001 From: Belcher Date: Wed, 31 Dec 2014 18:18:40 +0000 Subject: [PATCH 037/409] changed print, made taker actually pushtx() --- irclib.py | 2 +- taker.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/irclib.py b/irclib.py index 8ab2999a..1b527bfa 100644 --- a/irclib.py +++ b/irclib.py @@ -142,6 +142,7 @@ def run(self, server, port, nick, channel, username='username', realname='realna PingThread(self).start() while self.connect_attempts < 10 and not self.give_up: + print 'connecting' self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.sock.connect((server, port)) self.fd = self.sock.makefile() @@ -167,7 +168,6 @@ def run(self, server, port, nick, channel, username='username', realname='realna print 'disconnected irc' time.sleep(10) self.connect_attempts += 1 - print 'reconnecting' print 'ending irc' self.give_up = True diff --git a/taker.py b/taker.py index 45ba2c28..eb956bd7 100644 --- a/taker.py +++ b/taker.py @@ -118,8 +118,8 @@ def add_signature(self, sigb64): return debug('the entire tx is signed, ready to pushtx()') print btc.serialize(self.latest_tx) - #ret = btc.blockr_pushtx(btc.serialize(self.latest_tx), get_network()) - #print 'pushed tx ' + str(ret) + ret = btc.blockr_pushtx(btc.serialize(self.latest_tx), get_network()) + debug('pushed tx ' + str(ret)) if self.finishcallback != None: self.finishcallback() From be90d33b8e516e8a64cbd403466632483681dbc3 Mon Sep 17 00:00:00 2001 From: Belcher Date: Thu, 1 Jan 2015 23:30:27 +0000 Subject: [PATCH 038/409] added patient send payment --- patientsendpayment.py | 170 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 patientsendpayment.py diff --git a/patientsendpayment.py b/patientsendpayment.py new file mode 100644 index 00000000..767b0447 --- /dev/null +++ b/patientsendpayment.py @@ -0,0 +1,170 @@ + +from common import * +import taker +import maker +import bitcoin as btc +import sendpayment + +from optparse import OptionParser +from datetime import timedelta +import threading, time + +class TakerThread(threading.Thread): + def __init__(self, tmaker): + threading.Thread.__init__(self) + self.daemon = True + self.tmaker = tmaker + self.finished = False + + def finishcallback(self): + self.tmaker.shutdown() + + def run(self): + time.sleep(self.tmaker.waittime) + if self.finished: + return + print 'giving up waiting' + #cancel the remaining order + self.tmaker.modify_orders([0], []) + orders = sendpayment.choose_order(self.tmaker.db, self.tmaker.amount, self.tmaker.makercount) + print 'chosen orders to fill ' + str(orders) + + utxo_list = self.tmaker.wallet.get_mix_utxo_list()[self.tmaker.mixdepth] + unspent = [{'utxo': utxo, 'value': self.tmaker.wallet.unspent[utxo]['value']} + for utxo in utxo_list] + inputs = btc.select(unspent, self.tmaker.amount) + utxos = [i['utxo'] for i in inputs] + print 'will spend ' + str(inputs) + + self.tmaker.cjtx = taker.CoinJoinTX(self.tmaker, self.tmaker.amount, + orders, utxos, self.tmaker.destaddr, + self.tmaker.wallet.get_change_addr(self.tmaker.mixdepth), + self.tmaker.txfee, self.finishcallback) + +class PatientSendPayment(maker.Maker, taker.Taker): + def __init__(self, wallet, destaddr, amount, makercount, txfee, cjfee, + waittime, mixdepth): + self.destaddr = destaddr + self.amount = amount + self.makercount = makercount + self.txfee = txfee + self.cjfee = cjfee + self.waittime = waittime + self.mixdepth = mixdepth + maker.Maker.__init__(self, wallet) + taker.Taker.__init__(self) + + def on_privmsg(self, nick, message): + maker.Maker.on_privmsg(self, nick, message) + taker.Taker.on_privmsg(self, nick, message) + + def on_welcome(self): + maker.Maker.on_welcome(self) + taker.Taker.on_welcome(self) + self.takerthread = TakerThread(self) + self.takerthread.start() + + def on_pubmsg(self, nick, message): + maker.Maker.on_pubmsg(self, nick, message) + taker.Taker.on_pubmsg(self, nick, message) + + def create_my_orders(self): + #choose an absolute fee order to discourage people from + # mixing smaller amounts + order = {'oid': 0, 'ordertype': 'absorder', 'minsize': 0, + 'maxsize': self.amount, 'txfee': self.txfee, 'cjfee': self.cjfee} + return [order] + + def oid_to_order(self, oid, amount): + #TODO race condition (kinda) + #if an order arrives and before it finishes another order arrives + # its possible this bot will end up paying to the destaddr more than it + # intended + utxo_list = self.wallet.get_mix_utxo_list()[self.mixdepth] + unspent = [{'utxo': utxo, 'value': self.wallet.unspent[utxo]['value']} + for utxo in utxo_list] + inputs = btc.select(unspent, amount) + utxos = [i['utxo'] for i in inputs] + return utxos, self.destaddr, self.wallet.get_change_addr(self.mixdepth) + + def on_tx_unconfirmed(self, cjorder, balance, removed_utxos): + self.amount -= cjorder.cj_amount + if self.amount == 0: + self.takerthread.finished = True + print 'finished sending, exiting..' + self.shutdown() + utxo_list = self.wallet.get_mix_utxo_list()[self.mixdepth] + available_balance = 0 + for utxo in utxo_list: + available_balance = self.wallet.unspent[utxo]['value'] + if available_balance > self.amount: + order = {'oid': 0, 'ordertype': 'absorder', 'minsize': 0, + 'maxsize': self.amount, 'txfee': self.txfee, 'cjfee': self.cjfee} + return ([], [order]) + else: + debug('not enough money left, have to wait until tx confirms') + return ([0], []) + + def on_tx_confirmed(self, cjorder, confirmations, txid, balance, added_utxos): + if len(self.orderlist) == 0: + order = {'oid': 0, 'ordertype': 'absorder', 'minsize': 0, + 'maxsize': self.amount, 'txfee': self.txfee, 'cjfee': self.cjfee} + return ([], [order]) + else: + return ([], []) + + +def main(): + parser = OptionParser(usage='usage: %prog [options] [seed] [amount] [destaddr]', + description='Sends a payment from your wallet to an given address' + + ' using coinjoin. First acts as a maker, announcing an order and ' + + 'waiting for someone to fill it. After a set period of time, gives' + + ' up waiting and acts as a taker and coinjoins any remaining coins') + parser.add_option('-f', '--txfee', action='store', type='int', dest='txfee', + default=10000, help='miner fee contribution, in satoshis') + parser.add_option('-N', '--makercount', action='store', type='int', dest='makercount', + help='how many makers to coinjoin with when taking liquidity', default=2) + parser.add_option('-w', '--wait-time', action='store', type='float', dest='waittime', + help='wait time in hours as a maker before becoming a taker', default=8) + parser.add_option('-c', '--cjfee', action='store', type='int', dest='cjfee', + help='coinjoin fee for the maker, in satoshis per order filled', default=50000) + parser.add_option('-m', '--mixdepth', action='store', type='int', dest='mixdepth', + help='mixing depth to spend from', default=0) + (options, args) = parser.parse_args() + + if len(args) < 3: + parser.error('Needs a seed, amount and destination address') + sys.exit(0) + seed = args[0] + amount = int(args[1]) + destaddr = args[2] + + waittime = timedelta(hours=options.waittime).total_seconds() + print 'Running patient sender of a payment' + print 'txfee=%d cjfee=%d waittime=%s makercount=%d' % (options.txfee, options.cjfee, + str(timedelta(hours=options.waittime)), options.makercount) + + print 'downloading wallet history' + wallet = Wallet(seed) + wallet.download_wallet_history() + wallet.find_unspent_addresses() + + utxo_list = wallet.get_mix_utxo_list()[options.mixdepth] + available_balance = 0 + for utxo in utxo_list: + available_balance = wallet.unspent[utxo]['value'] + if available_balance < amount: + print 'not enough money at mixdepth=%d, exiting' % (options.mixdepth) + return + + from socket import gethostname + nickname = 'ppayer-' + btc.sha256(gethostname())[:6] + + print 'starting irc' + bot = PatientSendPayment(wallet, destaddr, amount, options.makercount, + options.txfee, options.cjfee, waittime, options.mixdepth) + bot.run(HOST, PORT, nickname, CHANNEL) + +if __name__ == "__main__": + main() + print('done') From 12b9e491760be1b40d9f6840b3c90aca7c499d0a Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 2 Jan 2015 21:54:02 +0000 Subject: [PATCH 039/409] added timeouts in the tx confirm checking loop --- README.txt | 7 ++++--- common.py | 27 +++++++++++++++++++++++---- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/README.txt b/README.txt index d51b510f..4ba2e7e5 100644 --- a/README.txt +++ b/README.txt @@ -124,9 +124,6 @@ e.g. patient-single-tx.py which does the above but doesnt mind waiting up to a l e.g. gui-taker.py has a gui which shows the user the orderbook and they can easily fill and order and see other statistics, could be easily done by opening a http port and sending a html form and graphics -TODO -extend send-payment.py to be able to do more than 2-party coinjoin - TODO implement this the thing that gmaxwell wrote about in the original coinjoin post, as a kind of tumbler "Isn't the anonymity set size limited by how many parties you can get in a single transaction?" @@ -139,6 +136,10 @@ Not sure if it will actually be possible in this liquidity maker/taker system TODO need to move onto the bip44 structure of HD wallets +TODO think about this +<> some coinjoin tools we use today were broken +<> one allowed people to use a mix of uncompressed and compressed keys, so it was obvious which party was which. + TODO probably a good idea to have a debug.log where loads of information is dumped diff --git a/common.py b/common.py index 05347303..1e7fc9a5 100644 --- a/common.py +++ b/common.py @@ -185,21 +185,33 @@ def find_unspent_addresses(self): dat['address'], 'value': int(u['amount'].replace('.', ''))} #awful way of doing this, but works for now -# later use websocket api for people who dont download the blockchain # and -walletnotify for people who do -def add_addr_notify(address, unconfirmfun, confirmfun): +#timeouts in minutes +def add_addr_notify(address, unconfirmfun, confirmfun, unconfirmtimeout=5, + unconfirmtimeoutfun=None, confirmtimeout=120, confirmtimeoutfun=None): class NotifyThread(threading.Thread): - def __init__(self, address, unconfirmfun, confirmfun): + def __init__(self, address, unconfirmfun, confirmfun, unconfirmtimeout, + unconfirmtimeoutfun, confirmtimeout, confirmtimeoutfun): threading.Thread.__init__(self) self.daemon = True self.address = address self.unconfirmfun = unconfirmfun self.confirmfun = confirmfun + self.unconfirmtimeout = unconfirmtimeout*60 + self.unconfirmtimeoutfun = unconfirmtimeoutfun + self.confirmtimeout = confirmtimeout*60 + self.confirmtimeoutfun = confirmtimeoutfun def run(self): + st = int(time.time()) while True: time.sleep(5) + if int(time.time()) - st > self.unconfirmtimeout: + if unconfirmtimeoutfun != None: + unconfirmtimeoutfun() + debug('checking for unconfirmed tx timed out') + return if get_network() == 'testnet': blockr_url = 'http://tbtc.blockr.io/api/v1/address/balance/' else: @@ -209,8 +221,14 @@ def run(self): if data['balance'] > 0: break self.unconfirmfun(data['balance']*1e8) + st = int(time.time()) while True: time.sleep(5 * 60) + if int(time.time()) - st > self.confirmtimeout: + if confirmtimeoutfun != None: + confirmtimeoutfun() + debug('checking for confirmed tx timed out') + return if get_network() == 'testnet': blockr_url = 'http://tbtc.blockr.io/api/v1/address/txs/' else: @@ -224,7 +242,8 @@ def run(self): self.confirmfun(data['txs'][0]['confirmations'], data['txs'][0]['tx'], data['txs'][0]['amount']*1e8) - NotifyThread(address, unconfirmfun, confirmfun).start() + NotifyThread(address, unconfirmfun, confirmfun, unconfirmtimeout, + unconfirmtimeoutfun, confirmtimeout, confirmtimeoutfun).start() def calc_cj_fee(ordertype, cjfee, cj_amount): real_cjfee = None From db1b50664ac12097f54897270128d5bf494a9d43 Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 2 Jan 2015 22:32:12 +0000 Subject: [PATCH 040/409] added notes on the default values --- bip32-tool.py | 6 +++--- patientsendpayment.py | 10 +++++----- sendpayment.py | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/bip32-tool.py b/bip32-tool.py index 457e4011..1aa8717a 100644 --- a/bip32-tool.py +++ b/bip32-tool.py @@ -21,11 +21,11 @@ + '. Combine- combines all utxos into one output for each mixing level. Used for' + ' testing and is detrimental to privacy.') parser.add_option('-p', '--privkey', action='store_true', dest='showprivkey', - help='print private key along with address') + help='print private key along with address, default false') parser.add_option('-m', '--maxmixdepth', action='store', type='int', dest='maxmixdepth', - default=2, help='maximum mixing depth to look for') + default=2, help='maximum mixing depth to look for, default=2') parser.add_option('-g', '--gap-limit', action='store', dest='gaplimit', - help='gap limit for wallet', default=6) + help='gap limit for wallet, default=6', default=6) (options, args) = parser.parse_args() if len(args) < 1: diff --git a/patientsendpayment.py b/patientsendpayment.py index 767b0447..3bce0742 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -121,15 +121,15 @@ def main(): 'waiting for someone to fill it. After a set period of time, gives' + ' up waiting and acts as a taker and coinjoins any remaining coins') parser.add_option('-f', '--txfee', action='store', type='int', dest='txfee', - default=10000, help='miner fee contribution, in satoshis') + default=10000, help='miner fee contribution, in satoshis, default=10000') parser.add_option('-N', '--makercount', action='store', type='int', dest='makercount', - help='how many makers to coinjoin with when taking liquidity', default=2) + help='how many makers to coinjoin with when taking liquidity, default=2', default=2) parser.add_option('-w', '--wait-time', action='store', type='float', dest='waittime', - help='wait time in hours as a maker before becoming a taker', default=8) + help='wait time in hours as a maker before becoming a taker, default=8', default=8) parser.add_option('-c', '--cjfee', action='store', type='int', dest='cjfee', - help='coinjoin fee for the maker, in satoshis per order filled', default=50000) + help='coinjoin fee asked for when being a maker, in satoshis per order filled, default=50000', default=50000) parser.add_option('-m', '--mixdepth', action='store', type='int', dest='mixdepth', - help='mixing depth to spend from', default=0) + help='mixing depth to spend from, default=0', default=0) (options, args) = parser.parse_args() if len(args) < 3: diff --git a/sendpayment.py b/sendpayment.py index 09408478..bdb1ead1 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -120,13 +120,13 @@ def main(): description='Sends a single payment from the zero mixing depth of your ' + ' wallet to an given address using coinjoin and then switches off.') parser.add_option('-f', '--txfee', action='store', type='int', dest='txfee', - default=10000, help='miner fee contribution, in satoshis') + default=10000, help='miner fee contribution, in satoshis, default=10000') parser.add_option('-w', '--wait-time', action='store', type='float', dest='waittime', - help='wait time in seconds to allow orders to arrive', default=5) + help='wait time in seconds to allow orders to arrive, default=5', default=5) parser.add_option('-N', '--makercount', action='store', type='int', dest='makercount', - help='how many makers to coinjoin with', default=2) + help='how many makers to coinjoin with, default=2', default=2) parser.add_option('-m', '--mixdepth', action='store', type='int', dest='mixdepth', - help='mixing depth to spend from', default=0) + help='mixing depth to spend from, default=0', default=0) (options, args) = parser.parse_args() if len(args) < 3: From 8e2db0958bfd3418f45cd7c4cee248f574a55631 Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 2 Jan 2015 22:45:00 +0000 Subject: [PATCH 041/409] bip32tool now called wallettool --- bip32-tool.py => wallet-tool.py | 3 +++ 1 file changed, 3 insertions(+) rename bip32-tool.py => wallet-tool.py (93%) diff --git a/bip32-tool.py b/wallet-tool.py similarity index 93% rename from bip32-tool.py rename to wallet-tool.py index 1aa8717a..3f0534a2 100644 --- a/bip32-tool.py +++ b/wallet-tool.py @@ -47,6 +47,7 @@ if method == 'display': for m in range(wallet.max_mix_depth): print 'mixing depth %d m/0/%d/' % (m, m) + balance_depth = 0 for forchange in [0, 1]: print(' ' + ('receive' if forchange==0 else 'change') + ' addresses m/0/%d/%d/' % (m, forchange)) @@ -56,8 +57,10 @@ for addrvalue in wallet.unspent.values(): if addr == addrvalue['address']: balance += addrvalue['value'] + balance_depth += balance used = ('used' if k < wallet.index[m][forchange] else ' new') print ' m/0/%d/%d/%02d %s %s %.8fbtc' % (m, forchange, k, addr, used, balance/1e8) + print 'for mixdepth=%d balance=%.8fbtc' % (m, balance_depth/1e8) elif method == 'combine': ins = [] outs = [] From 99d7d8972e52b989d6e9090c5723184a9593101b Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 2 Jan 2015 23:10:27 +0000 Subject: [PATCH 042/409] some more TODOs --- README.txt | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/README.txt b/README.txt index 4ba2e7e5..09e02f31 100644 --- a/README.txt +++ b/README.txt @@ -157,15 +157,20 @@ both are important for market forces, since markets emerge from human decisions #two options, random delay !orderbook for ones which dont mind, !orderbook without delay for bots # which need the orders asap +TODO +code something that extends orderbookwatch and creates graphs + those graphs can be posted to a bitcointalk thread (like the bitstamp wall watch thread) + and could be a nice historical record and guide to pricing + +TODO +code something that analyzes the blockchain, detects coinjoin tx likely made by joinmarket + and calculates the paid fee, therefore is a guide to pricing + TODO the add_addr_notify() stuff doesnt work, so if theres several CoinJoinOrder's open it will start a few threads to do the notifying, they could race condition or other multithreaded errors i suggest to create a single thread that sorts out all the stuff -#TODO error checking so you cant crash the bot by sending malformed orders -when an error happens, send back a !error command so the counterparty knows - something went wrong, and then cancel that partly filled order - #TODO make an ordertype where maker publishes the utxo he will use # this is a way to auction off the use of a desirable coin, maybe a # very newly mined coin or one which hasnt been moved for years From c203bf3f1b94f59449febf9dd6cdf348356e8798 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Sun, 4 Jan 2015 23:07:54 +0200 Subject: [PATCH 043/409] initial working version of encrypted privmsgs --- README.txt | 8 +++-- common.py | 5 ++- irclib.py | 92 +++++++++++++++++++++++++++++++++++++++++++++++++++--- maker.py | 51 ++++++++++++++++++++++++------ taker.py | 67 +++++++++++++++++++++++++++++++++------ 5 files changed, 196 insertions(+), 27 deletions(-) diff --git a/README.txt b/README.txt index d51b510f..21674f4e 100644 --- a/README.txt +++ b/README.txt @@ -45,8 +45,12 @@ taker collects all the utxos and outputs and makes a transaction taker then signs his own and pushtx() IRC commands used when starting a coinjoin, everything in pm - !fill [order id] [coinjoin amount] - !io [comma seperated list of utxos] [coinjoin address] [change address] + !fill [order id] [coinjoin amount] [input_pubkey] + !io [comma seperated list of utxos] [coinjoin address] [change address] [coinjoin pubkey] [bitcoin signature] [encryption pubkey] + + !auth [encryption pubkey] [btc_sig] +After this, messages sent between taker and maker will be encrypted. + when taker collects inputs and outputs of all the makers it's contacted, it creates a tx out of them !txpart [base64 encoded tx part] ... diff --git a/common.py b/common.py index 05347303..1ad87702 100644 --- a/common.py +++ b/common.py @@ -5,7 +5,7 @@ import threading HOST = 'irc.freenode.net' -CHANNEL = '#joinmarket-pit-test' +CHANNEL = '#joinmarket-pit-test2' PORT = 6667 #for the mainnet its #joinmarket-pit @@ -19,6 +19,9 @@ def debug(msg): print datetime.datetime.now().strftime("[%Y/%m/%d %H:%M:%S] ") + msg +def chunks(d, n): + return [d[x: x+n] for x in xrange(0, len(d), n)] + def get_network(): return 'testnet' diff --git a/irclib.py b/irclib.py index 8ab2999a..664c1c0f 100644 --- a/irclib.py +++ b/irclib.py @@ -1,6 +1,8 @@ import socket, threading, time -from common import debug +from common import debug, chunks +import enc_wrapper +import base64, os PING_INTERVAL = 40 PING_TIMEOUT = 10 @@ -45,7 +47,54 @@ def on_leave(self, nick): pass def on_disconnect(self): pass def on_connect(self): pass #TODO implement on_nick_change - + + ############################### + #Encryption code + ############################### + def init_encryption(self,fname): + if not os.path.isfile(fname): + self.enc_kp = enc_wrapper.init_keypair(fname) + else: + self.enc_kp = enc_wrapper.libnacl.utils.load_key(fname) + + def start_encryption(self,nick,c_pk_hex): + '''sets encryption mode on + for all succeeding messages + until end_encryption is called. + Public key of counterparty must be + passed in in hex.''' + if not self.enc_kp: + raise Exception("Cannot initialise encryption without a keypair") + self.cp_pubkeys[nick] = enc_wrapper.init_pubkey(c_pk_hex) + self.enc_boxes[nick] = enc_wrapper.as_init_encryption(self.enc_kp, self.cp_pubkeys[nick]) + self.encrypting[nick]= True + + def end_encryption(self, nick): + self.encrypting[nick]=False + #for safety, blank out all data related + #to the connection + self.cp_pubkeys[nick]=None + self.enc_boxes[nick]=None + + def encrypt_encode(self,msg,nick): + if not (nick in self.encrypting.keys()) or not (nick in self.enc_boxes.keys()): + raise Exception("Encryption is not switched on.") + if not (self.encrypting[nick] and self.enc_boxes[nick]): + raise Exception("Encryption is not switched on.") + encrypted = self.enc_boxes[nick].encrypt(msg) + return base64.b64encode(encrypted) + + def decode_decrypt(self,msg,nick): + if not (nick in self.encrypting.keys()) or not (nick in self.enc_boxes.keys()): + raise Exception("Encryption is not switched on.") + if not (self.encrypting[nick] and self.enc_boxes[nick]): + raise Exception("Encryption is not switched on.") + decoded = base64.b64decode(msg) + return self.enc_boxes[nick].decrypt(decoded) + ############################# + #End encryption code + ############################# + def close(self): self.send_raw("QUIT") @@ -59,7 +108,17 @@ def pubmsg(self, message): def privmsg(self, nick, message): #debug('privmsg to ' + nick + ' ' + message) - self.send_raw("PRIVMSG " + nick + " :" + message) + if nick in self.encrypting.keys() and self.encrypting[nick]: + message = self.encrypt_encode(message,nick) + if len(message) > 350: + message_chunks = chunks(message,350) + else: + message_chunks = [message] + print "We are going to send these chunks: " + print message_chunks + for m in message_chunks: + trailer = ' :' if m==message_chunks[-1] else ' ;' + self.send_raw("PRIVMSG " + nick + " :" + m + trailer) def send_raw(self, line): if not line.startswith('PING LAG'): @@ -75,8 +134,26 @@ def __handle_privmsg(self, source, target, message): ctcp = message[1:endindex + 1] #self.send_raw('PRIVMSG ' + nick + ' :\x01VERSION #TODO ctcp version here, since some servers dont let you get on without + if target == self.nick: - self.on_privmsg(nick, message) + if nick not in self.built_privmsg: + self.built_privmsg[nick] = message[:-2] + else: + self.built_privmsg[nick] += message[:-2] + if message[-1]==';': + self.waiting[nick]=True + elif message[-1]==':': + self.waiting[nick]=False + if nick in self.encrypting.keys() and self.encrypting[nick]: + print 'about to decrypt a message: ' + print self.built_privmsg[nick] + parsed = self.decode_decrypt(self.built_privmsg[nick],nick) + else: parsed = self.built_privmsg[nick] + #wipe the message buffer waiting for the next one + self.built_privmsg[nick]='' + self.on_privmsg(nick, parsed) + else: + raise Exception("message formatting error") else: self.on_pubmsg(nick, message) @@ -135,7 +212,12 @@ def run(self, server, port, nick, channel, username='username', realname='realna self.nick = nick self.channel = channel self.connect_attempts = 0 - + #set up encryption management variables + self.cp_pubkeys = {} + self.enc_boxes = {} + self.encrypting = {} + self.waiting = {} + self.built_privmsg = {} self.give_up = False self.ping_reply = True self.lockcond = threading.Condition() diff --git a/maker.py b/maker.py index f164cd6e..01adcecc 100644 --- a/maker.py +++ b/maker.py @@ -6,10 +6,12 @@ import base64, pprint class CoinJoinOrder(object): - def __init__(self, maker, nick, oid, amount): + def __init__(self, maker, nick, oid, amount, i_utxo_pubkey): self.maker = maker self.oid = oid self.cj_amount = amount + #the btc pubkey of the utxo that the taker plans to use as input + self.i_utxo_pubkey = i_utxo_pubkey order_s = [o for o in maker.orderlist if o['oid'] == oid] if len(order_s) == 0: self.maker.send_error(nick, 'oid not found') @@ -26,9 +28,15 @@ def __init__(self, maker, nick, oid, amount): #always a new address even if the order ends up never being # furfilled, you dont want someone pretending to fill all your # orders to find out which addresses you use + + #initiate encryption handshake by sending signed pubkey + btc_key = maker.wallet.get_key_from_addr(self.cj_addr) + btc_pub = btc.privtopub(btc_key) + btc_sig = btc.ecdsa_sign(maker.enc_kp.hex_pk(),btc_key) maker.privmsg(nick, command_prefix + 'io ' + ','.join(self.utxos) + ' ' + - self.cj_addr + ' ' + self.change_addr) - + self.cj_addr + ' ' + self.change_addr + ' ' + + btc_pub + ' ' + btc_sig + ' ' + maker.enc_kp.hex_pk()) + def recv_tx_part(self, b64txpart): self.b64txparts.append(b64txpart) size = sum([len(s) for s in self.b64txparts]) @@ -137,7 +145,8 @@ class CJMakerOrderError(StandardError): pass class Maker(irclib.IRCClient): - def __init__(self, wallet): + def __init__(self, wallet, keyfile): + self.init_encryption(keyfile) self.active_orders = {} self.wallet = wallet self.nextoid = -1 @@ -164,7 +173,19 @@ def send_error(self, nick, errmsg): def on_welcome(self): self.privmsg_all_orders(CHANNEL) - + + def auth_counterparty(self,nick,cjorder,enc_pk,btc_sig): + #TODO: add check that the pubkey's address is part of the order. + btc_pub = cjorder.i_utxo_pubkey + if not btc.ecdsa_verify(enc_pk,btc_sig,btc_pub): + print 'signature didnt match pubkey and message' + return False + #authorisation passed: initiate encryption for future + #messages + self.start_encryption(nick,enc_pk) + #send pubkey info to counterparty + return True + def on_privmsg(self, nick, message): debug("privmsg nick=%s message=%s" % (nick, message)) if message[0] != command_prefix: @@ -177,15 +198,27 @@ def on_privmsg(self, nick, message): try: if len(chunks) < 2: self.send_error(nick, 'Not enough arguments') + if chunks[0] == 'auth': + if nick not in self.active_orders: + self.send_error(nick,"Encryption handshake message received out of order, ignored.") + continue + try: + cjorder = self.active_orders[nick] + enc_pk = chunks[1] + btc_sig = chunks[2] + self.auth_counterparty(nick,cjorder,enc_pk,btc_sig) + except (ValueError,IndexError) as e: + self.send_error(nick, str(e)) if chunks[0] == 'fill': if nick in self.active_orders and self.active_orders[nick] != None: self.send_error(nick, 'Already have partially-filled order') try: oid = int(chunks[1]) amount = int(chunks[2]) + i_utxo_pubkey = chunks[3] except (ValueError, IndexError) as e: self.send_error(nick, str(e)) - self.active_orders[nick] = CoinJoinOrder(self, nick, oid, amount) + self.active_orders[nick] = CoinJoinOrder(self, nick, oid, amount, i_utxo_pubkey) elif chunks[0] == 'txpart' or chunks[0] == 'tx': if nick not in self.active_orders or self.active_orders[nick] == None: self.send_error(nick, 'No open order from this nick') @@ -308,13 +341,13 @@ def main(): nickname = 'cj-maker-' + btc.sha256(gethostname())[:6] import sys seed = sys.argv[1] #btc.sha256('dont use brainwallets except for holding testnet coins') - + keyfile = sys.argv[2] print 'downloading wallet history' - wallet = Wallet(seed) + wallet = Wallet(seed,max_mix_depth=5) wallet.download_wallet_history() wallet.find_unspent_addresses() - maker = Maker(wallet) + maker = Maker(wallet,keyfile) print 'connecting to irc' maker.run(HOST, PORT, nickname, CHANNEL) diff --git a/taker.py b/taker.py index 45ba2c28..240c86c9 100644 --- a/taker.py +++ b/taker.py @@ -26,9 +26,13 @@ def __init__(self, taker, cj_amount, orders, my_utxos, my_cj_addr, self.my_change_addr = my_change_addr self.cjfee_total = 0 self.latest_tx = None + #find the btc pubkey of the first utxo being used + self.signing_btc_add = taker.wallet.unspent[self.my_utxos[0]]['address'] + self.signing_btc_pub = btc.privtopub(taker.wallet.get_key_from_addr(self.signing_btc_add)) for c, oid in orders.iteritems(): - taker.privmsg(c, command_prefix + 'fill ' + str(oid) + ' ' + str(cj_amount)) - + taker.privmsg(c, command_prefix + 'fill ' + \ + str(oid) + ' ' + str(cj_amount) + ' ' + self.signing_btc_pub) + def recv_txio(self, nick, utxo_list, cj_addr, change_addr): if nick not in self.nonrespondants: debug('nick(' + nick + ') not in nonrespondants ' + str(self.nonrespondants)) @@ -188,13 +192,32 @@ def on_disconnect(self): #assume this only has one open cj tx at a time class Taker(OrderbookWatch): - def __init__(self): + def __init__(self,keyfile): OrderbookWatch.__init__(self) self.cjtx = None #TODO have a list of maker's nick we're coinjoining with, so # that some other guy doesnt send you confusing stuff #maybe a start_cj_tx() method is needed - + self.init_encryption(keyfile) + + def auth_counterparty(self,nick,btc_pub,btc_sig,cj_addr,enc_pk, signing_add): + if not cj_addr==btc.pubtoaddr(btc_pub,magicbyte=get_addr_vbyte()): + print 'coinjoin address and pubkey didnt match' + return False + if not btc.ecdsa_verify(enc_pk,btc_sig,btc_pub): + print 'signature didnt match pubkey and message' + return False + #send authorisation response + my_btc_sig = btc.ecdsa_sign(self.enc_kp.hex_pk(),\ + self.wallet.get_key_from_addr(signing_add)) + message = '!auth ' + self.enc_kp.hex_pk() + ' ' + my_btc_sig + self.privmsg(nick,message) #note: we do this *before* starting encryption + #TODO: should both sides send acks to acknowledge successful handshake? + #authorisation passed: initiate encryption for future + #messages + self.start_encryption(nick,enc_pk) + return True + def on_privmsg(self, nick, message): OrderbookWatch.on_privmsg(self, nick, message) #debug("privmsg nick=%s message=%s" % (nick, message)) @@ -206,18 +229,26 @@ def on_privmsg(self, nick, message): utxo_list = chunks[1].split(',') cj_addr = chunks[2] change_addr = chunks[3] + btc_pub = chunks[4] + btc_sig = chunks[5] + enc_pk = chunks[6] + if not self.auth_counterparty(nick,btc_pub,btc_sig,cj_addr,enc_pk,\ + self.cjtx.signing_btc_add): + print 'Authenticated encryption with counterparty: ' + nick + \ + ' not established. TODO: send rejection message' + continue self.cjtx.recv_txio(nick, utxo_list, cj_addr, change_addr) elif chunks[0] == 'sig': sig = chunks[1] self.cjtx.add_signature(sig) my_tx_fee = 10000 - + class TestTaker(Taker): - def __init__(self, wallet): - Taker.__init__(self) + def __init__(self, wallet,keyfile): + Taker.__init__(self,keyfile) self.wallet = wallet - + def on_pubmsg(self, nick, message): Taker.on_pubmsg(self, nick, message) if message[0] != command_prefix: @@ -225,6 +256,21 @@ def on_pubmsg(self, nick, message): for command in message[1:].split(command_prefix): #commands starting with % are for testing and will be removed in the final version chunks = command.split(" ") + if chunks[0] == '%go': + #!%go [counterparty] [oid] [amount] + cp = chunks[1] + oid = chunks[2] + amt = chunks[3] + utxo_list = self.wallet.get_mix_utxo_list()[0] #only spend from the unmixed funds + unspent = [{'utxo': utxo, 'value': self.wallet.unspent[utxo]['value']} \ + for utxo in utxo_list] + inputs = btc.select(unspent, amt) + utxos = [i['utxo'] for i in inputs] + print 'making cjtx' + self.cjtx = CoinJoinTX(self, int(amt), {cp: oid}, + [utxos[0]], self.wallet.get_receive_addr(mixing_depth=1), + self.wallet.get_change_addr(mixing_depth=0), my_tx_fee) + if chunks[0] == '%showob': print('printing orderbook') for o in self.db.execute('SELECT * FROM orderbook;').fetchall(): @@ -260,16 +306,17 @@ def on_pubmsg(self, nick, message): def main(): import sys seed = sys.argv[1] #btc.sha256('your brainwallet goes here') + keyfile = sys.argv[2] from socket import gethostname nickname = 'taker-' + btc.sha256(gethostname())[:6] print 'downloading wallet history' - wallet = Wallet(seed) + wallet = Wallet(seed, max_mix_depth=5) wallet.download_wallet_history() wallet.find_unspent_addresses() print 'starting irc' - taker = TestTaker(wallet) + taker = TestTaker(wallet,keyfile) taker.run(HOST, PORT, nickname, CHANNEL) if __name__ == "__main__": From 40af5b1a1eaf8f790a06dfcfd0c92706c73bcb23 Mon Sep 17 00:00:00 2001 From: Belcher Date: Thu, 8 Jan 2015 22:04:22 +0000 Subject: [PATCH 044/409] sendpayments doesnt hang after error --- sendpayment.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sendpayment.py b/sendpayment.py index bdb1ead1..2302e6aa 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -76,6 +76,7 @@ def run(self): counterparty_count = crow['COUNT(DISTINCT counterparty)'] if counterparty_count < self.taker.makercount: print 'not enough counterparties to fill order, ending' + self.taker.shutdown() return orders = choose_order(self.taker.db, self.taker.amount, self.taker.makercount) From 87973671342c1554e4691cb9493ce634bb17349d Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 9 Jan 2015 22:17:43 +0000 Subject: [PATCH 045/409] fixed bug when you had no money or a single utxo --- yield-generator.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/yield-generator.py b/yield-generator.py index 5a152468..198e43a3 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -38,6 +38,10 @@ def create_my_orders(self): total_value += self.wallet.unspent[utxo]['value'] mix_balance[mixdepth] = total_value + if len([b for m, b in mixbalance.iteritems() if b > 0]) == 0: + debug('do not have any coins left') + return [] + #print mix_balance max_mix = max(mix_balance, key=mix_balance.get) order = {'oid': 0, 'ordertype': 'relorder', 'minsize': 0, @@ -68,12 +72,14 @@ def oid_to_order(self, oid, amount): def on_tx_unconfirmed(self, cjorder, balance, removed_utxos): #if the balance of the highest-balance mixing depth change then reannounce it oldorder = self.orderlist[0] - neworder = self.create_my_orders()[0] - if oldorder['maxsize'] == neworder['maxsize']: - return ([], []) + neworders = self.create_my_orders() + if len(neworders) == 0: + return ([oldorder], []) #cancel old order + elif oldorder['maxsize'] == neworder['maxsize']: + return ([], []) #change nothing else: - return ([], [neworder]) - return ([], [neworder]) + #announce new order, replacing the old order + return ([], [neworder[0]]) def on_tx_confirmed(self, cjorder, confirmations, txid, balance, added_utxos): return self.on_tx_unconfirmed(None, None, None) From d5fcb23ec9a0e33b6fb15b092b5a363d13827237 Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 9 Jan 2015 22:22:05 +0000 Subject: [PATCH 046/409] spelling mistake --- yield-generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yield-generator.py b/yield-generator.py index 198e43a3..c305cfd0 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -38,7 +38,7 @@ def create_my_orders(self): total_value += self.wallet.unspent[utxo]['value'] mix_balance[mixdepth] = total_value - if len([b for m, b in mixbalance.iteritems() if b > 0]) == 0: + if len([b for m, b in mix_balance.iteritems() if b > 0]) == 0: debug('do not have any coins left') return [] From eaed61fed5464390b53aa37178b58ca22af0f8a8 Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 9 Jan 2015 22:27:10 +0000 Subject: [PATCH 047/409] made the nick different for each computer --- yield-generator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/yield-generator.py b/yield-generator.py index c305cfd0..aa305e2d 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -6,10 +6,12 @@ import pprint +from socket import gethostname +nickname = 'yigen-' + btc.sha256(gethostname())[:6] + txfee = 1000 cjfee = '0.01' # 1% fee mix_levels = 5 -nickname = 'yield-generate' nickserv_password = '' #is a maker for the purposes of generating a yield from held From 8ad0eebf1ca7b152ba4fc40a2a4fb1768a63bc7b Mon Sep 17 00:00:00 2001 From: Belcher Date: Mon, 12 Jan 2015 21:30:52 +0000 Subject: [PATCH 048/409] added debug and tryexcept since noticed a hanging --- irclib.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/irclib.py b/irclib.py index 1b527bfa..906f535a 100644 --- a/irclib.py +++ b/irclib.py @@ -17,6 +17,7 @@ def __init__(self, irc): self.daemon = True self.irc = irc def run(self): + debug('starting ping thread') while not self.irc.give_up: time.sleep(PING_INTERVAL) try: @@ -27,12 +28,16 @@ def run(self): self.irc.lockcond.wait(PING_TIMEOUT) self.irc.lockcond.release() if not self.irc.ping_reply: - print 'irc ping timed out' - self.irc.close() - self.irc.fd.close() - self.irc.sock.close() - except IOError: - pass + debug('irc ping timed out') + try: self.irc.close() + except IOError: pass + try: self.irc.fd.close() + except IOError: pass + try: self.irc.sock.close() + except IOError: pass + except IOError as e: + debug('ping thread: ' + repr(e)) + debug('ended ping thread') #handle one channel at a time class IRCClient(object): @@ -47,7 +52,10 @@ def on_connect(self): pass #TODO implement on_nick_change def close(self): - self.send_raw("QUIT") + try: + self.send_raw("QUIT") + except IOError as e: + debug('errored while trying to quit: ' + repr(e)) def shutdown(self): self.close() @@ -155,8 +163,10 @@ def run(self, server, port, nick, channel, username='username', realname='realna except AttributeError as e: raise IOError(repr(e)) if line == None: + debug('line returned null') break if len(line) == 0: + debug('line was zero length') break self.__handle_line(line) except IOError as e: @@ -168,7 +178,7 @@ def run(self, server, port, nick, channel, username='username', realname='realna print 'disconnected irc' time.sleep(10) self.connect_attempts += 1 - print 'ending irc' + debug('ending irc') self.give_up = True def irc_privmsg_size_throttle(irc, target, lines, prefix=''): From e14e60d0b4313b87b4357589182687f7ae1ed0ef Mon Sep 17 00:00:00 2001 From: Belcher Date: Mon, 12 Jan 2015 21:31:36 +0000 Subject: [PATCH 049/409] added a minimum size so you always make money --- yield-generator.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/yield-generator.py b/yield-generator.py index aa305e2d..2d8fc849 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -13,6 +13,9 @@ cjfee = '0.01' # 1% fee mix_levels = 5 nickserv_password = '' +minsize = int(2 * txfee / float(cjfee)) #minimum size is such that you always net profit at least the miners fee + + #is a maker for the purposes of generating a yield from held # bitcoins without ruining privacy for the taker, the taker could easily check @@ -46,7 +49,7 @@ def create_my_orders(self): #print mix_balance max_mix = max(mix_balance, key=mix_balance.get) - order = {'oid': 0, 'ordertype': 'relorder', 'minsize': 0, + order = {'oid': 0, 'ordertype': 'relorder', 'minsize': minsize, 'maxsize': mix_balance[max_mix], 'txfee': txfee, 'cjfee': cjfee, 'mix_balance': mix_balance} return [order] From 790fbc18d61a57019ee71b81f5a336f2a5c7ffaa Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Mon, 12 Jan 2015 23:36:38 +0200 Subject: [PATCH 050/409] new handshake algo --- enc_wrapper.py | 3 ++- encryption_protocol.txt | 32 ++++++++++++++++++++++ irclib.py | 12 ++++----- maker.py | 59 +++++++++++++++++++++-------------------- taker.py | 49 +++++++++++++++++----------------- 5 files changed, 94 insertions(+), 61 deletions(-) create mode 100644 encryption_protocol.txt diff --git a/enc_wrapper.py b/enc_wrapper.py index 7e761720..2cfbb4a0 100644 --- a/enc_wrapper.py +++ b/enc_wrapper.py @@ -56,5 +56,6 @@ def as_init_encryption(kp, c_pk): 2. Nonce is handled at the implementation layer. ''' -#TODO: Sign, verify. +#TODO: Sign, verify. At the moment we are using +#bitcoin signatures so it isn't necessary. diff --git a/encryption_protocol.txt b/encryption_protocol.txt new file mode 100644 index 00000000..82ad8b27 --- /dev/null +++ b/encryption_protocol.txt @@ -0,0 +1,32 @@ +Encryption handshake in JoinMarket +================================== + +In the clear +============ + +TAK: !fill +MAK: !pubkey + +Both maker and taker construct a crypto Box object to allow authenticated encryption between the parties. + +Encrypted +========= + +TAK: !auth +(Maker verifies the btc sig; if not valid, connection is dropped - send REJECT message) +MAK: !auth +(Taker verifies the btc sig; if not valid, as for previous) + +Because the !auth messages are under encryption, there is no privacy leak of bitcoin pubkeys or output addresses. + +If both verifications pass, the remainder of the messages exchanged between the two parties will continue under encryption. + +Note +==== +A key part of the authorisation process is the matching between the bitcoin pubkeys used in the coinjoin +transaction and the encryption pubkeys used. This ensures that the messages we are sending are only +readable by the entity which is conducting the bitcoin transaction with us. + +To ensure this, the maker should not sign any transaction that doesn't use the previously identified +input utxo as its input, and the taker should not push/sign any transaction that doesn't use the +previously identified maker coinjoin pubkey/address as its output. \ No newline at end of file diff --git a/irclib.py b/irclib.py index ca4dc624..df796e3f 100644 --- a/irclib.py +++ b/irclib.py @@ -117,7 +117,7 @@ def privmsg(self, nick, message): print "We are going to send these chunks: " print message_chunks for m in message_chunks: - trailer = ' :' if m==message_chunks[-1] else ' ;' + trailer = ' ~' if m==message_chunks[-1] else ' ;' self.send_raw("PRIVMSG " + nick + " :" + m + trailer) def send_raw(self, line): @@ -136,18 +136,16 @@ def __handle_privmsg(self, source, target, message): #TODO ctcp version here, since some servers dont let you get on without if target == self.nick: - if nick not in self.built_privmsg: + if nick not in self.built_privmsg or self.built_privmsg[nick]=='': self.built_privmsg[nick] = message[:-2] else: - self.built_privmsg[nick] += message[:-2] + self.built_privmsg[nick] += message[:-2] if message[-1]==';': self.waiting[nick]=True - elif message[-1]==':': + elif message[-1]=='~': self.waiting[nick]=False if nick in self.encrypting.keys() and self.encrypting[nick]: - print 'about to decrypt a message: ' - print self.built_privmsg[nick] - parsed = self.decode_decrypt(self.built_privmsg[nick],nick) + parsed = self.decode_decrypt(self.built_privmsg[nick],nick) else: parsed = self.built_privmsg[nick] #wipe the message buffer waiting for the next one self.built_privmsg[nick]='' diff --git a/maker.py b/maker.py index 65a0fbcb..8280722c 100644 --- a/maker.py +++ b/maker.py @@ -6,12 +6,12 @@ import base64, pprint class CoinJoinOrder(object): - def __init__(self, maker, nick, oid, amount, i_utxo_pubkey): + def __init__(self, maker, nick, oid, amount, taker_pk): self.maker = maker self.oid = oid self.cj_amount = amount #the btc pubkey of the utxo that the taker plans to use as input - self.i_utxo_pubkey = i_utxo_pubkey + self.taker_pk = taker_pk order_s = [o for o in maker.orderlist if o['oid'] == oid] if len(order_s) == 0: self.maker.send_error(nick, 'oid not found') @@ -28,15 +28,28 @@ def __init__(self, maker, nick, oid, amount, i_utxo_pubkey): #always a new address even if the order ends up never being # furfilled, you dont want someone pretending to fill all your # orders to find out which addresses you use + maker.privmsg(nick,command_prefix+ 'pubkey ' + maker.enc_kp.hex_pk()) - #initiate encryption handshake by sending signed pubkey - btc_key = maker.wallet.get_key_from_addr(self.cj_addr) - btc_pub = btc.privtopub(btc_key) - btc_sig = btc.ecdsa_sign(maker.enc_kp.hex_pk(),btc_key) - maker.privmsg(nick, command_prefix + 'io ' + ','.join(self.utxos) + ' ' + - self.cj_addr + ' ' + self.change_addr + ' ' + - btc_pub + ' ' + btc_sig + ' ' + maker.enc_kp.hex_pk()) + self.maker.start_encryption(nick,taker_pk) + + def auth_counterparty(self,nick,i_utxo_pubkey,btc_sig): + #TODO: add check that the pubkey's address is part of the order. + self.i_utxo_pubkey = i_utxo_pubkey + if not btc.ecdsa_verify(self.taker_pk,btc_sig,self.i_utxo_pubkey): + print 'signature didnt match pubkey and message' + return False + #authorisation of taker passed + #send auth request to taker + #TODO the next 2 lines are a little inefficient. + btc_key = self.maker.wallet.get_key_from_addr(self.cj_addr) + btc_pub = btc.privtopub(btc_key) + btc_sig = btc.ecdsa_sign(self.maker.enc_kp.hex_pk(),btc_key) + + self.maker.privmsg(nick, command_prefix + 'auth ' + str(','.join(self.utxos)) + ' ' + \ + btc_pub + ' ' + self.change_addr + ' ' + btc_sig) + return True + def recv_tx_part(self, b64txpart): self.b64txparts.append(b64txpart) size = sum([len(s) for s in self.b64txparts]) @@ -159,18 +172,6 @@ def send_error(self, nick, errmsg): def on_welcome(self): self.privmsg_all_orders(CHANNEL) - def auth_counterparty(self,nick,cjorder,enc_pk,btc_sig): - #TODO: add check that the pubkey's address is part of the order. - btc_pub = cjorder.i_utxo_pubkey - if not btc.ecdsa_verify(enc_pk,btc_sig,btc_pub): - print 'signature didnt match pubkey and message' - return False - #authorisation passed: initiate encryption for future - #messages - self.start_encryption(nick,enc_pk) - #send pubkey info to counterparty - return True - def on_privmsg(self, nick, message): debug("privmsg nick=%s message=%s" % (nick, message)) if message[0] != command_prefix: @@ -184,26 +185,26 @@ def on_privmsg(self, nick, message): if len(chunks) < 2: self.send_error(nick, 'Not enough arguments') if chunks[0] == 'auth': - if nick not in self.active_orders: - self.send_error(nick,"Encryption handshake message received out of order, ignored.") - continue + if nick not in self.active_orders or self.active_orders[nick] == None: + self.send_error(nick, 'No open order from this nick') + cjorder = self.active_orders[nick] try: - cjorder = self.active_orders[nick] - enc_pk = chunks[1] + i_utxo_pubkey = chunks[1] btc_sig = chunks[2] - self.auth_counterparty(nick,cjorder,enc_pk,btc_sig) except (ValueError,IndexError) as e: self.send_error(nick, str(e)) + self.active_orders[nick].auth_counterparty(nick,i_utxo_pubkey,btc_sig) + if chunks[0] == 'fill': if nick in self.active_orders and self.active_orders[nick] != None: self.send_error(nick, 'Already have partially-filled order') try: oid = int(chunks[1]) amount = int(chunks[2]) - i_utxo_pubkey = chunks[3] + taker_pk = chunks[3] except (ValueError, IndexError) as e: self.send_error(nick, str(e)) - self.active_orders[nick] = CoinJoinOrder(self, nick, oid, amount, i_utxo_pubkey) + self.active_orders[nick] = CoinJoinOrder(self, nick, oid, amount, taker_pk) elif chunks[0] == 'txpart' or chunks[0] == 'tx': if nick not in self.active_orders or self.active_orders[nick] == None: self.send_error(nick, 'No open order from this nick') diff --git a/taker.py b/taker.py index 5e4bb3bf..bc6c9ea8 100644 --- a/taker.py +++ b/taker.py @@ -31,9 +31,10 @@ def __init__(self, taker, cj_amount, orders, my_utxos, my_cj_addr, self.signing_btc_pub = btc.privtopub(taker.wallet.get_key_from_addr(self.signing_btc_add)) for c, oid in orders.iteritems(): taker.privmsg(c, command_prefix + 'fill ' + \ - str(oid) + ' ' + str(cj_amount) + ' ' + self.signing_btc_pub) + str(oid) + ' ' + str(cj_amount) + ' ' + taker.enc_kp.hex_pk()) - def recv_txio(self, nick, utxo_list, cj_addr, change_addr): + def recv_txio(self, nick, utxo_list, cj_pub, change_addr): + cj_addr = btc.pubtoaddr(cj_pub,get_addr_vbyte()) if nick not in self.nonrespondants: debug('nick(' + nick + ') not in nonrespondants ' + str(self.nonrespondants)) return @@ -195,27 +196,19 @@ class Taker(OrderbookWatch): def __init__(self,keyfile): OrderbookWatch.__init__(self) self.cjtx = None + self.maker_pks = {} #TODO have a list of maker's nick we're coinjoining with, so # that some other guy doesnt send you confusing stuff #maybe a start_cj_tx() method is needed self.init_encryption(keyfile) - def auth_counterparty(self,nick,btc_pub,btc_sig,cj_addr,enc_pk, signing_add): - if not cj_addr==btc.pubtoaddr(btc_pub,magicbyte=get_addr_vbyte()): - print 'coinjoin address and pubkey didnt match' - return False - if not btc.ecdsa_verify(enc_pk,btc_sig,btc_pub): + def auth_counterparty(self,nick,btc_sig,cj_pub): + '''Validate the counterpartys claim to own the btc + address/pubkey that will be used for coinjoining + with an ecdsa verification.''' + if not btc.ecdsa_verify(self.maker_pks[nick],btc_sig,cj_pub): print 'signature didnt match pubkey and message' return False - #send authorisation response - my_btc_sig = btc.ecdsa_sign(self.enc_kp.hex_pk(),\ - self.wallet.get_key_from_addr(signing_add)) - message = '!auth ' + self.enc_kp.hex_pk() + ' ' + my_btc_sig - self.privmsg(nick,message) #note: we do this *before* starting encryption - #TODO: should both sides send acks to acknowledge successful handshake? - #authorisation passed: initiate encryption for future - #messages - self.start_encryption(nick,enc_pk) return True def on_privmsg(self, nick, message): @@ -225,19 +218,27 @@ def on_privmsg(self, nick, message): return for command in message[1:].split(command_prefix): chunks = command.split(" ") - if chunks[0] == 'io': + if chunks[0] == 'pubkey': + maker_pk = chunks[1] + #store the declared pubkeys in a dict indexed by maker nick + self.maker_pks[nick] = maker_pk + self.start_encryption(nick, self.maker_pks[nick]) + #send authorisation request + my_btc_priv = self.wallet.get_key_from_addr(self.wallet.unspent[self.cjtx.my_utxos[0]]['address']) + my_btc_pub = btc.privtopub(my_btc_priv) + my_btc_sig = btc.ecdsa_sign(self.enc_kp.hex_pk(),my_btc_priv) + message = '!auth ' + my_btc_pub + ' ' + my_btc_sig + self.privmsg(nick,message) #note: we do this *before* starting encryption + if chunks[0] == 'auth': utxo_list = chunks[1].split(',') - cj_addr = chunks[2] + cj_pub = chunks[2] change_addr = chunks[3] - btc_pub = chunks[4] - btc_sig = chunks[5] - enc_pk = chunks[6] - if not self.auth_counterparty(nick,btc_pub,btc_sig,cj_addr,enc_pk,\ - self.cjtx.signing_btc_add): + btc_sig = chunks[4] + if not self.auth_counterparty(nick,btc_sig,cj_pub): print 'Authenticated encryption with counterparty: ' + nick + \ ' not established. TODO: send rejection message' continue - self.cjtx.recv_txio(nick, utxo_list, cj_addr, change_addr) + self.cjtx.recv_txio(nick, utxo_list, cj_pub, change_addr) elif chunks[0] == 'sig': sig = chunks[1] self.cjtx.add_signature(sig) From ff49b2239f12af65f0daf5a5b23f31edba641d67 Mon Sep 17 00:00:00 2001 From: Belcher Date: Mon, 12 Jan 2015 21:54:39 +0000 Subject: [PATCH 051/409] typo bugfix --- yield-generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yield-generator.py b/yield-generator.py index 2d8fc849..f27f5e10 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -80,11 +80,11 @@ def on_tx_unconfirmed(self, cjorder, balance, removed_utxos): neworders = self.create_my_orders() if len(neworders) == 0: return ([oldorder], []) #cancel old order - elif oldorder['maxsize'] == neworder['maxsize']: + elif oldorder['maxsize'] == neworders[0]['maxsize']: return ([], []) #change nothing else: #announce new order, replacing the old order - return ([], [neworder[0]]) + return ([], [neworders[0]]) def on_tx_confirmed(self, cjorder, confirmations, txid, balance, added_utxos): return self.on_tx_unconfirmed(None, None, None) From 5f11ef7b2ff0868e7e4931edde1249ff40c059f6 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Tue, 13 Jan 2015 00:05:24 +0200 Subject: [PATCH 052/409] encryption for yield-gen file --- yield-generator.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/yield-generator.py b/yield-generator.py index 2d8fc849..ae72efb1 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -27,8 +27,8 @@ #spent from utxos that try to make the highest balance even higher # so try to keep coins concentrated in one mixing depth class YieldGenerator(Maker): - def __init__(self, wallet): - Maker.__init__(self, wallet) + def __init__(self, wallet,keyfile): + Maker.__init__(self, wallet,keyfile) def on_connect(self): if len(nickserv_password) > 0: @@ -80,11 +80,11 @@ def on_tx_unconfirmed(self, cjorder, balance, removed_utxos): neworders = self.create_my_orders() if len(neworders) == 0: return ([oldorder], []) #cancel old order - elif oldorder['maxsize'] == neworder['maxsize']: + elif oldorder['maxsize'] == neworders[0]['maxsize']: return ([], []) #change nothing else: #announce new order, replacing the old order - return ([], [neworder[0]]) + return ([], [neworders[0]]) def on_tx_confirmed(self, cjorder, confirmations, txid, balance, added_utxos): return self.on_tx_unconfirmed(None, None, None) @@ -92,13 +92,13 @@ def on_tx_confirmed(self, cjorder, confirmations, txid, balance, added_utxos): def main(): import sys seed = sys.argv[1] #btc.sha256('dont use brainwallets except for holding testnet coins') - + keyfile = sys.argv[2] print 'downloading wallet history' wallet = Wallet(seed, max_mix_depth = mix_levels) wallet.download_wallet_history() wallet.find_unspent_addresses() - maker = YieldGenerator(wallet) + maker = YieldGenerator(wallet,keyfile) print 'connecting to irc' maker.run(HOST, PORT, nickname, CHANNEL) From ee3a589bc549e1b382101b7f1b8fb20115f0fd74 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Tue, 13 Jan 2015 23:10:58 +0200 Subject: [PATCH 053/409] back to default IRC chan --- common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common.py b/common.py index 4a697aa0..bf545b5d 100644 --- a/common.py +++ b/common.py @@ -5,7 +5,7 @@ import threading HOST = 'irc.freenode.net' -CHANNEL = '#joinmarket-pit-test2' +CHANNEL = '#joinmarket-pit-test' PORT = 6667 #for the mainnet its #joinmarket-pit From dbc22630b3ce8d8c71881118ee10a0a2f2a819a0 Mon Sep 17 00:00:00 2001 From: Belcher Date: Tue, 13 Jan 2015 22:46:09 +0000 Subject: [PATCH 054/409] made payments work with encryption, fixed bugs --- irclib.py | 1 + patientsendpayment.py | 11 ++++++----- sendpayment.py | 7 ++++--- yield-generator.py | 6 +++--- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/irclib.py b/irclib.py index 6c741360..4b3167c0 100644 --- a/irclib.py +++ b/irclib.py @@ -198,6 +198,7 @@ def __handle_line(self, line): self.on_leave(nick) elif chunks[1] == 'KICK': target = chunks[3] + nick = get_irc_nick(chunks[0]) self.on_leave(nick) elif chunks[1] == 'PART': nick = get_irc_nick(chunks[0]) diff --git a/patientsendpayment.py b/patientsendpayment.py index 3bce0742..a720316f 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -42,7 +42,7 @@ def run(self): self.tmaker.txfee, self.finishcallback) class PatientSendPayment(maker.Maker, taker.Taker): - def __init__(self, wallet, destaddr, amount, makercount, txfee, cjfee, + def __init__(self, wallet, keyfile, destaddr, amount, makercount, txfee, cjfee, waittime, mixdepth): self.destaddr = destaddr self.amount = amount @@ -51,8 +51,8 @@ def __init__(self, wallet, destaddr, amount, makercount, txfee, cjfee, self.cjfee = cjfee self.waittime = waittime self.mixdepth = mixdepth - maker.Maker.__init__(self, wallet) - taker.Taker.__init__(self) + maker.Maker.__init__(self, wallet, keyfile) + taker.Taker.__init__(self, keyfile) def on_privmsg(self, nick, message): maker.Maker.on_privmsg(self, nick, message) @@ -148,11 +148,12 @@ def main(): wallet = Wallet(seed) wallet.download_wallet_history() wallet.find_unspent_addresses() + keyfile = 'keyfile-' + str(seed) + '.txt' utxo_list = wallet.get_mix_utxo_list()[options.mixdepth] available_balance = 0 for utxo in utxo_list: - available_balance = wallet.unspent[utxo]['value'] + available_balance += wallet.unspent[utxo]['value'] if available_balance < amount: print 'not enough money at mixdepth=%d, exiting' % (options.mixdepth) return @@ -161,7 +162,7 @@ def main(): nickname = 'ppayer-' + btc.sha256(gethostname())[:6] print 'starting irc' - bot = PatientSendPayment(wallet, destaddr, amount, options.makercount, + bot = PatientSendPayment(wallet, keyfile, destaddr, amount, options.makercount, options.txfee, options.cjfee, waittime, options.mixdepth) bot.run(HOST, PORT, nickname, CHANNEL) diff --git a/sendpayment.py b/sendpayment.py index 2302e6aa..656c2721 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -102,8 +102,8 @@ def run(self): ''' class SendPayment(takermodule.Taker): - def __init__(self, wallet, destaddr, amount, makercount, txfee, waittime, mixdepth): - takermodule.Taker.__init__(self) + def __init__(self, wallet, keyfile, destaddr, amount, makercount, txfee, waittime, mixdepth): + takermodule.Taker.__init__(self, keyfile) self.wallet = wallet self.destaddr = destaddr self.amount = amount @@ -144,9 +144,10 @@ def main(): wallet = Wallet(seed) wallet.download_wallet_history() wallet.find_unspent_addresses() + keyfile = 'keyfile-' + str(seed) + '.txt' print 'starting irc' - taker = SendPayment(wallet, destaddr, amount, options.makercount, options.txfee, + taker = SendPayment(wallet, keyfile, destaddr, amount, options.makercount, options.txfee, options.waittime, options.mixdepth) taker.run(HOST, PORT, nickname, CHANNEL) diff --git a/yield-generator.py b/yield-generator.py index ae72efb1..d63c368f 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -79,7 +79,7 @@ def on_tx_unconfirmed(self, cjorder, balance, removed_utxos): oldorder = self.orderlist[0] neworders = self.create_my_orders() if len(neworders) == 0: - return ([oldorder], []) #cancel old order + return ([0], []) #cancel old order elif oldorder['maxsize'] == neworders[0]['maxsize']: return ([], []) #change nothing else: @@ -92,13 +92,13 @@ def on_tx_confirmed(self, cjorder, confirmations, txid, balance, added_utxos): def main(): import sys seed = sys.argv[1] #btc.sha256('dont use brainwallets except for holding testnet coins') - keyfile = sys.argv[2] print 'downloading wallet history' wallet = Wallet(seed, max_mix_depth = mix_levels) wallet.download_wallet_history() wallet.find_unspent_addresses() - maker = YieldGenerator(wallet,keyfile) + keyfile = 'keyfile-' + str(seed) + '.txt' + maker = YieldGenerator(wallet, keyfile) print 'connecting to irc' maker.run(HOST, PORT, nickname, CHANNEL) From fb9a266327f6109162027c178de4098573facdda Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 16 Jan 2015 17:03:41 +0000 Subject: [PATCH 055/409] socket bug fixed --- irclib.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/irclib.py b/irclib.py index 4b3167c0..c0f93b15 100644 --- a/irclib.py +++ b/irclib.py @@ -231,13 +231,13 @@ def run(self, server, port, nick, channel, username='username', realname='realna PingThread(self).start() while self.connect_attempts < 10 and not self.give_up: - print 'connecting' - self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.sock.connect((server, port)) - self.fd = self.sock.makefile() - self.send_raw('USER %s b c :%s' % (username, realname)) - self.send_raw('NICK ' + nick) try: + print 'connecting' + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.connect((server, port)) + self.fd = self.sock.makefile() + self.send_raw('USER %s b c :%s' % (username, realname)) + self.send_raw('NICK ' + nick) while 1: try: line = self.fd.readline() From 4a88c6e497095f07b5ec174828b4229e447ac503 Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 16 Jan 2015 17:38:12 +0000 Subject: [PATCH 056/409] attempted bugfix for hanging socket --- irclib.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/irclib.py b/irclib.py index c0f93b15..ef20f9eb 100644 --- a/irclib.py +++ b/irclib.py @@ -35,7 +35,9 @@ def run(self): except IOError: pass try: self.irc.fd.close() except IOError: pass - try: self.irc.sock.close() + try: + self.irc.sock.shutdown(SHUT_RDWR) + self.irc.sock.close() except IOError: pass except IOError as e: debug('ping thread: ' + repr(e)) From 4889312585f510f67896141fd1aad63a677ef3d7 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Sat, 17 Jan 2015 14:54:53 +0200 Subject: [PATCH 057/409] add reset option to wallet tool --- common.py | 7 +++++++ wallet-tool.py | 22 ++++++++++++++++------ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/common.py b/common.py index bf545b5d..41baa3f6 100644 --- a/common.py +++ b/common.py @@ -32,6 +32,13 @@ def get_addr_vbyte(): else: return 0x00 +def get_signed_tx(wallet,ins,outs): + tx = btc.mktx(ins, outs) + for index, utxo in enumerate(ins): + addr = wallet.unspent[utxo['output']]['address'] + tx = btc.sign(tx, index, wallet.get_key_from_addr(addr)) + return tx + class Wallet(object): def __init__(self, seed, max_mix_depth=2): self.max_mix_depth = max_mix_depth diff --git a/wallet-tool.py b/wallet-tool.py index 3f0534a2..9437a76e 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -1,6 +1,6 @@ import bitcoin as btc -from common import Wallet +from common import Wallet, get_signed_tx import sys from optparse import OptionParser @@ -61,6 +61,7 @@ used = ('used' if k < wallet.index[m][forchange] else ' new') print ' m/0/%d/%d/%02d %s %s %.8fbtc' % (m, forchange, k, addr, used, balance/1e8) print 'for mixdepth=%d balance=%.8fbtc' % (m, balance_depth/1e8) + elif method == 'combine': ins = [] outs = [] @@ -78,8 +79,17 @@ if balance > 0: destaddr = wallet.get_addr(m, forchange, wallet.index[m][forchange]) outs.append({'address': destaddr, 'value': balance}) - tx = btc.mktx(ins, outs) - for index, utxo in enumerate(ins): - addr = wallet.unspent[utxo['output']]['address'] - tx = btc.sign(tx, index, wallet.get_key_from_addr(addr)) - print tx + print get_signed_tx(wallet,ins,outs) + +elif method == 'reset': + ins = [] + outs = [] + balance = 0 + for utxo,addrvalue in wallet.unspent.iteritems(): + ins.append({'output': utxo}) + balance += addrvalue['value'] + destaddr = wallet.get_addr(0,0,0) + outs.append({'address': destaddr, 'value': balance}) + print get_signed_tx(wallet,ins,outs) + + \ No newline at end of file From a441c31079913e81876a0f575b68a44771c72e69 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sat, 17 Jan 2015 17:34:42 +0000 Subject: [PATCH 058/409] fixed bug --- irclib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/irclib.py b/irclib.py index ef20f9eb..a34abb61 100644 --- a/irclib.py +++ b/irclib.py @@ -36,7 +36,7 @@ def run(self): try: self.irc.fd.close() except IOError: pass try: - self.irc.sock.shutdown(SHUT_RDWR) + self.irc.sock.shutdown(socket.SHUT_RDWR) self.irc.sock.close() except IOError: pass except IOError as e: From fa62d543b3f484ce97070f291457e43217a3842d Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 18 Jan 2015 17:29:49 +0000 Subject: [PATCH 059/409] wallet prints out debug information at start --- common.py | 16 +++++++++++++++- maker.py | 5 ++--- patientsendpayment.py | 4 +--- sendpayment.py | 4 +--- taker.py | 4 +--- yield-generator.py | 5 ++--- 6 files changed, 22 insertions(+), 16 deletions(-) diff --git a/common.py b/common.py index bf545b5d..c0597a9f 100644 --- a/common.py +++ b/common.py @@ -1,7 +1,7 @@ import bitcoin as btc from decimal import Decimal -import sys, datetime, json, time +import sys, datetime, json, time, pprint import threading HOST = 'irc.freenode.net' @@ -111,6 +111,12 @@ def get_mix_utxo_list(self): mix_utxo_list[mixdepth].append(utxo) return mix_utxo_list + def sync_wallet(self, gaplimit=6): + debug('synchronizing wallet') + self.download_wallet_history(gaplimit) + self.find_unspent_addresses() + self.print_debug_wallet_info() + def download_wallet_history(self, gaplimit=6): ''' sets Wallet internal indexes to be at the next unused address @@ -187,6 +193,14 @@ def find_unspent_addresses(self): self.unspent[u['tx']+':'+str(u['n'])] = {'address': dat['address'], 'value': int(u['amount'].replace('.', ''))} + def print_debug_wallet_info(self): + debug('printing debug wallet information') + print 'utxos' + pprint.pprint(self.unspent) + print 'wallet.index' + pprint.pprint(self.index) + + #awful way of doing this, but works for now # and -walletnotify for people who do #timeouts in minutes diff --git a/maker.py b/maker.py index 8280722c..ad3874b3 100644 --- a/maker.py +++ b/maker.py @@ -343,10 +343,9 @@ def main(): import sys seed = sys.argv[1] #btc.sha256('dont use brainwallets except for holding testnet coins') keyfile = sys.argv[2] - print 'downloading wallet history' + wallet = Wallet(seed,max_mix_depth=5) - wallet.download_wallet_history() - wallet.find_unspent_addresses() + wallet.sync_wallet() maker = Maker(wallet,keyfile) print 'connecting to irc' diff --git a/patientsendpayment.py b/patientsendpayment.py index a720316f..d0558e44 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -144,10 +144,8 @@ def main(): print 'txfee=%d cjfee=%d waittime=%s makercount=%d' % (options.txfee, options.cjfee, str(timedelta(hours=options.waittime)), options.makercount) - print 'downloading wallet history' wallet = Wallet(seed) - wallet.download_wallet_history() - wallet.find_unspent_addresses() + wallet.sync_wallet() keyfile = 'keyfile-' + str(seed) + '.txt' utxo_list = wallet.get_mix_utxo_list()[options.mixdepth] diff --git a/sendpayment.py b/sendpayment.py index 656c2721..f264213a 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -140,10 +140,8 @@ def main(): from socket import gethostname nickname = 'payer-' + btc.sha256(gethostname())[:6] - print 'downloading wallet history' wallet = Wallet(seed) - wallet.download_wallet_history() - wallet.find_unspent_addresses() + wallet.sync_wallet() keyfile = 'keyfile-' + str(seed) + '.txt' print 'starting irc' diff --git a/taker.py b/taker.py index bc6c9ea8..9996af41 100644 --- a/taker.py +++ b/taker.py @@ -311,10 +311,8 @@ def main(): from socket import gethostname nickname = 'taker-' + btc.sha256(gethostname())[:6] - print 'downloading wallet history' wallet = Wallet(seed, max_mix_depth=5) - wallet.download_wallet_history() - wallet.find_unspent_addresses() + wallet.sync_wallet() print 'starting irc' taker = TestTaker(wallet,keyfile) diff --git a/yield-generator.py b/yield-generator.py index d63c368f..e3bce9a8 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -92,10 +92,9 @@ def on_tx_confirmed(self, cjorder, confirmations, txid, balance, added_utxos): def main(): import sys seed = sys.argv[1] #btc.sha256('dont use brainwallets except for holding testnet coins') - print 'downloading wallet history' + wallet = Wallet(seed, max_mix_depth = mix_levels) - wallet.download_wallet_history() - wallet.find_unspent_addresses() + wallet.sync_wallet() keyfile = 'keyfile-' + str(seed) + '.txt' maker = YieldGenerator(wallet, keyfile) From 76ff4be2e7854863d554a80782eac43f2ef300c4 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Sun, 18 Jan 2015 19:34:29 +0200 Subject: [PATCH 060/409] encryption end intermediate commit --- common.py | 2 +- maker.py | 4 +++- taker.py | 6 +++++- wallet-tool.py | 4 ++-- yield-generator.py | 5 ++--- 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/common.py b/common.py index 41baa3f6..d61120ed 100644 --- a/common.py +++ b/common.py @@ -32,7 +32,7 @@ def get_addr_vbyte(): else: return 0x00 -def get_signed_tx(wallet,ins,outs): +def get_signed_tx(wallet, ins, outs): tx = btc.mktx(ins, outs) for index, utxo in enumerate(ins): addr = wallet.unspent[utxo['output']]['address'] diff --git a/maker.py b/maker.py index 8280722c..ed9fc3fa 100644 --- a/maker.py +++ b/maker.py @@ -94,6 +94,8 @@ def recv_tx(self, nick, b64txpart): sigline = command_prefix + 'sig ' + sig if len(sigline) > 0: self.maker.privmsg(nick, sigline) + #once signature is sent, close encrypted channel to this taker. + self.maker.end_encryption(nick) def unconfirm_callback(self, balance): removed_utxos = self.maker.wallet.remove_old_utxos(self.tx) @@ -193,7 +195,7 @@ def on_privmsg(self, nick, message): btc_sig = chunks[2] except (ValueError,IndexError) as e: self.send_error(nick, str(e)) - self.active_orders[nick].auth_counterparty(nick,i_utxo_pubkey,btc_sig) + self.active_orders[nick].auth_counterparty(nick, i_utxo_pubkey, btc_sig) if chunks[0] == 'fill': if nick in self.active_orders and self.active_orders[nick] != None: diff --git a/taker.py b/taker.py index bc6c9ea8..531b0056 100644 --- a/taker.py +++ b/taker.py @@ -240,6 +240,10 @@ def on_privmsg(self, nick, message): continue self.cjtx.recv_txio(nick, utxo_list, cj_pub, change_addr) elif chunks[0] == 'sig': + #whether the signature is correct or not, + #we treat this as the last message for this tx from + #this nick, and therefore switch off encryption + self.end_encryption(nick) sig = chunks[1] self.cjtx.add_signature(sig) @@ -309,7 +313,7 @@ def main(): seed = sys.argv[1] #btc.sha256('your brainwallet goes here') keyfile = sys.argv[2] from socket import gethostname - nickname = 'taker-' + btc.sha256(gethostname())[:6] + nickname = 'taker-' + sys.argv[2][:3] + btc.sha256(gethostname())[:6] print 'downloading wallet history' wallet = Wallet(seed, max_mix_depth=5) diff --git a/wallet-tool.py b/wallet-tool.py index 9437a76e..a37b1bd6 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -79,7 +79,7 @@ if balance > 0: destaddr = wallet.get_addr(m, forchange, wallet.index[m][forchange]) outs.append({'address': destaddr, 'value': balance}) - print get_signed_tx(wallet,ins,outs) + print get_signed_tx(wallet, ins, outs) elif method == 'reset': ins = [] @@ -90,6 +90,6 @@ balance += addrvalue['value'] destaddr = wallet.get_addr(0,0,0) outs.append({'address': destaddr, 'value': balance}) - print get_signed_tx(wallet,ins,outs) + print get_signed_tx(wallet, ins, outs) \ No newline at end of file diff --git a/yield-generator.py b/yield-generator.py index d63c368f..e8e983e9 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -7,7 +7,6 @@ import pprint from socket import gethostname -nickname = 'yigen-' + btc.sha256(gethostname())[:6] txfee = 1000 cjfee = '0.01' # 1% fee @@ -96,8 +95,8 @@ def main(): wallet = Wallet(seed, max_mix_depth = mix_levels) wallet.download_wallet_history() wallet.find_unspent_addresses() - - keyfile = 'keyfile-' + str(seed) + '.txt' + keyfile = sys.argv[2] + nickname = 'yigen-' + sys.argv[2][:3] + btc.sha256(gethostname())[:6] maker = YieldGenerator(wallet, keyfile) print 'connecting to irc' maker.run(HOST, PORT, nickname, CHANNEL) From 8121f8dc02dc5587756eeee2f0fc7642ffe10d85 Mon Sep 17 00:00:00 2001 From: Belcher Date: Tue, 20 Jan 2015 22:30:36 +0000 Subject: [PATCH 061/409] added dump code when the bot crashes --- common.py | 15 +++++++++++++++ maker.py | 8 +++++++- patientsendpayment.py | 8 +++++++- sendpayment.py | 8 +++++++- taker.py | 9 +++++++-- yield-generator.py | 8 +++++++- 6 files changed, 50 insertions(+), 6 deletions(-) diff --git a/common.py b/common.py index c0597a9f..2683df29 100644 --- a/common.py +++ b/common.py @@ -32,6 +32,21 @@ def get_addr_vbyte(): else: return 0x00 +def debug_dump_object(obj, skip_fields=[]): + print 'Class debug dump, name:' + obj.__class__.__name__ + for k, v in obj.__dict__.iteritems(): + if k in skip_fields: + continue + print 'key=' + k + if isinstance(v, str): + print 'string: len:' + str(len(v)) + print v + elif isinstance(v, dict) or isinstance(v, list): + pprint.pprint(v) + else: + print v + + class Wallet(object): def __init__(self, seed, max_mix_depth=2): self.max_mix_depth = max_mix_depth diff --git a/maker.py b/maker.py index ad3874b3..0fae6797 100644 --- a/maker.py +++ b/maker.py @@ -349,7 +349,13 @@ def main(): maker = Maker(wallet,keyfile) print 'connecting to irc' - maker.run(HOST, PORT, nickname, CHANNEL) + try: + maker.run(HOST, PORT, nickname, CHANNEL) + finally: + debug('CRASHING, DUMPING EVERYTHING') + debug('wallet seed = ' + seed) + debug_dump_object(wallet, ['addr_cache']) + debug_dump_object(taker) if __name__ == "__main__": main() diff --git a/patientsendpayment.py b/patientsendpayment.py index d0558e44..47d4728a 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -162,7 +162,13 @@ def main(): print 'starting irc' bot = PatientSendPayment(wallet, keyfile, destaddr, amount, options.makercount, options.txfee, options.cjfee, waittime, options.mixdepth) - bot.run(HOST, PORT, nickname, CHANNEL) + try: + bot.run(HOST, PORT, nickname, CHANNEL) + finally: + debug('CRASHING, DUMPING EVERYTHING') + debug('wallet seed = ' + seed) + debug_dump_object(wallet, ['addr_cache']) + debug_dump_object(taker) if __name__ == "__main__": main() diff --git a/sendpayment.py b/sendpayment.py index f264213a..41d972b6 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -147,7 +147,13 @@ def main(): print 'starting irc' taker = SendPayment(wallet, keyfile, destaddr, amount, options.makercount, options.txfee, options.waittime, options.mixdepth) - taker.run(HOST, PORT, nickname, CHANNEL) + try: + taker.run(HOST, PORT, nickname, CHANNEL) + finally: + debug('CRASHING, DUMPING EVERYTHING') + debug('wallet seed = ' + seed) + debug_dump_object(wallet, ['addr_cache']) + debug_dump_object(taker) if __name__ == "__main__": main() diff --git a/taker.py b/taker.py index 9996af41..69e5486e 100644 --- a/taker.py +++ b/taker.py @@ -271,7 +271,6 @@ def on_pubmsg(self, nick, message): self.cjtx = CoinJoinTX(self, int(amt), {cp: oid}, [utxos[0]], self.wallet.get_receive_addr(mixing_depth=1), self.wallet.get_change_addr(mixing_depth=0), my_tx_fee) - if chunks[0] == '%showob': print('printing orderbook') for o in self.db.execute('SELECT * FROM orderbook;').fetchall(): @@ -316,7 +315,13 @@ def main(): print 'starting irc' taker = TestTaker(wallet,keyfile) - taker.run(HOST, PORT, nickname, CHANNEL) + try: + taker.run(HOST, PORT, nickname, CHANNEL) + finally: + debug('CRASHING, DUMPING EVERYTHING') + debug('wallet seed = ' + seed) + debug_dump_object(wallet, ['addr_cache']) + debug_dump_object(taker) if __name__ == "__main__": main() diff --git a/yield-generator.py b/yield-generator.py index e3bce9a8..4362889a 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -99,7 +99,13 @@ def main(): keyfile = 'keyfile-' + str(seed) + '.txt' maker = YieldGenerator(wallet, keyfile) print 'connecting to irc' - maker.run(HOST, PORT, nickname, CHANNEL) + try: + maker.run(HOST, PORT, nickname, CHANNEL) + finally: + debug('CRASHING, DUMPING EVERYTHING') + debug('wallet seed = ' + seed) + debug_dump_object(wallet, ['addr_cache']) + debug_dump_object(taker) if __name__ == "__main__": main() From 93d34f70be9a74192051acf36501411e1a5845af Mon Sep 17 00:00:00 2001 From: Belcher Date: Tue, 20 Jan 2015 22:47:00 +0000 Subject: [PATCH 062/409] rewrote topic reading/printing code --- maker.py | 8 ++++---- taker.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/maker.py b/maker.py index 0fae6797..d7a944ca 100644 --- a/maker.py +++ b/maker.py @@ -232,11 +232,11 @@ def on_pubmsg(self, nick, message): def on_set_topic(self, newtopic): chunks = newtopic.split('|') - try: + if len(chunks) > 1: + print '=' * 60 + print 'MESSAGE FROM BELCHER!' print chunks[1].strip() - print chunks[3].strip() - except IndexError: - pass + print '=' * 60 def on_leave(self, nick): self.active_orders[nick] = None diff --git a/taker.py b/taker.py index 69e5486e..c7d82ee0 100644 --- a/taker.py +++ b/taker.py @@ -179,11 +179,11 @@ def on_welcome(self): def on_set_topic(self, newtopic): chunks = newtopic.split('|') - try: + if len(chunks) > 1: + print '=' * 60 + print 'MESSAGE FROM BELCHER!' print chunks[1].strip() - print chunks[2].strip() - except IndexError: - pass + print '=' * 60 def on_leave(self, nick): self.db.execute('DELETE FROM orderbook WHERE counterparty=?;', (nick,)) From 0592813fc0252323762c4bbef5bfd86ec33f7a11 Mon Sep 17 00:00:00 2001 From: Belcher Date: Tue, 20 Jan 2015 22:53:46 +0000 Subject: [PATCH 063/409] removed !%quit --- maker.py | 2 -- taker.py | 4 +--- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/maker.py b/maker.py index d7a944ca..a548cec1 100644 --- a/maker.py +++ b/maker.py @@ -227,8 +227,6 @@ def on_pubmsg(self, nick, message): chunks = message[1:].split(" ") if chunks[0] == 'orderbook': self.privmsg_all_orders(nick) - elif chunks[0] == '%quit' or chunks[0] == '%makerquit': - self.shutdown() def on_set_topic(self, newtopic): chunks = newtopic.split('|') diff --git a/taker.py b/taker.py index c7d82ee0..e4646ab1 100644 --- a/taker.py +++ b/taker.py @@ -160,9 +160,7 @@ def on_pubmsg(self, nick, message): for command in message[1:].split(command_prefix): #commands starting with % are for testing and will be removed in the final version chunks = command.split(" ") - if chunks[0] == '%quit' or chunks[0] == '%takerquit': - self.shutdown() - elif chunks[0] == 'cancel': + if chunks[0] == 'cancel': #!cancel [oid] try: oid = int(chunks[1]) From 723b662e5612cc6c8376e696f05195cce33864f5 Mon Sep 17 00:00:00 2001 From: Belcher Date: Wed, 21 Jan 2015 00:40:28 +0000 Subject: [PATCH 064/409] fixed bug --- maker.py | 2 +- patientsendpayment.py | 2 +- yield-generator.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/maker.py b/maker.py index a548cec1..1bc16389 100644 --- a/maker.py +++ b/maker.py @@ -353,7 +353,7 @@ def main(): debug('CRASHING, DUMPING EVERYTHING') debug('wallet seed = ' + seed) debug_dump_object(wallet, ['addr_cache']) - debug_dump_object(taker) + debug_dump_object(maker) if __name__ == "__main__": main() diff --git a/patientsendpayment.py b/patientsendpayment.py index 47d4728a..f705ec6e 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -168,7 +168,7 @@ def main(): debug('CRASHING, DUMPING EVERYTHING') debug('wallet seed = ' + seed) debug_dump_object(wallet, ['addr_cache']) - debug_dump_object(taker) + debug_dump_object(bot) if __name__ == "__main__": main() diff --git a/yield-generator.py b/yield-generator.py index 4362889a..cdcc4438 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -105,7 +105,7 @@ def main(): debug('CRASHING, DUMPING EVERYTHING') debug('wallet seed = ' + seed) debug_dump_object(wallet, ['addr_cache']) - debug_dump_object(taker) + debug_dump_object(maker) if __name__ == "__main__": main() From e144ac17c37d42c54fb21134ce56773ae674d73c Mon Sep 17 00:00:00 2001 From: Belcher Date: Wed, 21 Jan 2015 00:42:06 +0000 Subject: [PATCH 065/409] fixed support for nickserv --- yield-generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yield-generator.py b/yield-generator.py index cdcc4438..1a832092 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -32,7 +32,7 @@ def __init__(self, wallet,keyfile): def on_connect(self): if len(nickserv_password) > 0: - self.privmsg('NickServ', 'identify ' + nickserv_password) + self.send_raw('PRIVMSG NickServ :identify ' + nickserv_password) def create_my_orders(self): mix_utxo_list = self.wallet.get_mix_utxo_list() From 4914dc340ed539e59676f86451cfd0db5a293060 Mon Sep 17 00:00:00 2001 From: Belcher Date: Wed, 21 Jan 2015 00:44:29 +0000 Subject: [PATCH 066/409] seperated TODOs into their own file --- README.txt | 119 +---------------------------------------------------- TODO | 117 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+), 118 deletions(-) create mode 100644 TODO diff --git a/README.txt b/README.txt index f4f1e5dc..33a8b9b9 100644 --- a/README.txt +++ b/README.txt @@ -6,7 +6,7 @@ you will need to know python somewhat to play around with it HOWTO try 1. create two wallet seeds string (can be just brainwallets if you're only storing testnet btc) one seed for each maker and taker - use bip32-tool.py to output a bunch of addresses from the seeds + use wallet-tool.py to output a bunch of addresses from the seeds send testnet coins to one mixing-depth=0 receive address seeds are taken as a command line argument @@ -61,120 +61,3 @@ it signs its own utxos and extracts just the script from it which contains signa taker collects all scripts and places them into the tx taker pushes tx when all the scripts have arrived - -#TODO -#ask people on the testnet stuff to code up a few trading algos to see if the interface/protocol that -# iv invented is general enough -a few algos: -fees proportional to how many utxos used, since the marginal cost is unrelated to your cj amount, only to - the amount of utxos you use up - -#TODO dont always pick the lowest cost order, instead have an exponentially decaying -# distribution, so most of the time you pick the lowest and sometimes you take higher ones -# this represents your uncertainty in sybil attackers, the cheapest may not always be the best -#i.e. randomly chosen makers, weighted by the price they offer - -#TODO on nickname change, change also the counterparty variable in any open orders - -#TODO use electrum json_rpc instead of the pybitcointools stuff -# problem, i dont think that supports testnet -# bitcoind json_rpc obviously supports testnet, but someone else can download -# the blockchain, actually it seems you cant replace pybitcointools with bitcoind -# cant look up any txid or address -# could use a websocket api for learning when new blocks/tx appear -# could use python-bitcoinlib to be a node in the p2p network - -#TODO option for how many blocks deep to wait before using a utxo for more mixing -# 1 confirm is probably enough -TODO -have the taker enforce this, look up the txhash of the maker's utxo and make sure - it is already in a block - - -TODO implement rate limiting for irc.privmsg to stop the bot being killed due to flood -i suggest creating a thread that only dispatches/writes to the irc socket - -TODO sort out the nick = nick + '_' stuff in irclib -its not a good way of doing it - -#TODO encrypt messages between taker and maker, to stop trivial server eavesdropping -# but that wont stop mitm -# after chats on irc, easiest is to do Trust On First Use, maker sends a pubkey over -# TOFU requires a human to verify each first time, might not be practical -# skip the human verification, it will probably be okay -# make the irc nick be a hash of the pubkey -# also theres some algorithm for detecting mitm - -#TODO implement something against dust -# e.g. where the change address ends up having an output of value 1000 satoshis - -#TODO completely abstract away the irc stuff, so it can be switched to something else -# e.g. twitter but more likely darkwallet obelisk and/or electrum server - -TODO combine the taker and maker code into one file where you can make different kinds of - bot which combine both roles -e.g. tumbler.py repeatedly takes orders on the same coins again and again in an effort - to improve privacy and break the link between them, make sure to split up and combine them again - in random amounts, because the yield-generator will also be splitting and combining coins - random intervals between blocks included might be worth it too, since yield-generator.py - will appear to have coins which dont get mixed again for a while -e.g. patient-tumbler.py which waits a while being a maker, then just starts to take orders - after a time limit for people who want to mix coins but dont mind waiting until a fixed upper time limit -e.g. yield-generator.py which acts as a maker solely for the purpose of making money - might need to take orders at some point, for very small outputs which have a small probability of being filled -e.g. single-tx.py which takes a single order, using it to send coins to some address - typically as a payment, so this is what the electrum plugin would look like -e.g. patient-single-tx.py which does the above but doesnt mind waiting up to a limit -e.g. gui-taker.py has a gui which shows the user the orderbook and they can easily fill and order - and see other statistics, could be easily done by opening a http port and sending a html form and graphics - -TODO -implement this the thing that gmaxwell wrote about in the original coinjoin post, as a kind of tumbler -"Isn't the anonymity set size limited by how many parties you can get in a single transaction?" - -"Not quite. The anonymity set size of a single transaction is limited by the number of parties in it, obviously. And transaction size limits as well as failure (retry) risk mean that really huge joint transactions would not be wise. But because these transactions are cheap, there is no limit to the number of transactions you can cascade. - -In particular, if you have can build transactions with m participants per transaction you can create a sequence of m*3 transactions which form a three-stage switching network that permits any of m^2 final outputs to have come from any of m^2 original inputs (e.g. using three stages of 32 transactions with 32 inputs each 1024 users can be joined with a total of 96 transactions). This allows the anonymity set to be any size, limited only by participation." -https://en.wikipedia.org/wiki/Clos_network -Not sure if it will actually be possible in this liquidity maker/taker system - -TODO need to move onto the bip44 structure of HD wallets - -TODO think about this -<> some coinjoin tools we use today were broken -<> one allowed people to use a mix of uncompressed and compressed keys, so it was obvious which party was which. - -TODO -probably a good idea to have a debug.log where loads of information is dumped - -TODO -for the !addrs command, firstly change its name since it also includes the utxo inputs - secondly, the utxo list might be longer than can fit in an irc message, so create a - !addrsparts or something command - -TODO -code a gui where a human can see the state of the orderbook and easily choose orders to fill -code a gui that easily explains to a human how they can choose a fee for their yield-generator.py -both are important for market forces, since markets emerge from human decisions and actions - -#TODO add random delays to the orderbook stuff so there isnt such a traffic spike when a new bot joins -#two options, random delay !orderbook for ones which dont mind, !orderbook without delay for bots -# which need the orders asap - -TODO -code something that extends orderbookwatch and creates graphs - those graphs can be posted to a bitcointalk thread (like the bitstamp wall watch thread) - and could be a nice historical record and guide to pricing - -TODO -code something that analyzes the blockchain, detects coinjoin tx likely made by joinmarket - and calculates the paid fee, therefore is a guide to pricing - -TODO -the add_addr_notify() stuff doesnt work, so if theres several CoinJoinOrder's open it will start a few - threads to do the notifying, they could race condition or other multithreaded errors -i suggest to create a single thread that sorts out all the stuff - -#TODO make an ordertype where maker publishes the utxo he will use -# this is a way to auction off the use of a desirable coin, maybe a -# very newly mined coin or one which hasnt been moved for years diff --git a/TODO b/TODO new file mode 100644 index 00000000..47bfb6ff --- /dev/null +++ b/TODO @@ -0,0 +1,117 @@ + +#TODO +#ask people on the testnet stuff to code up a few trading algos to see if the interface/protocol that +# iv invented is general enough +a few algos: +fees proportional to how many utxos used, since the marginal cost is unrelated to your cj amount, only to + the amount of utxos you use up + +#TODO dont always pick the lowest cost order, instead have an exponentially decaying +# distribution, so most of the time you pick the lowest and sometimes you take higher ones +# this represents your uncertainty in sybil attackers, the cheapest may not always be the best +#i.e. randomly chosen makers, weighted by the price they offer + +#TODO on nickname change, change also the counterparty variable in any open orders + +#TODO use electrum json_rpc instead of the pybitcointools stuff +# problem, i dont think that supports testnet +# bitcoind json_rpc obviously supports testnet, but someone else can download +# the blockchain, actually it seems you cant replace pybitcointools with bitcoind +# cant look up any txid or address +# could use a websocket api for learning when new blocks/tx appear +# could use python-bitcoinlib to be a node in the p2p network + +#TODO option for how many blocks deep to wait before using a utxo for more mixing +# 1 confirm is probably enough +TODO +have the taker enforce this, look up the txhash of the maker's utxo and make sure + it is already in a block + + +TODO implement rate limiting for irc.privmsg to stop the bot being killed due to flood +i suggest creating a thread that only dispatches/writes to the irc socket + +TODO sort out the nick = nick + '_' stuff in irclib +its not a good way of doing it + +#TODO encrypt messages between taker and maker, to stop trivial server eavesdropping +# but that wont stop mitm +# after chats on irc, easiest is to do Trust On First Use, maker sends a pubkey over +# TOFU requires a human to verify each first time, might not be practical +# skip the human verification, it will probably be okay +# make the irc nick be a hash of the pubkey +# also theres some algorithm for detecting mitm + +#TODO implement something against dust +# e.g. where the change address ends up having an output of value 1000 satoshis + +#TODO completely abstract away the irc stuff, so it can be switched to something else +# e.g. twitter but more likely darkwallet obelisk and/or electrum server + +TODO combine the taker and maker code into one file where you can make different kinds of + bot which combine both roles +e.g. tumbler.py repeatedly takes orders on the same coins again and again in an effort + to improve privacy and break the link between them, make sure to split up and combine them again + in random amounts, because the yield-generator will also be splitting and combining coins + random intervals between blocks included might be worth it too, since yield-generator.py + will appear to have coins which dont get mixed again for a while +e.g. patient-tumbler.py which waits a while being a maker, then just starts to take orders + after a time limit for people who want to mix coins but dont mind waiting until a fixed upper time limit +e.g. yield-generator.py which acts as a maker solely for the purpose of making money + might need to take orders at some point, for very small outputs which have a small probability of being filled +e.g. single-tx.py which takes a single order, using it to send coins to some address + typically as a payment, so this is what the electrum plugin would look like +e.g. patient-single-tx.py which does the above but doesnt mind waiting up to a limit +e.g. gui-taker.py has a gui which shows the user the orderbook and they can easily fill and order + and see other statistics, could be easily done by opening a http port and sending a html form and graphics + +TODO +implement this the thing that gmaxwell wrote about in the original coinjoin post, as a kind of tumbler +"Isn't the anonymity set size limited by how many parties you can get in a single transaction?" + +"Not quite. The anonymity set size of a single transaction is limited by the number of parties in it, obviously. And transaction size limits as well as failure (retry) risk mean that really huge joint transactions would not be wise. But because these transactions are cheap, there is no limit to the number of transactions you can cascade. + +In particular, if you have can build transactions with m participants per transaction you can create a sequence of m*3 transactions which form a three-stage switching network that permits any of m^2 final outputs to have come from any of m^2 original inputs (e.g. using three stages of 32 transactions with 32 inputs each 1024 users can be joined with a total of 96 transactions). This allows the anonymity set to be any size, limited only by participation." +https://en.wikipedia.org/wiki/Clos_network +Not sure if it will actually be possible in this liquidity maker/taker system + +TODO need to move onto the bip44 structure of HD wallets + +TODO think about this +<> some coinjoin tools we use today were broken +<> one allowed people to use a mix of uncompressed and compressed keys, so it was obvious which party was which. + +TODO +probably a good idea to have a debug.log where loads of information is dumped + +TODO +for the !addrs command, firstly change its name since it also includes the utxo inputs + secondly, the utxo list might be longer than can fit in an irc message, so create a + !addrsparts or something command + +TODO +code a gui where a human can see the state of the orderbook and easily choose orders to fill +code a gui that easily explains to a human how they can choose a fee for their yield-generator.py +both are important for market forces, since markets emerge from human decisions and actions + +#TODO add random delays to the orderbook stuff so there isnt such a traffic spike when a new bot joins +#two options, random delay !orderbook for ones which dont mind, !orderbook without delay for bots +# which need the orders asap + +TODO +code something that extends orderbookwatch and creates graphs + those graphs can be posted to a bitcointalk thread (like the bitstamp wall watch thread) + and could be a nice historical record and guide to pricing + +TODO +code something that analyzes the blockchain, detects coinjoin tx likely made by joinmarket + and calculates the paid fee, therefore is a guide to pricing + +TODO +the add_addr_notify() stuff doesnt work, so if theres several CoinJoinOrder's open it will start a few + threads to do the notifying, they could race condition or other multithreaded errors +i suggest to create a single thread that sorts out all the stuff + +#TODO make an ordertype where maker publishes the utxo he will use +# this is a way to auction off the use of a desirable coin, maybe a +# very newly mined coin or one which hasnt been moved for years From 7911f85a53f16e965d87da2ead5534290832f3ca Mon Sep 17 00:00:00 2001 From: Belcher Date: Wed, 21 Jan 2015 00:56:55 +0000 Subject: [PATCH 067/409] edited readme --- README.txt | 41 +++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/README.txt b/README.txt index 33a8b9b9..c0049457 100644 --- a/README.txt +++ b/README.txt @@ -1,32 +1,41 @@ +Bitcointalk thread: +https://bitcointalk.org/index.php?topic=919116.msg10096563 + + FIRST IMPLEMENTATION OF JOINMARKET you will need to know python somewhat to play around with it also get some testnet coins HOWTO try -1. create two wallet seeds string (can be just brainwallets if you're only storing testnet btc) - one seed for each maker and taker - use wallet-tool.py to output a bunch of addresses from the seeds - send testnet coins to one mixing-depth=0 receive address - seeds are taken as a command line argument +1. You will need libsodium installed + Get it here: http://doc.libsodium.org/installation/README.html + +2. Come up with a wallet seed. This is a bit like a brainwallet, it can be any string. + For real bitcoins you would probably generate it from 128 bits of entropy and encode + in a 12-word mnemonic. For testnet just use anything. -2. join irc.freenode.net #joinmarket-pit-test and run both taker.py and yield-generator.py +$ python wallet-tool.py [seed] + To print out a bunch of addresses, send some testnet coins to an address -3. when both bots join and have announced their orders, use this - command to start a coinjoining - !%fill [counterparty] [order-id] [cj-amount] [utxo] +$ python sendpayment.py -N 1 [seed] [amount-in-satoshi] [destination address] + Chooses the cheapest offer to do a 2-party coinjoin to send money to a destination address -so for example if the maker is called 'cj-maker' and you want to mix 1.9btc - !%fill cj-maker 0 190000000 5cf68d4c42132f8f0bef8573454036953ddb3ba77a3bf3797d9862b7102d65cd:1 +If you're a frugal user and don't want to pay for a coinjoin if you dont have to, use this command +$ python patientsendpayments.py -N 1 -w 2 [wallet seed] [amount in satoshi] [destination address] + Announces orders and waits to coinjoin for a maximum of 2 hours. Once that time it up cancels the + orders and pays to do a 2-party coinjoin. -all values are in satoshis, the first order has order-id 0 and it counts up -you can use !%unspent to see a printout of taker's unspent transaction outputs -and !%showob to see the orderbook +$ python gui-taker.py + Starts a local http server which you can connect to and will display the orderbook as well as some graphs -4. watch the outputs of both bots, soon enough taker.py will say it has completed + +Watch the output of your bot(s), soon enough the taker will say it has completed a transaction, maker will wait for the transaction to be seen and confirmed +If there are no orders, you could run two bots from the same machine. Be sure to use + two seperate wallet seeds though. + -theres lots that needs to be done some other notes below.. #COINJOIN PROTOCOL From eed67f3865764fcced0e2a41f05b2119ad216ba8 Mon Sep 17 00:00:00 2001 From: Belcher Date: Wed, 21 Jan 2015 01:29:56 +0000 Subject: [PATCH 068/409] fixed bug where orders were not overwritten --- taker.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/taker.py b/taker.py index e4646ab1..70525bc3 100644 --- a/taker.py +++ b/taker.py @@ -137,6 +137,8 @@ def __init__(self): + "minsize INTEGER, maxsize INTEGER, txfee INTEGER, cjfee TEXT);") def add_order(self, nick, chunks): + self.db.execute("DELETE FROM orderbook WHERE counterparty=? AND oid=?;", + (nick, chunks[1])) self.db.execute('INSERT INTO orderbook VALUES(?, ?, ?, ?, ?, ?, ?);', (nick, chunks[1], chunks[0], chunks[2], chunks[3], chunks[4], chunks[5])) @@ -171,6 +173,12 @@ def on_pubmsg(self, nick, message): return elif chunks[0] in ordername_list: self.add_order(nick, chunks) + elif chunks[0] == '%showob': + print('printing orderbook') + for o in self.db.execute('SELECT * FROM orderbook;').fetchall(): + print '(%s %s %d %d-%d %d %s)' % (o['counterparty'], o['ordertype'], o['oid'], + o['minsize'], o['maxsize'], o['txfee'], o['cjfee']) + print('done') def on_welcome(self): self.pubmsg(command_prefix + 'orderbook') @@ -269,12 +277,6 @@ def on_pubmsg(self, nick, message): self.cjtx = CoinJoinTX(self, int(amt), {cp: oid}, [utxos[0]], self.wallet.get_receive_addr(mixing_depth=1), self.wallet.get_change_addr(mixing_depth=0), my_tx_fee) - if chunks[0] == '%showob': - print('printing orderbook') - for o in self.db.execute('SELECT * FROM orderbook;').fetchall(): - print '(%s %s %d %d-%d %d %s)' % (o['counterparty'], o['ordertype'], o['oid'], - o['minsize'], o['maxsize'], o['txfee'], o['cjfee']) - print('done') elif chunks[0] == '%unspent': from pprint import pprint pprint(self.wallet.unspent) From d2c4f55fdd7eb87fb09521b11f0b5f3b8a401b39 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Wed, 21 Jan 2015 20:12:27 +0200 Subject: [PATCH 069/409] add addr notifier for taker, change test taker to work, other bug fixes --- taker.py | 21 ++++++++++++++++++--- wallet-tool.py | 4 +++- yield-generator.py | 13 ++++++------- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/taker.py b/taker.py index 62e4e83d..550b4a6c 100644 --- a/taker.py +++ b/taker.py @@ -125,8 +125,19 @@ def add_signature(self, sigb64): print btc.serialize(self.latest_tx) ret = btc.blockr_pushtx(btc.serialize(self.latest_tx), get_network()) debug('pushed tx ' + str(ret)) + #watch thread to remove used utxos + add_addr_notify(self.my_change_addr, self.unconfirm_callback, self.confirm_callback) + if self.finishcallback != None: self.finishcallback() + + def unconfirm_callback(self, balance): + removed_utxos = self.taker.wallet.remove_old_utxos(self.latest_tx) + debug('saw tx on network, removed_utxos=\n' + pprint.pformat(removed_utxos)) + + def confirm_callback(self, confirmations, txid, balance): + added_utxos = self.taker.wallet.add_new_utxos(self.latest_tx, txid) + debug('tx in a block, added_utxos=\n' + pprint.pformat(added_utxos)) class OrderbookWatch(irclib.IRCClient): def __init__(self): @@ -252,7 +263,7 @@ def on_privmsg(self, nick, message): class TestTaker(Taker): def __init__(self, wallet,keyfile): Taker.__init__(self,keyfile) - self.wallet = wallet + self.wallet = wallet def on_pubmsg(self, nick, message): Taker.on_pubmsg(self, nick, message) @@ -266,14 +277,18 @@ def on_pubmsg(self, nick, message): cp = chunks[1] oid = chunks[2] amt = chunks[3] - utxo_list = self.wallet.get_mix_utxo_list()[0] #only spend from the unmixed funds + #this testing command implements a very dumb algorithm. + #just take 1 utxo from anywhere and output it to a level 1 + #change address. + utxo_dict = self.wallet.get_mix_utxo_list() + utxo_list = [x for v in utxo_dict.itervalues() for x in v] unspent = [{'utxo': utxo, 'value': self.wallet.unspent[utxo]['value']} \ for utxo in utxo_list] inputs = btc.select(unspent, amt) utxos = [i['utxo'] for i in inputs] print 'making cjtx' self.cjtx = CoinJoinTX(self, int(amt), {cp: oid}, - [utxos[0]], self.wallet.get_receive_addr(mixing_depth=1), + utxos, self.wallet.get_receive_addr(mixing_depth=1), self.wallet.get_change_addr(mixing_depth=0), my_tx_fee) if chunks[0] == '%showob': diff --git a/wallet-tool.py b/wallet-tool.py index a37b1bd6..cf60809b 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -19,7 +19,9 @@ description='Does useful little lasts involving your bip32 wallet. The' + ' method is one of the following: Display- shows all addresses and balances' + '. Combine- combines all utxos into one output for each mixing level. Used for' - + ' testing and is detrimental to privacy.') + + ' testing and is detrimental to privacy.' + + ' reset - send all utxos back to first receiving address at zeroth mixing level.' + + ' Also only for testing and destroys privacy.') parser.add_option('-p', '--privkey', action='store_true', dest='showprivkey', help='print private key along with address, default false') parser.add_option('-m', '--maxmixdepth', action='store', type='int', dest='maxmixdepth', diff --git a/yield-generator.py b/yield-generator.py index 7cbb15cb..a06e55c8 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -64,7 +64,6 @@ def oid_to_order(self, oid, amount): break mixdepth = (mixdepth - 1) % self.wallet.max_mix_depth #mixdepth is the chosen depth we'll be spending from - mix_utxo_list = self.wallet.get_mix_utxo_list() unspent = [{'utxo': utxo, 'value': self.wallet.unspent[utxo]['value']} for utxo in mix_utxo_list[mixdepth]] @@ -75,15 +74,15 @@ def oid_to_order(self, oid, amount): def on_tx_unconfirmed(self, cjorder, balance, removed_utxos): #if the balance of the highest-balance mixing depth change then reannounce it - oldorder = self.orderlist[0] + oldorder = self.orderlist[0] if len(self.orderlist) > 0 else None neworders = self.create_my_orders() if len(neworders) == 0: return ([0], []) #cancel old order - elif oldorder['maxsize'] == neworders[0]['maxsize']: - return ([], []) #change nothing - else: - #announce new order, replacing the old order - return ([], [neworders[0]]) + if oldorder: #oldorder may not exist when this is called from on_tx_confirmed + if oldorder['maxsize'] == neworders[0]['maxsize']: + return ([], []) #change nothing + #announce new order, replacing the old order + return ([], [neworders[0]]) def on_tx_confirmed(self, cjorder, confirmations, txid, balance, added_utxos): return self.on_tx_unconfirmed(None, None, None) From f29995ba95234b505450ea4806cc50f36aea2685 Mon Sep 17 00:00:00 2001 From: Belcher Date: Wed, 21 Jan 2015 20:37:37 +0000 Subject: [PATCH 070/409] updated todos --- TODO | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/TODO b/TODO index 47bfb6ff..5c594fa2 100644 --- a/TODO +++ b/TODO @@ -27,6 +27,13 @@ TODO have the taker enforce this, look up the txhash of the maker's utxo and make sure it is already in a block +TODO +remove the badly done chunking code in the coinjoin layer, such as !txpart !tx + since chunking should be done in the messaging layer + +TODO +remove the keyfile feature totally, there is no need for persistent keys +just generate a key at startup, TODO implement rate limiting for irc.privmsg to stop the bot being killed due to flood i suggest creating a thread that only dispatches/writes to the irc socket From 459ce6840c22f936f73d3ee31fa7ca89a6cf10e4 Mon Sep 17 00:00:00 2001 From: Belcher Date: Wed, 21 Jan 2015 21:56:52 +0000 Subject: [PATCH 071/409] made a general function for some repeated code --- TODO | 3 +++ common.py | 10 ++++++++++ patientsendpayment.py | 14 ++------------ sendpayment.py | 8 +------- yield-generator.py | 7 ++----- 5 files changed, 18 insertions(+), 24 deletions(-) diff --git a/TODO b/TODO index 5c594fa2..ea905a39 100644 --- a/TODO +++ b/TODO @@ -38,6 +38,9 @@ just generate a key at startup, TODO implement rate limiting for irc.privmsg to stop the bot being killed due to flood i suggest creating a thread that only dispatches/writes to the irc socket +TODO have an option for sendpayment.py take advantage of the small orders posted +by patientsendpayment.py + TODO sort out the nick = nick + '_' stuff in irclib its not a good way of doing it diff --git a/common.py b/common.py index 2683df29..81d5fc4a 100644 --- a/common.py +++ b/common.py @@ -126,6 +126,15 @@ def get_mix_utxo_list(self): mix_utxo_list[mixdepth].append(utxo) return mix_utxo_list + def select_utxos(self, mixdepth, amount): + utxo_list = self.get_mix_utxo_list()[mixdepth] + unspent = [{'utxo': utxo, 'value': self.unspent[utxo]['value']} + for utxo in utxo_list] + inputs = btc.select(unspent, amount) + debug('for mixdepth=' + str(mixdepth) + ' amount=' + str(amount) + ' selected:') + pprint.pprint(inputs) + return [i['utxo'] for i in inputs] + def sync_wallet(self, gaplimit=6): debug('synchronizing wallet') self.download_wallet_history(gaplimit) @@ -277,6 +286,7 @@ def run(self): NotifyThread(address, unconfirmfun, confirmfun, unconfirmtimeout, unconfirmtimeoutfun, confirmtimeout, confirmtimeoutfun).start() + def calc_cj_fee(ordertype, cjfee, cj_amount): real_cjfee = None if ordertype == 'absorder': diff --git a/patientsendpayment.py b/patientsendpayment.py index f705ec6e..6df94581 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -29,13 +29,7 @@ def run(self): orders = sendpayment.choose_order(self.tmaker.db, self.tmaker.amount, self.tmaker.makercount) print 'chosen orders to fill ' + str(orders) - utxo_list = self.tmaker.wallet.get_mix_utxo_list()[self.tmaker.mixdepth] - unspent = [{'utxo': utxo, 'value': self.tmaker.wallet.unspent[utxo]['value']} - for utxo in utxo_list] - inputs = btc.select(unspent, self.tmaker.amount) - utxos = [i['utxo'] for i in inputs] - print 'will spend ' + str(inputs) - + utxos = self.taker.wallet.select_utxos(self.tmaker.mixdepth, self.tmaker.amount) self.tmaker.cjtx = taker.CoinJoinTX(self.tmaker, self.tmaker.amount, orders, utxos, self.tmaker.destaddr, self.tmaker.wallet.get_change_addr(self.tmaker.mixdepth), @@ -80,11 +74,7 @@ def oid_to_order(self, oid, amount): #if an order arrives and before it finishes another order arrives # its possible this bot will end up paying to the destaddr more than it # intended - utxo_list = self.wallet.get_mix_utxo_list()[self.mixdepth] - unspent = [{'utxo': utxo, 'value': self.wallet.unspent[utxo]['value']} - for utxo in utxo_list] - inputs = btc.select(unspent, amount) - utxos = [i['utxo'] for i in inputs] + utxos = self.wallet.select_utxos(self.mixdepth, amount) return utxos, self.destaddr, self.wallet.get_change_addr(self.mixdepth) def on_tx_unconfirmed(self, cjorder, balance, removed_utxos): diff --git a/sendpayment.py b/sendpayment.py index 41d972b6..66689fc5 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -82,13 +82,7 @@ def run(self): orders = choose_order(self.taker.db, self.taker.amount, self.taker.makercount) print 'chosen orders to fill ' + str(orders) - utxo_list = self.taker.wallet.get_mix_utxo_list()[self.taker.mixdepth] - unspent = [{'utxo': utxo, 'value': self.taker.wallet.unspent[utxo]['value']} - for utxo in utxo_list] - inputs = btc.select(unspent, self.taker.amount) - utxos = [i['utxo'] for i in inputs] - print 'will spend ' + str(inputs) - + utxos = self.taker.wallet.select_utxos(self.taker.mixdepth, self.taker.amount) self.taker.cjtx = takermodule.CoinJoinTX(self.taker, self.taker.amount, orders, utxos, self.taker.destaddr, self.taker.wallet.get_change_addr(self.taker.mixdepth), self.taker.txfee, diff --git a/yield-generator.py b/yield-generator.py index 1a832092..9da20da0 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -66,13 +66,10 @@ def oid_to_order(self, oid, amount): mixdepth = (mixdepth - 1) % self.wallet.max_mix_depth #mixdepth is the chosen depth we'll be spending from - mix_utxo_list = self.wallet.get_mix_utxo_list() - unspent = [{'utxo': utxo, 'value': self.wallet.unspent[utxo]['value']} - for utxo in mix_utxo_list[mixdepth]] - inputs = btc.select(unspent, amount) + utxos = self.wallet.select_utxos(mixdepth, amount) cj_addr = self.wallet.get_receive_addr((mixdepth + 1) % self.wallet.max_mix_depth) change_addr = self.wallet.get_change_addr(mixdepth) - return [i['utxo'] for i in inputs], cj_addr, change_addr + return utxos, cj_addr, change_addr def on_tx_unconfirmed(self, cjorder, balance, removed_utxos): #if the balance of the highest-balance mixing depth change then reannounce it From 33d4d90ff774ad40c9799da35b3bf14f12fbda67 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Thu, 22 Jan 2015 19:15:15 +0200 Subject: [PATCH 072/409] move callback to testtaker, corrections to end encryption --- irclib.py | 4 ++++ taker.py | 29 +++++++++++------------------ 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/irclib.py b/irclib.py index a34abb61..052e3e3a 100644 --- a/irclib.py +++ b/irclib.py @@ -83,6 +83,10 @@ def end_encryption(self, nick): self.cp_pubkeys[nick]=None self.enc_boxes[nick]=None + def end_all_encryption(self): + for k,v in self.encrypting.iteritems(): + if v: self.end_encryption(k) + def encrypt_encode(self,msg,nick): if not (nick in self.encrypting.keys()) or not (nick in self.enc_boxes.keys()): raise Exception("Encryption is not switched on.") diff --git a/taker.py b/taker.py index 5f7b4968..fe8da66d 100644 --- a/taker.py +++ b/taker.py @@ -122,23 +122,14 @@ def add_signature(self, sigb64): if not tx_signed: return debug('the entire tx is signed, ready to pushtx()') + #end encryption channel with all counterparties + self.taker.end_all_encryption() print btc.serialize(self.latest_tx) ret = btc.blockr_pushtx(btc.serialize(self.latest_tx), get_network()) debug('pushed tx ' + str(ret)) - #watch thread to remove used utxos - add_addr_notify(self.my_change_addr, self.unconfirm_callback, self.confirm_callback) - if self.finishcallback != None: self.finishcallback() - def unconfirm_callback(self, balance): - removed_utxos = self.taker.wallet.remove_old_utxos(self.latest_tx) - debug('saw tx on network, removed_utxos=\n' + pprint.pformat(removed_utxos)) - - def confirm_callback(self, confirmations, txid, balance): - added_utxos = self.taker.wallet.add_new_utxos(self.latest_tx, txid) - debug('tx in a block, added_utxos=\n' + pprint.pformat(added_utxos)) - class OrderbookWatch(irclib.IRCClient): def __init__(self): con = sqlite3.connect(":memory:", check_same_thread=False) @@ -257,10 +248,6 @@ def on_privmsg(self, nick, message): continue self.cjtx.recv_txio(nick, utxo_list, cj_pub, change_addr) elif chunks[0] == 'sig': - #whether the signature is correct or not, - #we treat this as the last message for this tx from - #this nick, and therefore switch off encryption - self.end_encryption(nick) sig = chunks[1] self.cjtx.add_signature(sig) @@ -270,6 +257,12 @@ class TestTaker(Taker): def __init__(self, wallet,keyfile): Taker.__init__(self,keyfile) self.wallet = wallet + + def finish_callback(self): + removed_utxos = self.wallet.remove_old_utxos(self.cjtx.latest_tx) + added_utxos = self.wallet.add_new_utxos(self.cjtx.latest_tx, btc.txhash(btc.serialize(self.cjtx.latest_tx))) + debug('tx published, added_utxos=\n' + pprint.pformat(added_utxos)) + debug('removed_utxos=\n' + pprint.pformat(removed_utxos)) def on_pubmsg(self, nick, message): Taker.on_pubmsg(self, nick, message) @@ -295,7 +288,7 @@ def on_pubmsg(self, nick, message): print 'making cjtx' self.cjtx = CoinJoinTX(self, int(amt), {cp: oid}, utxos, self.wallet.get_receive_addr(mixing_depth=1), - self.wallet.get_change_addr(mixing_depth=0), my_tx_fee) + self.wallet.get_change_addr(mixing_depth=0), my_tx_fee,self.finish_callback) elif chunks[0] == '%unspent': from pprint import pprint pprint(self.wallet.unspent) @@ -308,7 +301,7 @@ def on_pubmsg(self, nick, message): print 'making cjtx' self.cjtx = CoinJoinTX(self, int(amount), {counterparty: oid}, [my_utxo], self.wallet.get_receive_addr(mixing_depth=1), - self.wallet.get_change_addr(mixing_depth=0), my_tx_fee) + self.wallet.get_change_addr(mixing_depth=0), my_tx_fee,self.finish_callback) elif chunks[0] == '%2fill': #!2fill [amount] [utxo] [counterparty1] [oid1] [counterparty2] [oid2] amount = int(chunks[1]) @@ -320,7 +313,7 @@ def on_pubmsg(self, nick, message): print 'creating cjtx' self.cjtx = CoinJoinTX(self, amount, {cp1: oid1, cp2: oid2}, [my_utxo], self.wallet.get_receive_addr(mixing_depth=1), - self.wallet.get_change_addr(mixing_depth=0), my_tx_fee) + self.wallet.get_change_addr(mixing_depth=0), my_tx_fee,self.finish_callback) def main(): import sys From 7a21a074e5bdf9f66f24f830ac530fe1f008dca7 Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 23 Jan 2015 22:20:07 +0000 Subject: [PATCH 073/409] added some more debug prints to help fix a bug --- common.py | 2 ++ maker.py | 1 + 2 files changed, 3 insertions(+) diff --git a/common.py b/common.py index 9cd6697b..f8f3ab68 100644 --- a/common.py +++ b/common.py @@ -107,6 +107,7 @@ def remove_old_utxos(self, tx): continue removed_utxos[utxo] = self.unspent[utxo] del self.unspent[utxo] + debug('removed utxos, wallet now is \n' + pprint.pformat(self.get_mix_utxo_list())) return removed_utxos def add_new_utxos(self, tx, txid): @@ -119,6 +120,7 @@ def add_new_utxos(self, tx, txid): utxo = txid + ':' + str(index) added_utxos[utxo] = addrdict self.unspent[utxo] = addrdict + debug('added utxos, wallet now is \n' + pprint.pformat(self.get_mix_utxo_list())) return added_utxos def get_mix_utxo_list(self): diff --git a/maker.py b/maker.py index 2afb746e..509bcd3d 100644 --- a/maker.py +++ b/maker.py @@ -242,6 +242,7 @@ def on_leave(self, nick): self.active_orders[nick] = None def modify_orders(self, to_cancel, to_announce): + debug('modifying orders. to_cancel=' + str(to_cancel) + '\nto_announce=' + str(to_announce)) for oid in to_cancel: order = [o for o in self.orderlist if o['oid'] == oid][0] self.orderlist.remove(order) From dc3c1ae63fe0dc7b95866aebb72f843ab1f47db0 Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 23 Jan 2015 22:48:42 +0000 Subject: [PATCH 074/409] fixed payment senders to take into account fees --- TODO | 4 ++++ common.py | 16 ++++++++++++++++ patientsendpayment.py | 8 +++++--- sendpayment.py | 24 +++++------------------- 4 files changed, 30 insertions(+), 22 deletions(-) diff --git a/TODO b/TODO index ea905a39..01ea919a 100644 --- a/TODO +++ b/TODO @@ -44,6 +44,10 @@ by patientsendpayment.py TODO sort out the nick = nick + '_' stuff in irclib its not a good way of doing it +TODO +robustness in the taker code +for instance if the maker being joined with quits halfway through + #TODO encrypt messages between taker and maker, to stop trivial server eavesdropping # but that wont stop mitm # after chats on irc, easiest is to do Trust On First Use, maker sends a pubkey over diff --git a/common.py b/common.py index f8f3ab68..ce26ca0c 100644 --- a/common.py +++ b/common.py @@ -313,4 +313,20 @@ def calc_total_input_value(utxos): input_sum += int(btc.deserialize(tx)['outs'][int(utxo[65:])]['value']) return input_sum +def choose_order(db, cj_amount, n): + + sqlorders = db.execute('SELECT * FROM orderbook;').fetchall() + orders = [(o['counterparty'], o['oid'], calc_cj_fee(o['ordertype'], o['cjfee'], cj_amount)) + for o in sqlorders if cj_amount >= o['minsize'] or cj_amount <= o['maxsize']] + orders = sorted(orders, key=lambda k: k[2]) + debug('considered orders = ' + str(orders)) + total_cj_fee = 0 + chosen_orders = [] + for i in range(n): + chosen_order = orders[0] #choose the cheapest, later this will be chosen differently + orders = [o for o in orders if o[0] != chosen_order[0]] + chosen_orders.append(chosen_order) + total_cj_fee += chosen_order[2] + chosen_orders = [o[:2] for o in chosen_orders] + return dict(chosen_orders), total_cj_fee diff --git a/patientsendpayment.py b/patientsendpayment.py index 6df94581..86851f03 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -26,10 +26,12 @@ def run(self): print 'giving up waiting' #cancel the remaining order self.tmaker.modify_orders([0], []) - orders = sendpayment.choose_order(self.tmaker.db, self.tmaker.amount, self.tmaker.makercount) - print 'chosen orders to fill ' + str(orders) + orders, total_cj_fee = choose_order(self.tmaker.db, self.tmaker.amount, self.tmaker.makercount) + print 'chosen orders to fill ' + str(orders) + ' totalcjfee=' + str(total_cj_fee) + total_amount = self.tmaker.amount + total_cj_fee + self.tmaker.txfee + print 'total amount spent = ' + str(total_amount) - utxos = self.taker.wallet.select_utxos(self.tmaker.mixdepth, self.tmaker.amount) + utxos = self.taker.wallet.select_utxos(self.tmaker.mixdepth, total_amount) self.tmaker.cjtx = taker.CoinJoinTX(self.tmaker, self.tmaker.amount, orders, utxos, self.tmaker.destaddr, self.tmaker.wallet.get_change_addr(self.tmaker.mixdepth), diff --git a/sendpayment.py b/sendpayment.py index 66689fc5..346f4556 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -7,22 +7,6 @@ from optparse import OptionParser import threading - -def choose_order(db, cj_amount, n): - - sqlorders = db.execute('SELECT * FROM orderbook;').fetchall() - orders = [(o['counterparty'], o['oid'], calc_cj_fee(o['ordertype'], o['cjfee'], cj_amount)) - for o in sqlorders if cj_amount >= o['minsize'] or cj_amount <= o['maxsize']] - orders = sorted(orders, key=lambda k: k[2]) - print 'considered orders = ' + str(orders) - chosen_orders = [] - for i in range(n): - chosen_order = orders[0] #choose the cheapest, later this will be chosen differently - orders = [o for o in orders if o[0] != chosen_order[0]] - chosen_orders.append(chosen_order) - chosen_orders = [o[:2] for o in chosen_orders] - return dict(chosen_orders) - def choose_sweep_order(db, my_total_input, my_tx_fee): ''' choose an order given that we want to be left with no change @@ -79,10 +63,12 @@ def run(self): self.taker.shutdown() return - orders = choose_order(self.taker.db, self.taker.amount, self.taker.makercount) - print 'chosen orders to fill ' + str(orders) + orders, total_cj_fee = choose_order(self.taker.db, self.taker.amount, self.taker.makercount) + print 'chosen orders to fill ' + str(orders) + ' totalcjfee=' + str(total_cj_fee) + total_amount = self.taker.amount + total_cj_fee + self.taker.txfee + print 'total amount spent = ' + str(total_amount) - utxos = self.taker.wallet.select_utxos(self.taker.mixdepth, self.taker.amount) + utxos = self.taker.wallet.select_utxos(self.taker.mixdepth, total_amount) self.taker.cjtx = takermodule.CoinJoinTX(self.taker, self.taker.amount, orders, utxos, self.taker.destaddr, self.taker.wallet.get_change_addr(self.taker.mixdepth), self.taker.txfee, From d4abb61048cda231ab3ab7edc9bf52c58fc5af7f Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Sat, 24 Jan 2015 11:46:02 +0200 Subject: [PATCH 075/409] rationalised encryption testing remove enc-wrapper-tester --- enc_wrapper.py | 41 +++++++++++++++++++++++++++++++++++++++++ enc_wrapper_tester.py | 38 -------------------------------------- 2 files changed, 41 insertions(+), 38 deletions(-) delete mode 100644 enc_wrapper_tester.py diff --git a/enc_wrapper.py b/enc_wrapper.py index 2cfbb4a0..7c54999d 100644 --- a/enc_wrapper.py +++ b/enc_wrapper.py @@ -59,3 +59,44 @@ def as_init_encryption(kp, c_pk): #TODO: Sign, verify. At the moment we are using #bitcoin signatures so it isn't necessary. + +def test_case(case_name, alice_box, bob_box, ab_message, ba_message, num_iterations=1): + for i in range(num_iterations): + otw_amsg = alice_box.encrypt(ab_message) + bob_ptext = bob_box.decrypt(otw_amsg) + assert bob_ptext == ab_message, "Encryption test: FAILED. Alice sent: "\ + +ab_message+" , Bob received: " + bob_ptext + + otw_bmsg = bob_box.encrypt(ba_message) + alice_ptext = alice_box.decrypt(otw_bmsg) + assert alice_ptext == ba_message, "Encryption test: FAILED. Bob sent: "\ + +ba_message+" , Alice received: " + alice_ptext + + print "Encryption test PASSED for case: "+case_name + +#to test the encryption functionality +if __name__ == "__main__": + alice_kp = init_keypair(fname='alice1.txt') + bob_kp = init_keypair(fname='bob1.txt') + + #this is the DH key exchange part + bob_otwpk = get_pubkey(bob_kp,True) + alice_otwpk = get_pubkey(alice_kp,True) + + bob_pk = init_pubkey(bob_otwpk) + alice_box = as_init_encryption(alice_kp,bob_pk) + alice_pk = init_pubkey(alice_otwpk) + bob_box = as_init_encryption(bob_kp,alice_pk) + + #now Alice and Bob can use their 'box' + #constructs (both of which utilise the same + #shared secret) to perform encryption/decryption + + test_case("short ascii", alice_box, bob_box,"Attack at dawn","Not tonight Josephine!",5) + + import base64, string, random + longb641 = base64.b64encode(''.join(random.choice(string.ascii_letters) for x in range(5000))) + longb642 = base64.b64encode(''.join(random.choice(string.ascii_letters) for x in range(5000))) + test_case("long b64", alice_box, bob_box, longb641, longb642,5) + + print "All test cases passed - encryption and decryption should work correctly." \ No newline at end of file diff --git a/enc_wrapper_tester.py b/enc_wrapper_tester.py deleted file mode 100644 index 2115e337..00000000 --- a/enc_wrapper_tester.py +++ /dev/null @@ -1,38 +0,0 @@ -import enc_wrapper as e -import binascii - -alice_kp = e.init_keypair(fname='alice1.txt') -bob_kp = e.init_keypair(fname='bob1.txt') - -#this is the DH key exchange part -bob_otwpk = e.get_pubkey(bob_kp,True) -print "sending pubkey from bob to alice: "+bob_otwpk -alice_otwpk = e.get_pubkey(alice_kp,True) -print "sending pubkey from bob to alice: "+alice_otwpk - -bob_pk = e.init_pubkey(bob_otwpk) -alice_box = e.as_init_encryption(alice_kp,bob_pk) -alice_pk = e.init_pubkey(alice_otwpk) -bob_box = e.as_init_encryption(bob_kp,alice_pk) - -#now Alice and Bob can use their 'box' -#constructs (both of which utilise the same -#shared secret) to perform encryption/decryption -for i in range(8): - alice_message = 'Attack at dawn ! \n\n x'+str(i) - - otw_amsg = alice_box.encrypt(alice_message) - print "Sending from alice to bob: " + otw_amsg - - bob_ptext = bob_box.decrypt(otw_amsg) - print "Bob received: " + bob_ptext - - bob_message = 'Not tonight Josephine.' + str(i) * 45 - otw_bmsg = bob_box.encrypt(bob_message) - print "Sending from bob to alice: " + otw_bmsg - - alice_ptext = alice_box.decrypt(otw_bmsg) - print "Alice received: " + alice_ptext - - - From cebfb60def4ecce9a710d0bec0bdc3b857d73a21 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Sat, 24 Jan 2015 13:37:07 +0200 Subject: [PATCH 076/409] remove encryption keyfile persistence --- TODO | 14 +------------- irclib.py | 7 ++----- maker.py | 7 +++---- patientsendpayment.py | 9 ++++----- sendpayment.py | 7 +++---- taker.py | 13 ++++++------- yield-generator.py | 9 ++++----- 7 files changed, 23 insertions(+), 43 deletions(-) diff --git a/TODO b/TODO index 01ea919a..a90a5e40 100644 --- a/TODO +++ b/TODO @@ -29,11 +29,7 @@ have the taker enforce this, look up the txhash of the maker's utxo and make sur TODO remove the badly done chunking code in the coinjoin layer, such as !txpart !tx - since chunking should be done in the messaging layer - -TODO -remove the keyfile feature totally, there is no need for persistent keys -just generate a key at startup, + since chunking should be done in the messaging layer TODO implement rate limiting for irc.privmsg to stop the bot being killed due to flood i suggest creating a thread that only dispatches/writes to the irc socket @@ -48,14 +44,6 @@ TODO robustness in the taker code for instance if the maker being joined with quits halfway through -#TODO encrypt messages between taker and maker, to stop trivial server eavesdropping -# but that wont stop mitm -# after chats on irc, easiest is to do Trust On First Use, maker sends a pubkey over -# TOFU requires a human to verify each first time, might not be practical -# skip the human verification, it will probably be okay -# make the irc nick be a hash of the pubkey -# also theres some algorithm for detecting mitm - #TODO implement something against dust # e.g. where the change address ends up having an output of value 1000 satoshis diff --git a/irclib.py b/irclib.py index 052e3e3a..c43e2bbd 100644 --- a/irclib.py +++ b/irclib.py @@ -58,11 +58,8 @@ def on_connect(self): pass ############################### #Encryption code ############################### - def init_encryption(self,fname): - if not os.path.isfile(fname): - self.enc_kp = enc_wrapper.init_keypair(fname) - else: - self.enc_kp = enc_wrapper.libnacl.utils.load_key(fname) + def init_encryption(self): + self.enc_kp = enc_wrapper.init_keypair() def start_encryption(self,nick,c_pk_hex): '''sets encryption mode on diff --git a/maker.py b/maker.py index 509bcd3d..0a6bac25 100644 --- a/maker.py +++ b/maker.py @@ -145,8 +145,8 @@ class CJMakerOrderError(StandardError): pass class Maker(irclib.IRCClient): - def __init__(self, wallet, keyfile): - self.init_encryption(keyfile) + def __init__(self, wallet): + self.init_encryption() self.active_orders = {} self.wallet = wallet self.nextoid = -1 @@ -343,12 +343,11 @@ def main(): nickname = 'cj-maker-' + btc.sha256(gethostname())[:6] import sys seed = sys.argv[1] #btc.sha256('dont use brainwallets except for holding testnet coins') - keyfile = sys.argv[2] wallet = Wallet(seed,max_mix_depth=5) wallet.sync_wallet() - maker = Maker(wallet,keyfile) + maker = Maker(wallet) print 'connecting to irc' try: maker.run(HOST, PORT, nickname, CHANNEL) diff --git a/patientsendpayment.py b/patientsendpayment.py index 86851f03..6d6bb24b 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -38,7 +38,7 @@ def run(self): self.tmaker.txfee, self.finishcallback) class PatientSendPayment(maker.Maker, taker.Taker): - def __init__(self, wallet, keyfile, destaddr, amount, makercount, txfee, cjfee, + def __init__(self, wallet, destaddr, amount, makercount, txfee, cjfee, waittime, mixdepth): self.destaddr = destaddr self.amount = amount @@ -47,8 +47,8 @@ def __init__(self, wallet, keyfile, destaddr, amount, makercount, txfee, cjfee, self.cjfee = cjfee self.waittime = waittime self.mixdepth = mixdepth - maker.Maker.__init__(self, wallet, keyfile) - taker.Taker.__init__(self, keyfile) + maker.Maker.__init__(self, wallet) + taker.Taker.__init__(self) def on_privmsg(self, nick, message): maker.Maker.on_privmsg(self, nick, message) @@ -138,7 +138,6 @@ def main(): wallet = Wallet(seed) wallet.sync_wallet() - keyfile = 'keyfile-' + str(seed) + '.txt' utxo_list = wallet.get_mix_utxo_list()[options.mixdepth] available_balance = 0 @@ -152,7 +151,7 @@ def main(): nickname = 'ppayer-' + btc.sha256(gethostname())[:6] print 'starting irc' - bot = PatientSendPayment(wallet, keyfile, destaddr, amount, options.makercount, + bot = PatientSendPayment(wallet, destaddr, amount, options.makercount, options.txfee, options.cjfee, waittime, options.mixdepth) try: bot.run(HOST, PORT, nickname, CHANNEL) diff --git a/sendpayment.py b/sendpayment.py index 346f4556..ac3d93c4 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -82,8 +82,8 @@ def run(self): ''' class SendPayment(takermodule.Taker): - def __init__(self, wallet, keyfile, destaddr, amount, makercount, txfee, waittime, mixdepth): - takermodule.Taker.__init__(self, keyfile) + def __init__(self, wallet, destaddr, amount, makercount, txfee, waittime, mixdepth): + takermodule.Taker.__init__(self) self.wallet = wallet self.destaddr = destaddr self.amount = amount @@ -122,10 +122,9 @@ def main(): wallet = Wallet(seed) wallet.sync_wallet() - keyfile = 'keyfile-' + str(seed) + '.txt' print 'starting irc' - taker = SendPayment(wallet, keyfile, destaddr, amount, options.makercount, options.txfee, + taker = SendPayment(wallet, destaddr, amount, options.makercount, options.txfee, options.waittime, options.mixdepth) try: taker.run(HOST, PORT, nickname, CHANNEL) diff --git a/taker.py b/taker.py index fe8da66d..f70b4da0 100644 --- a/taker.py +++ b/taker.py @@ -201,14 +201,14 @@ def on_disconnect(self): #assume this only has one open cj tx at a time class Taker(OrderbookWatch): - def __init__(self,keyfile): + def __init__(self): OrderbookWatch.__init__(self) self.cjtx = None self.maker_pks = {} #TODO have a list of maker's nick we're coinjoining with, so # that some other guy doesnt send you confusing stuff #maybe a start_cj_tx() method is needed - self.init_encryption(keyfile) + self.init_encryption() def auth_counterparty(self,nick,btc_sig,cj_pub): '''Validate the counterpartys claim to own the btc @@ -254,8 +254,8 @@ def on_privmsg(self, nick, message): my_tx_fee = 10000 class TestTaker(Taker): - def __init__(self, wallet,keyfile): - Taker.__init__(self,keyfile) + def __init__(self, wallet): + Taker.__init__(self) self.wallet = wallet def finish_callback(self): @@ -318,15 +318,14 @@ def on_pubmsg(self, nick, message): def main(): import sys seed = sys.argv[1] #btc.sha256('your brainwallet goes here') - keyfile = sys.argv[2] from socket import gethostname - nickname = 'taker-' + sys.argv[2][:3] + btc.sha256(gethostname())[:6] + nickname = 'taker-' + btc.sha256(gethostname())[:6] wallet = Wallet(seed, max_mix_depth=5) wallet.sync_wallet() print 'starting irc' - taker = TestTaker(wallet,keyfile) + taker = TestTaker(wallet) try: taker.run(HOST, PORT, nickname, CHANNEL) finally: diff --git a/yield-generator.py b/yield-generator.py index 37c046d1..886a19ad 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -26,8 +26,8 @@ #spent from utxos that try to make the highest balance even higher # so try to keep coins concentrated in one mixing depth class YieldGenerator(Maker): - def __init__(self, wallet,keyfile): - Maker.__init__(self, wallet,keyfile) + def __init__(self, wallet): + Maker.__init__(self, wallet) def on_connect(self): if len(nickserv_password) > 0: @@ -90,9 +90,8 @@ def main(): wallet = Wallet(seed, max_mix_depth = mix_levels) wallet.sync_wallet() - keyfile = sys.argv[2] - nickname = 'yigen-' + sys.argv[2][:3] + btc.sha256(gethostname())[:6] - maker = YieldGenerator(wallet, keyfile) + nickname = 'yigen-' + btc.sha256(gethostname())[:6] + maker = YieldGenerator(wallet) print 'connecting to irc' try: maker.run(HOST, PORT, nickname, CHANNEL) From 00b758042b700535743925a7d8f850d35f7905da Mon Sep 17 00:00:00 2001 From: Belcher Date: Sat, 24 Jan 2015 17:21:22 +0000 Subject: [PATCH 077/409] fixed bug in choose_order() --- common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common.py b/common.py index ce26ca0c..9a861d8f 100644 --- a/common.py +++ b/common.py @@ -317,7 +317,7 @@ def choose_order(db, cj_amount, n): sqlorders = db.execute('SELECT * FROM orderbook;').fetchall() orders = [(o['counterparty'], o['oid'], calc_cj_fee(o['ordertype'], o['cjfee'], cj_amount)) - for o in sqlorders if cj_amount >= o['minsize'] or cj_amount <= o['maxsize']] + for o in sqlorders if cj_amount >= o['minsize'] and cj_amount <= o['maxsize']] orders = sorted(orders, key=lambda k: k[2]) debug('considered orders = ' + str(orders)) total_cj_fee = 0 From a2a843805884cd8b04f214ca5f8c748056bd7af6 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Sat, 24 Jan 2015 19:37:50 +0200 Subject: [PATCH 078/409] remove keyfiles --- enc_wrapper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/enc_wrapper.py b/enc_wrapper.py index 7c54999d..36531e5d 100644 --- a/enc_wrapper.py +++ b/enc_wrapper.py @@ -76,8 +76,8 @@ def test_case(case_name, alice_box, bob_box, ab_message, ba_message, num_iterati #to test the encryption functionality if __name__ == "__main__": - alice_kp = init_keypair(fname='alice1.txt') - bob_kp = init_keypair(fname='bob1.txt') + alice_kp = init_keypair() + bob_kp = init_keypair() #this is the DH key exchange part bob_otwpk = get_pubkey(bob_kp,True) From 402e0ec30df10415e7971901b034bb25e62e1171 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sat, 24 Jan 2015 18:04:04 +0000 Subject: [PATCH 079/409] fixed bug involving chunking --- maker.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/maker.py b/maker.py index 0a6bac25..1f18d5bd 100644 --- a/maker.py +++ b/maker.py @@ -84,16 +84,7 @@ def recv_tx(self, nick, b64txpart): add_addr_notify(self.change_addr, self.unconfirm_callback, self.confirm_callback) debug('sending sigs ' + str(sigs)) - #TODO make this a function in irclib.py - sigline = '' - for sig in sigs: - prev_sigline = sigline - sigline = sigline + command_prefix + 'sig ' + sig - if len(sigline) > MAX_PRIVMSG_LEN: - self.maker.privmsg(nick, prev_sigline) - sigline = command_prefix + 'sig ' + sig - if len(sigline) > 0: - self.maker.privmsg(nick, sigline) + self.maker.privmsg(nick, ''.join([command_prefix + 'sig ' + s for s in sigs])) #once signature is sent, close encrypted channel to this taker. self.maker.end_encryption(nick) From 7542500d1892a661820b08d26dc20fa41dafcb1b Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 25 Jan 2015 17:52:40 +0000 Subject: [PATCH 080/409] edited debugging so its more useful / easy to read --- irclib.py | 16 +++++++++------- maker.py | 2 -- taker.py | 2 -- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/irclib.py b/irclib.py index c43e2bbd..91585d41 100644 --- a/irclib.py +++ b/irclib.py @@ -114,26 +114,26 @@ def shutdown(self): self.give_up = True def pubmsg(self, message): - #debug('pubmsg ' + message) + debug('>>pubmsg ' + message) self.send_raw("PRIVMSG " + self.channel + " :" + message) def privmsg(self, nick, message): - #debug('privmsg to ' + nick + ' ' + message) + debug('>>privmsg to ' + nick + ' ' + message) if nick in self.encrypting.keys() and self.encrypting[nick]: message = self.encrypt_encode(message,nick) if len(message) > 350: message_chunks = chunks(message,350) else: message_chunks = [message] - print "We are going to send these chunks: " - print message_chunks + #print "We are going to send these chunks: " + #print message_chunks for m in message_chunks: trailer = ' ~' if m==message_chunks[-1] else ' ;' self.send_raw("PRIVMSG " + nick + " :" + m + trailer) def send_raw(self, line): - if not line.startswith('PING LAG'): - debug('sendraw ' + line) + #if not line.startswith('PING LAG'): + # debug('sendraw ' + line) self.sock.sendall(line + '\r\n') def __handle_privmsg(self, source, target, message): @@ -160,10 +160,12 @@ def __handle_privmsg(self, source, target, message): else: parsed = self.built_privmsg[nick] #wipe the message buffer waiting for the next one self.built_privmsg[nick]='' + debug("< Date: Sun, 25 Jan 2015 18:32:07 +0000 Subject: [PATCH 081/409] fixed bug in the debugging code --- TODO | 5 +++++ irclib.py | 6 ++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/TODO b/TODO index a90a5e40..72ef6a9a 100644 --- a/TODO +++ b/TODO @@ -37,6 +37,11 @@ i suggest creating a thread that only dispatches/writes to the irc socket TODO have an option for sendpayment.py take advantage of the small orders posted by patientsendpayment.py +TODO patientsendpayment.py needs a different algo +if someone fills the entire order all at once, they pay very little + +TODO remove the !txpart !tx stuff since irclib.py now handles chunking + TODO sort out the nick = nick + '_' stuff in irclib its not a good way of doing it diff --git a/irclib.py b/irclib.py index 91585d41..0d218026 100644 --- a/irclib.py +++ b/irclib.py @@ -118,9 +118,11 @@ def pubmsg(self, message): self.send_raw("PRIVMSG " + self.channel + " :" + message) def privmsg(self, nick, message): - debug('>>privmsg to ' + nick + ' ' + message) + will_encrypt = '' if nick in self.encrypting.keys() and self.encrypting[nick]: message = self.encrypt_encode(message,nick) + will_encrypt = 'enc ' + debug('>>privmsg ' + will_encrypt + 'nick=' + nick + ' msg=' + message) if len(message) > 350: message_chunks = chunks(message,350) else: @@ -160,7 +162,7 @@ def __handle_privmsg(self, source, target, message): else: parsed = self.built_privmsg[nick] #wipe the message buffer waiting for the next one self.built_privmsg[nick]='' - debug("< Date: Sun, 25 Jan 2015 19:34:25 +0000 Subject: [PATCH 082/409] slight useful edit for changing nick of yieldgen --- yield-generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yield-generator.py b/yield-generator.py index 886a19ad..8b73ef8b 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -11,6 +11,7 @@ txfee = 1000 cjfee = '0.01' # 1% fee mix_levels = 5 +nickname = 'yigen-' + btc.sha256(gethostname())[:6] nickserv_password = '' minsize = int(2 * txfee / float(cjfee)) #minimum size is such that you always net profit at least the miners fee @@ -90,7 +91,6 @@ def main(): wallet = Wallet(seed, max_mix_depth = mix_levels) wallet.sync_wallet() - nickname = 'yigen-' + btc.sha256(gethostname())[:6] maker = YieldGenerator(wallet) print 'connecting to irc' try: From a64d92edc326d2a4b310f9b829de0793188dfc4e Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 25 Jan 2015 21:01:07 +0000 Subject: [PATCH 083/409] added beginnings of sweep --- common.py | 79 +++++++++++++++++++++++++++++++++++++++++++++++++- sendpayment.py | 38 +++--------------------- 2 files changed, 82 insertions(+), 35 deletions(-) diff --git a/common.py b/common.py index 9a861d8f..b65f5aa8 100644 --- a/common.py +++ b/common.py @@ -1,6 +1,7 @@ import bitcoin as btc from decimal import Decimal +from math import factorial import sys, datetime, json, time, pprint import threading @@ -324,9 +325,85 @@ def choose_order(db, cj_amount, n): chosen_orders = [] for i in range(n): chosen_order = orders[0] #choose the cheapest, later this will be chosen differently - orders = [o for o in orders if o[0] != chosen_order[0]] + orders = [o for o in orders if o[0] != chosen_order[0]] #remove all orders from that same counterparty chosen_orders.append(chosen_order) total_cj_fee += chosen_order[2] chosen_orders = [o[:2] for o in chosen_orders] return dict(chosen_orders), total_cj_fee +def nCk(n, k): + ''' + n choose k + ''' + return factorial(n) / factorial(k) / factorial(n - k) + +def create_combination(li, n): + ''' + Creates a list of combinations of elements of a given list + For example, combination(['apple', 'orange', 'pear'], 2) + = [('apple', 'orange'), ('apple', 'pear'), ('orange', 'pear')] + ''' + if n < 2: + raise ValueError('n must be >= 2') + result = [] + if n == 2: + #creates a list oft + for i, e1 in enumerate(li): + for e2 in li[i+1:]: + result.append((e1, e2)) + else: + for i, e in enumerate(li): + if len(li[i:]) < n: + #there wont be + continue + combn1 = create_combination(li[i:], n - 1) + for c in combn1: + if e not in c: + result.append((e,) + c) + + assert len(result) == nCk(len(li), n) + return result + +def choose_sweep_order(db, my_total_input, my_tx_fee, n): + ''' + choose an order given that we want to be left with no change + i.e. sweep an entire group of utxos + + solve for cjamount when mychange = 0 + for an order with many makers, a mixture of absorder and relorder + mychange = totalin - cjamount - mytxfee - sum(absfee) - sum(relfee*cjamount) + => 0 = totalin - mytxfee - sum(absfee) - cjamount*(1 + sum(relfee)) + => cjamount = (totalin - mytxfee - sum(absfee)) / (1 + sum(relfee)) + ''' + def calc_zero_change_cj_amount(ordertype, cjfee): + cj_amount = None + if ordertype == 'absorder': + cj_amount = my_total_input - my_tx_fee - cjfee + elif ordertype == 'relorder': + cj_amount = (my_total_input - my_tx_fee) / (Decimal(cjfee) + 1) + cj_amount = int(cj_amount.quantize(Decimal(1))) + else: + raise RuntimeError('unknown order type: ' + str(ordertype)) + return cj_amount + + #def calc_zero_change_cj_amount( + + sqlorders = db.execute('SELECT * FROM orderbook;').fetchall() + orderkeys = ['counterparty', 'oid', 'ordertype', 'minsize', 'maxsize', 'txfee', 'cjfee'] + orderlist = [dict([(k, o[k]) for k in orderkeys]) for o in sqlorders] + + ordercombos = create_combination(orderlist, n) + print 'order combos' + pprint.pprint(ordercombos) + + print 'with cjamount' + ordercombos = [(c, calc_zero_change_cj_amount(c)) for c in ordercombos] + return None + + orders = [(o['counterparty'], o['oid'], calc_zero_change_cj_amount(o['ordertype'], o['cjfee']), + o['minsize'], o['maxsize']) for o in sqlorders] + #filter cj_amounts that are not in range + orders = [o[:3] for o in orders if o[2] >= o[3] and o[2] <= o[4]] + orders = sorted(orders, key=lambda k: k[2]) + print 'sweep orders = ' + str(orders) + return orders[-1] #choose one with the highest cj_amount, most left over after paying everything else diff --git a/sendpayment.py b/sendpayment.py index ac3d93c4..d63a2063 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -7,40 +7,6 @@ from optparse import OptionParser import threading -def choose_sweep_order(db, my_total_input, my_tx_fee): - ''' - choose an order given that we want to be left with no change - i.e. sweep an entire group of utxos - - solve for cjamount when mychange = 0 - ABS FEE - mychange = totalin - cjamount - mytxfee - absfee = 0 - => cjamount = totalin - mytxfee - absfee - REL FEE - mychange = totalin - cjamount - mytxfee - relfee*cjamount - => 0 = totalin - mytxfee - cjamount*(1 + relfee) - => cjamount = (totalin - mytxfee) / (1 + relfee) - ''' - def calc_zero_change_cj_amount(ordertype, cjfee): - cj_amount = None - if ordertype == 'absorder': - cj_amount = my_total_input - my_tx_fee - cjfee - elif ordertype == 'relorder': - cj_amount = (my_total_input - my_tx_fee) / (Decimal(cjfee) + 1) - cj_amount = int(cj_amount.quantize(Decimal(1))) - else: - raise RuntimeError('unknown order type: ' + str(ordertype)) - return cj_amount - - sqlorders = db.execute('SELECT * FROM orderbook;').fetchall() - orders = [(o['counterparty'], o['oid'], calc_zero_change_cj_amount(o['ordertype'], o['cjfee']), - o['minsize'], o['maxsize']) for o in sqlorders] - #filter cj_amounts that are not in range - orders = [o[:3] for o in orders if o[2] >= o[3] and o[2] <= o[4]] - orders = sorted(orders, key=lambda k: k[2]) - print 'sweep orders = ' + str(orders) - return orders[-1] #choose one with the highest cj_amount, most left over after paying everything else - #thread which does the buy-side algorithm # chooses which coinjoins to initiate and when class PaymentThread(threading.Thread): @@ -63,6 +29,10 @@ def run(self): self.taker.shutdown() return + totalin = 450000000 + ret = choose_sweep_order(self.taker.db, totalin, self.taker.txfee, self.taker.makercount) + return + orders, total_cj_fee = choose_order(self.taker.db, self.taker.amount, self.taker.makercount) print 'chosen orders to fill ' + str(orders) + ' totalcjfee=' + str(total_cj_fee) total_amount = self.taker.amount + total_cj_fee + self.taker.txfee From f17e452f3e308d085839205f46326269131870d3 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 25 Jan 2015 21:08:38 +0000 Subject: [PATCH 084/409] further fix of the debug printing --- irclib.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/irclib.py b/irclib.py index 0d218026..983321a7 100644 --- a/irclib.py +++ b/irclib.py @@ -118,13 +118,14 @@ def pubmsg(self, message): self.send_raw("PRIVMSG " + self.channel + " :" + message) def privmsg(self, nick, message): - will_encrypt = '' + clearmsg = message + will_encrypt = False if nick in self.encrypting.keys() and self.encrypting[nick]: - message = self.encrypt_encode(message,nick) - will_encrypt = 'enc ' - debug('>>privmsg ' + will_encrypt + 'nick=' + nick + ' msg=' + message) + message = self.encrypt_encode(message, nick) + will_encrypt = True + debug('>>privmsg ' + ('enc ' if will_encrypt else '') + 'nick=' + nick + ' msg=' + clearmsg) if len(message) > 350: - message_chunks = chunks(message,350) + message_chunks = chunks(message, 350) else: message_chunks = [message] #print "We are going to send these chunks: " From 3678d7c747660dc682d38409cb7a045d750bcec9 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 25 Jan 2015 21:50:56 +0000 Subject: [PATCH 085/409] moving towards sweep --- common.py | 34 ++++++++++++++++++++-------------- sendpayment.py | 2 +- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/common.py b/common.py index b65f5aa8..855e71c1 100644 --- a/common.py +++ b/common.py @@ -375,29 +375,35 @@ def choose_sweep_order(db, my_total_input, my_tx_fee, n): => 0 = totalin - mytxfee - sum(absfee) - cjamount*(1 + sum(relfee)) => cjamount = (totalin - mytxfee - sum(absfee)) / (1 + sum(relfee)) ''' - def calc_zero_change_cj_amount(ordertype, cjfee): - cj_amount = None - if ordertype == 'absorder': - cj_amount = my_total_input - my_tx_fee - cjfee - elif ordertype == 'relorder': - cj_amount = (my_total_input - my_tx_fee) / (Decimal(cjfee) + 1) - cj_amount = int(cj_amount.quantize(Decimal(1))) - else: - raise RuntimeError('unknown order type: ' + str(ordertype)) - return cj_amount - - #def calc_zero_change_cj_amount( + def calc_zero_change_cj_amount(ordercombo): + sumabsfee = 0 + sumrelfee = Decimal('0') + for order in ordercombo: + if order['ordertype'] == 'absorder': + sumabsfee += int(order['cjfee']) + elif order['ordertype'] == 'relorder': + sumrelfee += Decimal(order['cjfee']) + else: + raise RuntimeError('unknown order type: ' + str(ordertype)) + cjamount = (my_total_input - my_tx_fee - sumabsfee) / (1 + sumrelfee) + cjamount = int(cjamount.quantize(Decimal(1))) + return cjamount + + def amount_in_range(ordercombo, cjamount): + for order in ordercombo: + if order['maxsize' sqlorders = db.execute('SELECT * FROM orderbook;').fetchall() orderkeys = ['counterparty', 'oid', 'ordertype', 'minsize', 'maxsize', 'txfee', 'cjfee'] orderlist = [dict([(k, o[k]) for k in orderkeys]) for o in sqlorders] ordercombos = create_combination(orderlist, n) - print 'order combos' - pprint.pprint(ordercombos) print 'with cjamount' ordercombos = [(c, calc_zero_change_cj_amount(c)) for c in ordercombos] + #ordercombos = [for oc in ordercombos if oc[1] > ] + ordercombos = sorted(ordercombos, key=lambda k: k[1]) + pprint.pprint(ordercombos) return None orders = [(o['counterparty'], o['oid'], calc_zero_change_cj_amount(o['ordertype'], o['cjfee']), diff --git a/sendpayment.py b/sendpayment.py index d63a2063..91804163 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -29,7 +29,7 @@ def run(self): self.taker.shutdown() return - totalin = 450000000 + totalin = 100000000 ret = choose_sweep_order(self.taker.db, totalin, self.taker.txfee, self.taker.makercount) return From ecc79edf3fea00c5af25c23758b881a3b4cd59dc Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 25 Jan 2015 23:54:27 +0000 Subject: [PATCH 086/409] finished choose_sweep_order --- common.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/common.py b/common.py index 855e71c1..5a7dfa80 100644 --- a/common.py +++ b/common.py @@ -389,9 +389,11 @@ def calc_zero_change_cj_amount(ordercombo): cjamount = int(cjamount.quantize(Decimal(1))) return cjamount - def amount_in_range(ordercombo, cjamount): + def is_amount_in_range(ordercombo, cjamount): for order in ordercombo: - if order['maxsize' + if cjamount >= order['maxsize'] or cjamount <= order['minsize']: + return False + return True sqlorders = db.execute('SELECT * FROM orderbook;').fetchall() orderkeys = ['counterparty', 'oid', 'ordertype', 'minsize', 'maxsize', 'txfee', 'cjfee'] @@ -399,17 +401,16 @@ def amount_in_range(ordercombo, cjamount): ordercombos = create_combination(orderlist, n) - print 'with cjamount' ordercombos = [(c, calc_zero_change_cj_amount(c)) for c in ordercombos] - #ordercombos = [for oc in ordercombos if oc[1] > ] + ordercombos = [oc for oc in ordercombos if is_amount_in_range(oc[0], oc[1])] ordercombos = sorted(ordercombos, key=lambda k: k[1]) - pprint.pprint(ordercombos) - return None + dbgprint = [([(o['counterparty'], o['oid']) for o in oc[0]], oc[1]) for oc in ordercombos] + debug('considered order combinations') + pprint.pprint(dbgprint) + ordercombo = ordercombos[-1] #choose the cheapest, i.e. highest cj_amount + orders = dict([(o['counterparty'], o['oid']) for o in ordercombo[0]]) + cjamount = ordercombo[1] + debug('chosen orders = ' + str(orders)) + debug('cj amount = ' + str(cjamount)) + return orders, cjamount - orders = [(o['counterparty'], o['oid'], calc_zero_change_cj_amount(o['ordertype'], o['cjfee']), - o['minsize'], o['maxsize']) for o in sqlorders] - #filter cj_amounts that are not in range - orders = [o[:3] for o in orders if o[2] >= o[3] and o[2] <= o[4]] - orders = sorted(orders, key=lambda k: k[2]) - print 'sweep orders = ' + str(orders) - return orders[-1] #choose one with the highest cj_amount, most left over after paying everything else From f7e923fdd40ca7a61fabe7998d3a8c4008e8d794 Mon Sep 17 00:00:00 2001 From: Belcher Date: Mon, 26 Jan 2015 01:01:20 +0000 Subject: [PATCH 087/409] feature to sweep an whole mixdepth of sendpayment --- sendpayment.py | 45 +++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/sendpayment.py b/sendpayment.py index 91804163..7188dbdf 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -29,27 +29,27 @@ def run(self): self.taker.shutdown() return - totalin = 100000000 - ret = choose_sweep_order(self.taker.db, totalin, self.taker.txfee, self.taker.makercount) - return - - orders, total_cj_fee = choose_order(self.taker.db, self.taker.amount, self.taker.makercount) - print 'chosen orders to fill ' + str(orders) + ' totalcjfee=' + str(total_cj_fee) - total_amount = self.taker.amount + total_cj_fee + self.taker.txfee - print 'total amount spent = ' + str(total_amount) - - utxos = self.taker.wallet.select_utxos(self.taker.mixdepth, total_amount) - self.taker.cjtx = takermodule.CoinJoinTX(self.taker, self.taker.amount, - orders, utxos, self.taker.destaddr, - self.taker.wallet.get_change_addr(self.taker.mixdepth), self.taker.txfee, - self.finishcallback) - - ''' - counterparty, oid, cj_amount = choose_sweep_order(addrvalue['value'], my_tx_fee) - cjtx = CoinJoinTX(self.taker, cj_amount, [counterparty], [int(oid)], - [utxo], self.taker.wallet.get_receive_addr(mixing_depth=1), None, - my_tx_fee, self.finished_cj_callback) - ''' + if self.taker.amount == 0: + total_value = 0 + utxo_list = self.taker.wallet.get_mix_utxo_list()[self.taker.mixdepth] + for utxo in utxo_list: + total_value += self.taker.wallet.unspent[utxo]['value'] + orders, cjamount = choose_sweep_order(self.taker.db, total_value, self.taker.txfee, self.taker.makercount) + self.taker.cjtx = takermodule.CoinJoinTX(self.taker, cjamount, + orders, utxo_list, self.taker.destaddr, + self.taker.wallet.get_change_addr(self.taker.mixdepth), self.taker.txfee, + self.finishcallback) + else: + orders, total_cj_fee = choose_order(self.taker.db, self.taker.amount, self.taker.makercount) + print 'chosen orders to fill ' + str(orders) + ' totalcjfee=' + str(total_cj_fee) + total_amount = self.taker.amount + total_cj_fee + self.taker.txfee + print 'total amount spent = ' + str(total_amount) + + utxos = self.taker.wallet.select_utxos(self.taker.mixdepth, total_amount) + self.taker.cjtx = takermodule.CoinJoinTX(self.taker, self.taker.amount, + orders, utxos, self.taker.destaddr, + self.taker.wallet.get_change_addr(self.taker.mixdepth), self.taker.txfee, + self.finishcallback) class SendPayment(takermodule.Taker): def __init__(self, wallet, destaddr, amount, makercount, txfee, waittime, mixdepth): @@ -69,7 +69,8 @@ def on_welcome(self): def main(): parser = OptionParser(usage='usage: %prog [options] [seed] [amount] [destaddr]', description='Sends a single payment from the zero mixing depth of your ' + - ' wallet to an given address using coinjoin and then switches off.') + 'wallet to an given address using coinjoin and then switches off. ' + + 'Setting amount to zero will do a sweep, where the entire mix depth is emptied') parser.add_option('-f', '--txfee', action='store', type='int', dest='txfee', default=10000, help='miner fee contribution, in satoshis, default=10000') parser.add_option('-w', '--wait-time', action='store', type='float', dest='waittime', From f1f547b49f3fdee59f66d92986c521cc6cad5ee3 Mon Sep 17 00:00:00 2001 From: Belcher Date: Mon, 26 Jan 2015 01:03:50 +0000 Subject: [PATCH 088/409] added some TODOs --- TODO | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/TODO b/TODO index 72ef6a9a..56045360 100644 --- a/TODO +++ b/TODO @@ -29,7 +29,7 @@ have the taker enforce this, look up the txhash of the maker's utxo and make sur TODO remove the badly done chunking code in the coinjoin layer, such as !txpart !tx - since chunking should be done in the messaging layer + since chunking should be done in the messaging layer TODO implement rate limiting for irc.privmsg to stop the bot being killed due to flood i suggest creating a thread that only dispatches/writes to the irc socket @@ -40,11 +40,12 @@ by patientsendpayment.py TODO patientsendpayment.py needs a different algo if someone fills the entire order all at once, they pay very little -TODO remove the !txpart !tx stuff since irclib.py now handles chunking - TODO sort out the nick = nick + '_' stuff in irclib its not a good way of doing it +TODO +implement sasl in the irc code, required for freenode over tor and better than sending nickserv :identify + TODO robustness in the taker code for instance if the maker being joined with quits halfway through From 3cf7cb6eab89c64fba7852315f650fe509e6ea57 Mon Sep 17 00:00:00 2001 From: Belcher Date: Mon, 26 Jan 2015 01:15:46 +0000 Subject: [PATCH 089/409] added more debugs, fixed slightly --- sendpayment.py | 6 ++---- taker.py | 5 +++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/sendpayment.py b/sendpayment.py index 7188dbdf..e1dab938 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -35,10 +35,8 @@ def run(self): for utxo in utxo_list: total_value += self.taker.wallet.unspent[utxo]['value'] orders, cjamount = choose_sweep_order(self.taker.db, total_value, self.taker.txfee, self.taker.makercount) - self.taker.cjtx = takermodule.CoinJoinTX(self.taker, cjamount, - orders, utxo_list, self.taker.destaddr, - self.taker.wallet.get_change_addr(self.taker.mixdepth), self.taker.txfee, - self.finishcallback) + self.taker.cjtx = takermodule.CoinJoinTX(self.taker, cjamount, orders, utxo_list, + self.taker.destaddr, None, self.taker.txfee, self.finishcallback) else: orders, total_cj_fee = choose_order(self.taker.db, self.taker.amount, self.taker.makercount) print 'chosen orders to fill ' + str(orders) + ' totalcjfee=' + str(total_cj_fee) diff --git a/taker.py b/taker.py index 8e3d45f6..9b485ff9 100644 --- a/taker.py +++ b/taker.py @@ -14,6 +14,7 @@ def __init__(self, taker, cj_amount, orders, my_utxos, my_cj_addr, thats used if you want to entirely coinjoin one utxo with no change left over orders is the orders you want to fill {'counterpartynick': oid, 'cp2': oid2} ''' + debug('starting cj to ' + my_cj_addr + ' with change at ' + my_change_addr) self.taker = taker self.cj_amount = cj_amount self.active_orders = dict(orders) @@ -61,8 +62,8 @@ def recv_txio(self, nick, utxo_list, cj_pub, change_addr): my_total_in += int(usvals['value']) my_change_value = my_total_in - self.cj_amount - self.cjfee_total - self.my_txfee - print 'fee breakdown for me totalin=%d txfee=%d cjfee_total=%d' % (my_total_in, - self.my_txfee, self.cjfee_total) + print 'fee breakdown for me totalin=%d txfee=%d cjfee_total=%d => changevalue=%d' % (my_total_in, + self.my_txfee, self.cjfee_total, my_change_value) if self.my_change_addr == None: if my_change_value != 0: print 'WARNING CHANGE NOT BEING USED\nCHANGEVALUE = ' + str(my_change_value) From 1adc26e0fd9df7c17dcaaf5f21ab78f980941bb8 Mon Sep 17 00:00:00 2001 From: Belcher Date: Tue, 27 Jan 2015 00:12:06 +0000 Subject: [PATCH 090/409] slight edit / bugfix --- taker.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/taker.py b/taker.py index 9b485ff9..0ecca5b1 100644 --- a/taker.py +++ b/taker.py @@ -14,7 +14,7 @@ def __init__(self, taker, cj_amount, orders, my_utxos, my_cj_addr, thats used if you want to entirely coinjoin one utxo with no change left over orders is the orders you want to fill {'counterpartynick': oid, 'cp2': oid2} ''' - debug('starting cj to ' + my_cj_addr + ' with change at ' + my_change_addr) + debug('starting cj to ' + my_cj_addr + ' with change at ' + str(my_change_addr)) self.taker = taker self.cj_amount = cj_amount self.active_orders = dict(orders) @@ -65,7 +65,9 @@ def recv_txio(self, nick, utxo_list, cj_pub, change_addr): print 'fee breakdown for me totalin=%d txfee=%d cjfee_total=%d => changevalue=%d' % (my_total_in, self.my_txfee, self.cjfee_total, my_change_value) if self.my_change_addr == None: - if my_change_value != 0: + if my_change_value != 0 or abs(my_change_value) != 1: + #seems you wont always get exactly zero because of integer rounding + # so 1 satoshi extra or fewer being spent as miner fees is acceptable print 'WARNING CHANGE NOT BEING USED\nCHANGEVALUE = ' + str(my_change_value) else: self.outputs.append({'address': self.my_change_addr, 'value': my_change_value}) From 3251abd1deec7b1868e7d3d4e6dade711aaf4f5b Mon Sep 17 00:00:00 2001 From: Belcher Date: Tue, 27 Jan 2015 22:09:26 +0000 Subject: [PATCH 091/409] fixed bug with sending from mix depths > 1 --- patientsendpayment.py | 2 +- sendpayment.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/patientsendpayment.py b/patientsendpayment.py index 6d6bb24b..ed7cef64 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -136,7 +136,7 @@ def main(): print 'txfee=%d cjfee=%d waittime=%s makercount=%d' % (options.txfee, options.cjfee, str(timedelta(hours=options.waittime)), options.makercount) - wallet = Wallet(seed) + wallet = Wallet(seed, options.mixdepth + 1) wallet.sync_wallet() utxo_list = wallet.get_mix_utxo_list()[options.mixdepth] diff --git a/sendpayment.py b/sendpayment.py index e1dab938..95e87219 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -89,7 +89,7 @@ def main(): from socket import gethostname nickname = 'payer-' + btc.sha256(gethostname())[:6] - wallet = Wallet(seed) + wallet = Wallet(seed, options.mixdepth + 1) wallet.sync_wallet() print 'starting irc' From 6847af923a7b967d262e1f7212db3da7837aaecc Mon Sep 17 00:00:00 2001 From: Belcher Date: Wed, 28 Jan 2015 00:39:55 +0000 Subject: [PATCH 092/409] fixed bug involved in cleaning up encryption state --- TODO | 3 +++ maker.py | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/TODO b/TODO index 56045360..cf3b6e67 100644 --- a/TODO +++ b/TODO @@ -43,6 +43,9 @@ if someone fills the entire order all at once, they pay very little TODO sort out the nick = nick + '_' stuff in irclib its not a good way of doing it +TODO +bug in sweep, doesnt work for N=1 (just one maker) + TODO implement sasl in the irc code, required for freenode over tor and better than sending nickserv :identify diff --git a/maker.py b/maker.py index 30a8d9d4..58ab833e 100644 --- a/maker.py +++ b/maker.py @@ -87,6 +87,7 @@ def recv_tx(self, nick, b64txpart): self.maker.privmsg(nick, ''.join([command_prefix + 'sig ' + s for s in sigs])) #once signature is sent, close encrypted channel to this taker. self.maker.end_encryption(nick) + self.maker.active_orders[nick] = None def unconfirm_callback(self, balance): removed_utxos = self.maker.wallet.remove_old_utxos(self.tx) @@ -189,7 +190,9 @@ def on_privmsg(self, nick, message): if chunks[0] == 'fill': if nick in self.active_orders and self.active_orders[nick] != None: - self.send_error(nick, 'Already have partially-filled order') + self.active_orders[nick] = None + self.end_encryption(nick) + debug('had a partially filled order but starting over now') try: oid = int(chunks[1]) amount = int(chunks[2]) @@ -205,7 +208,6 @@ def on_privmsg(self, nick, message): self.active_orders[nick].recv_tx_part(b64txpart) else: self.active_orders[nick].recv_tx(nick, b64txpart) - self.active_orders[nick] = None except CJMakerOrderError: self.active_orders[nick] = None continue @@ -228,6 +230,7 @@ def on_set_topic(self, newtopic): print '=' * 60 def on_leave(self, nick): + self.end_encryption(nick) self.active_orders[nick] = None def modify_orders(self, to_cancel, to_announce): From 25a1862bc8fb153a55bffe490da335ab5ecb45be Mon Sep 17 00:00:00 2001 From: Belcher Date: Wed, 28 Jan 2015 01:14:27 +0000 Subject: [PATCH 093/409] removed !txpart message chunking since thats already handled by irclib --- TODO | 4 ---- maker.py | 21 +++++---------------- taker.py | 7 +------ 3 files changed, 6 insertions(+), 26 deletions(-) diff --git a/TODO b/TODO index cf3b6e67..0a08f3b1 100644 --- a/TODO +++ b/TODO @@ -27,10 +27,6 @@ TODO have the taker enforce this, look up the txhash of the maker's utxo and make sure it is already in a block -TODO -remove the badly done chunking code in the coinjoin layer, such as !txpart !tx - since chunking should be done in the messaging layer - TODO implement rate limiting for irc.privmsg to stop the bot being killed due to flood i suggest creating a thread that only dispatches/writes to the irc socket diff --git a/maker.py b/maker.py index 58ab833e..e47a4ef6 100644 --- a/maker.py +++ b/maker.py @@ -23,7 +23,6 @@ def __init__(self, maker, nick, oid, amount, taker_pk): self.ordertype = order['ordertype'] self.txfee = order['txfee'] self.cjfee = order['cjfee'] - self.b64txparts = [] debug('new cjorder nick=%s oid=%d amount=%d' % (nick, oid, amount)) #always a new address even if the order ends up never being # furfilled, you dont want someone pretending to fill all your @@ -50,16 +49,9 @@ def auth_counterparty(self,nick,i_utxo_pubkey,btc_sig): btc_pub + ' ' + self.change_addr + ' ' + btc_sig) return True - def recv_tx_part(self, b64txpart): - self.b64txparts.append(b64txpart) - size = sum([len(s) for s in self.b64txparts]) - if size > 60000000: #~2MB * 2 * 4/3 - self.maker.send_error(nick, 'tx too large, buffer limit reached') - - def recv_tx(self, nick, b64txpart): - self.b64txparts.append(b64txpart) + def recv_tx(self, nick, b64tx): try: - txhex = base64.b64decode(''.join(self.b64txparts)).encode('hex') + txhex = base64.b64decode(b64tx).encode('hex') except TypeError as e: self.maker.send_error(nick, 'bad base64 tx. ' + repr(e)) try: @@ -200,14 +192,11 @@ def on_privmsg(self, nick, message): except (ValueError, IndexError) as e: self.send_error(nick, str(e)) self.active_orders[nick] = CoinJoinOrder(self, nick, oid, amount, taker_pk) - elif chunks[0] == 'txpart' or chunks[0] == 'tx': + elif chunks[0] == 'tx': if nick not in self.active_orders or self.active_orders[nick] == None: self.send_error(nick, 'No open order from this nick') - b64txpart = chunks[1] - if chunks[0] == 'txpart': - self.active_orders[nick].recv_tx_part(b64txpart) - else: - self.active_orders[nick].recv_tx(nick, b64txpart) + b64tx = chunks[1] + self.active_orders[nick].recv_tx(nick, b64tx) except CJMakerOrderError: self.active_orders[nick] = None continue diff --git a/taker.py b/taker.py index 0ecca5b1..4396efa9 100644 --- a/taker.py +++ b/taker.py @@ -77,13 +77,8 @@ def recv_txio(self, nick, utxo_list, cj_pub, change_addr): import pprint debug('obtained tx\n' + pprint.pformat(btc.deserialize(tx))) txb64 = base64.b64encode(tx.decode('hex')) - n = MAX_PRIVMSG_LEN - txparts = [txb64[i:i+n] for i in range(0, len(txb64), n)] - for p in txparts[:-1]: - for nickk in self.active_orders.keys(): - self.taker.privmsg(nickk, command_prefix + 'txpart ' + p) for nickk in self.active_orders.keys(): - self.taker.privmsg(nickk, command_prefix + 'tx ' + txparts[-1]) + self.taker.privmsg(nickk, command_prefix + 'tx ' + txb64) #now sign it ourselves here for index, ins in enumerate(btc.deserialize(tx)['ins']): From 86d7184143b409dadc2f4ebfeea1fbffdb647b0f Mon Sep 17 00:00:00 2001 From: Belcher Date: Wed, 28 Jan 2015 01:59:32 +0000 Subject: [PATCH 094/409] brought TODO up to speed --- TODO | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/TODO b/TODO index 0a08f3b1..265c3f8f 100644 --- a/TODO +++ b/TODO @@ -91,16 +91,6 @@ TODO think about this TODO probably a good idea to have a debug.log where loads of information is dumped -TODO -for the !addrs command, firstly change its name since it also includes the utxo inputs - secondly, the utxo list might be longer than can fit in an irc message, so create a - !addrsparts or something command - -TODO -code a gui where a human can see the state of the orderbook and easily choose orders to fill -code a gui that easily explains to a human how they can choose a fee for their yield-generator.py -both are important for market forces, since markets emerge from human decisions and actions - #TODO add random delays to the orderbook stuff so there isnt such a traffic spike when a new bot joins #two options, random delay !orderbook for ones which dont mind, !orderbook without delay for bots # which need the orders asap From 7c34085b58a742f2c6e394e29075e7bfb016fbd8 Mon Sep 17 00:00:00 2001 From: Belcher Date: Wed, 28 Jan 2015 22:42:51 +0000 Subject: [PATCH 095/409] fixed bug by removing mix_balance caching from yieldgen --- common.py | 11 +++++++++++ yield-generator.py | 14 +++----------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/common.py b/common.py index 5a7dfa80..58709509 100644 --- a/common.py +++ b/common.py @@ -124,6 +124,7 @@ def add_new_utxos(self, tx, txid): debug('added utxos, wallet now is \n' + pprint.pformat(self.get_mix_utxo_list())) return added_utxos + #TODO change the name of this to get_utxo_list_by_mixdepth def get_mix_utxo_list(self): ''' returns a list of utxos sorted by different mix levels @@ -136,6 +137,16 @@ def get_mix_utxo_list(self): mix_utxo_list[mixdepth].append(utxo) return mix_utxo_list + def get_balance_by_mixdepth(self): + mix_utxo_list = self.wallet.get_mix_utxo_list() + mix_balance = {} + for mixdepth, utxo_list in mix_utxo_list.iteritems(): + total_value = 0 + for utxo in utxo_list: + total_value += self.wallet.unspent[utxo]['value'] + mix_balance[mixdepth] = total_value + return mix_balance + def select_utxos(self, mixdepth, amount): utxo_list = self.get_mix_utxo_list()[mixdepth] unspent = [{'utxo': utxo, 'value': self.unspent[utxo]['value']} diff --git a/yield-generator.py b/yield-generator.py index 8b73ef8b..1453a32b 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -35,14 +35,7 @@ def on_connect(self): self.send_raw('PRIVMSG NickServ :identify ' + nickserv_password) def create_my_orders(self): - mix_utxo_list = self.wallet.get_mix_utxo_list() - mix_balance = {} - for mixdepth, utxo_list in mix_utxo_list.iteritems(): - total_value = 0 - for utxo in utxo_list: - total_value += self.wallet.unspent[utxo]['value'] - mix_balance[mixdepth] = total_value - + mix_balance = self.wallet.get_balance_by_mixdepth() if len([b for m, b in mix_balance.iteritems() if b > 0]) == 0: debug('do not have any coins left') return [] @@ -50,12 +43,11 @@ def create_my_orders(self): #print mix_balance max_mix = max(mix_balance, key=mix_balance.get) order = {'oid': 0, 'ordertype': 'relorder', 'minsize': minsize, - 'maxsize': mix_balance[max_mix], 'txfee': txfee, 'cjfee': cjfee, - 'mix_balance': mix_balance} + 'maxsize': mix_balance[max_mix], 'txfee': txfee, 'cjfee': cjfee} return [order] def oid_to_order(self, oid, amount): - mix_balance = self.orderlist[0]['mix_balance'] + mix_balance = self.wallet.get_balance_by_mixdepth() max_mix = max(mix_balance, key=mix_balance.get) #algo attempts to make the largest-balance mixing depth get an even larger balance From 3f5b18a85b3e5ab7db7bc9d3ff3b46cda72dca67 Mon Sep 17 00:00:00 2001 From: Belcher Date: Thu, 29 Jan 2015 00:38:47 +0000 Subject: [PATCH 096/409] created abstract class message channel --- message_channel.py | 30 ++++++++++++++++++++++++++++++ taker.py | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 message_channel.py diff --git a/message_channel.py b/message_channel.py new file mode 100644 index 00000000..90ef0b41 --- /dev/null +++ b/message_channel.py @@ -0,0 +1,30 @@ + +class MessageChannel(object): + ''' + Abstract class which implements a way for bots to communicate + ''' + + def run(self): pass + def shutdown(self): pass + + #orderbook watcher commands + def register_orderbookwatch_callbacks(self, on_order_seen=None, + on_order_cancel=None): pass + def request_orderbook(self): pass + + #taker commands + def register_taker_callbacks(self, on_error=None, on_pubkey=None, on_ioauth=None, + on_sigs=None): pass + def fill_order(self, nick, oid, cj_amount, taker_pubkey): pass + def send_auth(self, nick, pubkey, sig): pass + def send_tx(self, nick, txhex): pass + + #maker commands + def register_maker_callbacks(self, on_orderbook_requested=None, on_order_filled=None, + on_seen_auth=None, on_seen_tx=None): pass + def announce_orders(self, orderlist, nick=None): pass #nick=None means announce publicly + def cancel_orders(self, oid_list): pass + def send_error(self, nick, errormsg): pass + def send_pubkey(self, nick, pubkey): pass + def send_ioauth(self, nick, utxo_list, cj_pubkey, change_addr, sig): pass + def send_sigs(self, nick, sig_list): pass diff --git a/taker.py b/taker.py index 4396efa9..2c58c0a4 100644 --- a/taker.py +++ b/taker.py @@ -233,7 +233,7 @@ def on_privmsg(self, nick, message): my_btc_sig = btc.ecdsa_sign(self.enc_kp.hex_pk(),my_btc_priv) message = '!auth ' + my_btc_pub + ' ' + my_btc_sig self.privmsg(nick,message) #note: we do this *before* starting encryption - if chunks[0] == 'auth': + if chunks[0] == 'auth': #TODO will be changing this name, to iosig or something utxo_list = chunks[1].split(',') cj_pub = chunks[2] change_addr = chunks[3] From 11716cc2a92d367feadec3a4283f8ddaa06ec7b9 Mon Sep 17 00:00:00 2001 From: Belcher Date: Thu, 29 Jan 2015 00:54:10 +0000 Subject: [PATCH 097/409] bugfix --- common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common.py b/common.py index 58709509..fb94d729 100644 --- a/common.py +++ b/common.py @@ -138,12 +138,12 @@ def get_mix_utxo_list(self): return mix_utxo_list def get_balance_by_mixdepth(self): - mix_utxo_list = self.wallet.get_mix_utxo_list() + mix_utxo_list = self.get_mix_utxo_list() mix_balance = {} for mixdepth, utxo_list in mix_utxo_list.iteritems(): total_value = 0 for utxo in utxo_list: - total_value += self.wallet.unspent[utxo]['value'] + total_value += self.unspent[utxo]['value'] mix_balance[mixdepth] = total_value return mix_balance From acaaf0e3707e6fad3ceca8efe00f0a7e1d3393a4 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Thu, 29 Jan 2015 14:44:55 +0200 Subject: [PATCH 098/409] first steps of redoing encryption --- enc_wrapper.py | 10 +++++++++- maker.py | 18 ++++++++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/enc_wrapper.py b/enc_wrapper.py index 36531e5d..d355e845 100644 --- a/enc_wrapper.py +++ b/enc_wrapper.py @@ -59,7 +59,15 @@ def as_init_encryption(kp, c_pk): #TODO: Sign, verify. At the moment we are using #bitcoin signatures so it isn't necessary. - +#encoding for passing over the wire +def encrypt_encode(msg,box): + encrypted = box.encrypt(msg) + return base64.b64encode(encrypted) + +def decode_decrypt(msg,box): + decoded = base64.b64decode(msg) + return box.decrypt(decoded) + def test_case(case_name, alice_box, bob_box, ab_message, ba_message, num_iterations=1): for i in range(num_iterations): otw_amsg = alice_box.encrypt(ab_message) diff --git a/maker.py b/maker.py index e47a4ef6..cba94aae 100644 --- a/maker.py +++ b/maker.py @@ -4,6 +4,7 @@ import irclib import bitcoin as btc import base64, pprint +import enc_wrapper class CoinJoinOrder(object): def __init__(self, maker, nick, oid, amount, taker_pk): @@ -12,6 +13,11 @@ def __init__(self, maker, nick, oid, amount, taker_pk): self.cj_amount = amount #the btc pubkey of the utxo that the taker plans to use as input self.taker_pk = taker_pk + #create DH keypair on the fly for this Order object + self.kp = enc_wrapper.init_keypair() + #the encryption channel crypto box for this Order object + self.crypto_box = enc_wrapper.as_init_encryption(self.kp, self.taker_pk) + order_s = [o for o in maker.orderlist if o['oid'] == oid] if len(order_s) == 0: self.maker.send_error(nick, 'oid not found') @@ -27,10 +33,15 @@ def __init__(self, maker, nick, oid, amount, taker_pk): #always a new address even if the order ends up never being # furfilled, you dont want someone pretending to fill all your # orders to find out which addresses you use - maker.privmsg(nick,command_prefix+ 'pubkey ' + maker.enc_kp.hex_pk()) - - self.maker.start_encryption(nick,taker_pk) + pubkeymsg = command_prefix+ 'pubkey ' + self.kp.hex_pk() + self.send_priv(nick,pubkeymsg,False) + def send_priv(self, nick, msg, enc=False): + if enc: + self.maker.privmsg(nick,enc_wrapper.encrypt_encode(msg, self.crypto_box)) + else: + self.maker.privmsg(nick,msg) + def auth_counterparty(self,nick,i_utxo_pubkey,btc_sig): #TODO: add check that the pubkey's address is part of the order. self.i_utxo_pubkey = i_utxo_pubkey @@ -183,7 +194,6 @@ def on_privmsg(self, nick, message): if chunks[0] == 'fill': if nick in self.active_orders and self.active_orders[nick] != None: self.active_orders[nick] = None - self.end_encryption(nick) debug('had a partially filled order but starting over now') try: oid = int(chunks[1]) From 79db6b515beb3fb9408fccf2df17eb99833026ff Mon Sep 17 00:00:00 2001 From: Belcher Date: Thu, 29 Jan 2015 23:46:46 +0000 Subject: [PATCH 099/409] wallettool shows total depth --- wallet-tool.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/wallet-tool.py b/wallet-tool.py index cf60809b..440485c3 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -47,6 +47,7 @@ wallet.find_unspent_addresses() if method == 'display': + total_balance = 0 for m in range(wallet.max_mix_depth): print 'mixing depth %d m/0/%d/' % (m, m) balance_depth = 0 @@ -63,6 +64,8 @@ used = ('used' if k < wallet.index[m][forchange] else ' new') print ' m/0/%d/%d/%02d %s %s %.8fbtc' % (m, forchange, k, addr, used, balance/1e8) print 'for mixdepth=%d balance=%.8fbtc' % (m, balance_depth/1e8) + total_balance += balance_depth + print 'total balance = %.8fbtc' % (total_balance/1e8) elif method == 'combine': ins = [] @@ -94,4 +97,4 @@ outs.append({'address': destaddr, 'value': balance}) print get_signed_tx(wallet, ins, outs) - \ No newline at end of file + From cc405d7ab7c605151fdec18ff7faa8a830456eea Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 30 Jan 2015 00:02:41 +0000 Subject: [PATCH 100/409] added summary method in wallettool --- .gitignore | 3 ++- wallet-tool.py | 21 ++++++++++++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 7e99e367..c9b568f7 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -*.pyc \ No newline at end of file +*.pyc +*.swp diff --git a/wallet-tool.py b/wallet-tool.py index 440485c3..f28b9870 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -17,8 +17,9 @@ parser = OptionParser(usage='usage: %prog [options] [seed] [method]', description='Does useful little lasts involving your bip32 wallet. The' - + ' method is one of the following: Display- shows all addresses and balances' - + '. Combine- combines all utxos into one output for each mixing level. Used for' + + ' method is one of the following: display- shows all addresses and balances.' + + ' summary- shows a summary of mixing depth balances.' + + ' combine- combines all utxos into one output for each mixing level. Used for' + ' testing and is detrimental to privacy.' + ' reset - send all utxos back to first receiving address at zeroth mixing level.' + ' Also only for testing and destroys privacy.') @@ -66,7 +67,21 @@ print 'for mixdepth=%d balance=%.8fbtc' % (m, balance_depth/1e8) total_balance += balance_depth print 'total balance = %.8fbtc' % (total_balance/1e8) - + +elif method == 'summary': + total_balance = 0 + for m in range(wallet.max_mix_depth): + balance_depth = 0 + for forchange in [0, 1]: + for k in range(wallet.index[m][forchange]): + addr = wallet.get_addr(m, forchange, k) + for addrvalue in wallet.unspent.values(): + if addr == addrvalue['address']: + balance_depth += addrvalue['value'] + print 'for mixdepth=%d balance=%.8fbtc' % (m, balance_depth/1e8) + total_balance += balance_depth + print 'total balance = %.8fbtc' % (total_balance/1e8) + elif method == 'combine': ins = [] outs = [] From a1bba7fa4812d8eaba79102c2e678a9f10db74d4 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sat, 31 Jan 2015 15:47:09 +0000 Subject: [PATCH 101/409] fixed a race condition bug in maker by adding thread locks --- maker.py | 59 +++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 39 insertions(+), 20 deletions(-) diff --git a/maker.py b/maker.py index e47a4ef6..176b3edd 100644 --- a/maker.py +++ b/maker.py @@ -3,7 +3,7 @@ from common import * import irclib import bitcoin as btc -import base64, pprint +import base64, pprint, threading class CoinJoinOrder(object): def __init__(self, maker, nick, oid, amount, taker_pk): @@ -18,7 +18,6 @@ def __init__(self, maker, nick, oid, amount, taker_pk): order = order_s[0] if amount < order['minsize'] or amount > order['maxsize']: self.maker.send_error(nick, 'amount out of range') - #TODO return addresses, not mixing depths, so you can coinjoin to outside your own wallet self.utxos, self.cj_addr, self.change_addr = maker.oid_to_order(oid, amount) self.ordertype = order['ordertype'] self.txfee = order['txfee'] @@ -28,7 +27,6 @@ def __init__(self, maker, nick, oid, amount, taker_pk): # furfilled, you dont want someone pretending to fill all your # orders to find out which addresses you use maker.privmsg(nick,command_prefix+ 'pubkey ' + maker.enc_kp.hex_pk()) - self.maker.start_encryption(nick,taker_pk) def auth_counterparty(self,nick,i_utxo_pubkey,btc_sig): @@ -82,22 +80,34 @@ def recv_tx(self, nick, b64tx): self.maker.active_orders[nick] = None def unconfirm_callback(self, balance): - removed_utxos = self.maker.wallet.remove_old_utxos(self.tx) + self.wallet_unspent_lock.acquire() + try: + removed_utxos = self.maker.wallet.remove_old_utxos(self.tx) + finally: + self.wallet_unspent_lock.release() debug('saw tx on network, removed_utxos=\n' + pprint.pformat(removed_utxos)) to_cancel, to_announce = self.maker.on_tx_unconfirmed(self, balance, removed_utxos) self.maker.modify_orders(to_cancel, to_announce) def confirm_callback(self, confirmations, txid, balance): - added_utxos = self.maker.wallet.add_new_utxos(self.tx, txid) + self.wallet_unspent_lock.acquire() + try: + added_utxos = self.maker.wallet.add_new_utxos(self.tx, txid) + finally: + self.wallet_unspent_lock.release() debug('tx in a block, added_utxos=\n' + pprint.pformat(added_utxos)) to_cancel, to_announce = self.maker.on_tx_confirmed(self, confirmations, txid, balance, added_utxos) self.maker.modify_orders(to_cancel, to_announce) def verify_unsigned_tx(self, txd): - tx_utxos = set([ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) for ins in txd['ins']]) - if not tx_utxos.issuperset(set(self.utxos)): + tx_utxo_set = set([ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) for ins in txd['ins']]) + my_uxto_set = set(self.utxos) + wallet_uxtos = set(self.wallet.unspent) + if not tx_utxo_set.issuperset(my_utxo_set): return False, 'my utxos are not contained' + if not wallet.utxos.issuperset(my_utxo_set): + return False, 'my utxos already spent' my_total_in = 0 for u in self.utxos: usvals = self.maker.wallet.unspent[u] @@ -135,6 +145,7 @@ def __init__(self, wallet): self.wallet = wallet self.nextoid = -1 self.orderlist = self.create_my_orders() + self.wallet_unspent_lock = threading.Lock() def privmsg_all_orders(self, target, orderlist=None): if orderlist == None: @@ -169,17 +180,6 @@ def on_privmsg(self, nick, message): try: if len(chunks) < 2: self.send_error(nick, 'Not enough arguments') - if chunks[0] == 'auth': - if nick not in self.active_orders or self.active_orders[nick] == None: - self.send_error(nick, 'No open order from this nick') - cjorder = self.active_orders[nick] - try: - i_utxo_pubkey = chunks[1] - btc_sig = chunks[2] - except (ValueError,IndexError) as e: - self.send_error(nick, str(e)) - self.active_orders[nick].auth_counterparty(nick, i_utxo_pubkey, btc_sig) - if chunks[0] == 'fill': if nick in self.active_orders and self.active_orders[nick] != None: self.active_orders[nick] = None @@ -191,12 +191,31 @@ def on_privmsg(self, nick, message): taker_pk = chunks[3] except (ValueError, IndexError) as e: self.send_error(nick, str(e)) - self.active_orders[nick] = CoinJoinOrder(self, nick, oid, amount, taker_pk) + self.wallet_unspent_lock.acquire() + try: + self.active_orders[nick] = CoinJoinOrder(self, nick, oid, amount, taker_pk) + finally: + self.wallet_unspent_lock.release() + elif chunks[0] == 'auth': + if nick not in self.active_orders or self.active_orders[nick] == None: + self.send_error(nick, 'No open order from this nick') + cjorder = self.active_orders[nick] + try: + i_utxo_pubkey = chunks[1] + btc_sig = chunks[2] + except (ValueError,IndexError) as e: + self.send_error(nick, str(e)) + self.active_orders[nick].auth_counterparty(nick, i_utxo_pubkey, btc_sig) + elif chunks[0] == 'tx': if nick not in self.active_orders or self.active_orders[nick] == None: self.send_error(nick, 'No open order from this nick') b64tx = chunks[1] - self.active_orders[nick].recv_tx(nick, b64tx) + self.wallet_unspent_lock.acquire() + try: + self.active_orders[nick].recv_tx(nick, b64tx) + finally: + self.wallet_unspent_lock.release() except CJMakerOrderError: self.active_orders[nick] = None continue From 8d1247c6ccd2ae9466eea701ff50a424a9b958c6 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Sat, 31 Jan 2015 20:07:33 +0200 Subject: [PATCH 102/409] refactor encryption, crypto-boxes are properties of CoinJoinTx and CoinJoinOrder --- common.py | 4 +- enc_wrapper.py | 2 +- encryption_protocol.txt | 7 ++++ irclib.py | 48 ---------------------- maker.py | 39 +++++++++--------- taker.py | 88 ++++++++++++++++++++++++----------------- 6 files changed, 81 insertions(+), 107 deletions(-) diff --git a/common.py b/common.py index 5a7dfa80..cadc4ab6 100644 --- a/common.py +++ b/common.py @@ -6,7 +6,7 @@ import threading HOST = 'irc.freenode.net' -CHANNEL = '#joinmarket-pit-test' +CHANNEL = '#joinmarket-pit-test2' PORT = 6667 #for the mainnet its #joinmarket-pit @@ -16,7 +16,7 @@ MAX_PRIVMSG_LEN = 400 ordername_list = ["absorder", "relorder"] - +valid_commands = ["absorder", "relorder", "fill", "pubkey", "auth", "tx", "sig", "error", "orderbook"] def debug(msg): print datetime.datetime.now().strftime("[%Y/%m/%d %H:%M:%S] ") + msg diff --git a/enc_wrapper.py b/enc_wrapper.py index d355e845..aa21cf67 100644 --- a/enc_wrapper.py +++ b/enc_wrapper.py @@ -5,7 +5,7 @@ #symmetric encryption. import libnacl.public -import binascii +import binascii, base64 def init_keypair(fname = None): '''Create a new encryption diff --git a/encryption_protocol.txt b/encryption_protocol.txt index 82ad8b27..bef83d0b 100644 --- a/encryption_protocol.txt +++ b/encryption_protocol.txt @@ -8,6 +8,8 @@ TAK: !fill MAK: !pubkey Both maker and taker construct a crypto Box object to allow authenticated encryption between the parties. +These Box objects are properties of the CoinJoinTx and CoinJoinOrder objects, so they are specific to +transactions and not to Maker and Taker entities. Encrypted ========= @@ -21,6 +23,11 @@ Because the !auth messages are under encryption, there is no privacy leak of bit If both verifications pass, the remainder of the messages exchanged between the two parties will continue under encryption. +Specifically, these message types will be encrypted: +!auth +!tx +!sig + Note ==== A key part of the authorisation process is the matching between the bitcoin pubkeys used in the coinjoin diff --git a/irclib.py b/irclib.py index 983321a7..be5ad925 100644 --- a/irclib.py +++ b/irclib.py @@ -55,54 +55,6 @@ def on_disconnect(self): pass def on_connect(self): pass #TODO implement on_nick_change - ############################### - #Encryption code - ############################### - def init_encryption(self): - self.enc_kp = enc_wrapper.init_keypair() - - def start_encryption(self,nick,c_pk_hex): - '''sets encryption mode on - for all succeeding messages - until end_encryption is called. - Public key of counterparty must be - passed in in hex.''' - if not self.enc_kp: - raise Exception("Cannot initialise encryption without a keypair") - self.cp_pubkeys[nick] = enc_wrapper.init_pubkey(c_pk_hex) - self.enc_boxes[nick] = enc_wrapper.as_init_encryption(self.enc_kp, self.cp_pubkeys[nick]) - self.encrypting[nick]= True - - def end_encryption(self, nick): - self.encrypting[nick]=False - #for safety, blank out all data related - #to the connection - self.cp_pubkeys[nick]=None - self.enc_boxes[nick]=None - - def end_all_encryption(self): - for k,v in self.encrypting.iteritems(): - if v: self.end_encryption(k) - - def encrypt_encode(self,msg,nick): - if not (nick in self.encrypting.keys()) or not (nick in self.enc_boxes.keys()): - raise Exception("Encryption is not switched on.") - if not (self.encrypting[nick] and self.enc_boxes[nick]): - raise Exception("Encryption is not switched on.") - encrypted = self.enc_boxes[nick].encrypt(msg) - return base64.b64encode(encrypted) - - def decode_decrypt(self,msg,nick): - if not (nick in self.encrypting.keys()) or not (nick in self.enc_boxes.keys()): - raise Exception("Encryption is not switched on.") - if not (self.encrypting[nick] and self.enc_boxes[nick]): - raise Exception("Encryption is not switched on.") - decoded = base64.b64decode(msg) - return self.enc_boxes[nick].decrypt(decoded) - ############################# - #End encryption code - ############################# - def close(self): try: self.send_raw("QUIT") diff --git a/maker.py b/maker.py index cba94aae..b13f0d85 100644 --- a/maker.py +++ b/maker.py @@ -16,7 +16,7 @@ def __init__(self, maker, nick, oid, amount, taker_pk): #create DH keypair on the fly for this Order object self.kp = enc_wrapper.init_keypair() #the encryption channel crypto box for this Order object - self.crypto_box = enc_wrapper.as_init_encryption(self.kp, self.taker_pk) + self.crypto_box = enc_wrapper.as_init_encryption(self.kp, enc_wrapper.init_pubkey(self.taker_pk)) order_s = [o for o in maker.orderlist if o['oid'] == oid] if len(order_s) == 0: @@ -33,14 +33,13 @@ def __init__(self, maker, nick, oid, amount, taker_pk): #always a new address even if the order ends up never being # furfilled, you dont want someone pretending to fill all your # orders to find out which addresses you use - pubkeymsg = command_prefix+ 'pubkey ' + self.kp.hex_pk() - self.send_priv(nick,pubkeymsg,False) + self.send_priv(nick, '!pubkey', self.kp.hex_pk(), False) - def send_priv(self, nick, msg, enc=False): + def send_priv(self, nick, cmd, msg, enc=False): if enc: - self.maker.privmsg(nick,enc_wrapper.encrypt_encode(msg, self.crypto_box)) + self.maker.privmsg(nick, cmd + ' ' + enc_wrapper.encrypt_encode(msg, self.crypto_box)) else: - self.maker.privmsg(nick,msg) + self.maker.privmsg(nick, cmd + ' ' + msg) def auth_counterparty(self,nick,i_utxo_pubkey,btc_sig): #TODO: add check that the pubkey's address is part of the order. @@ -54,10 +53,10 @@ def auth_counterparty(self,nick,i_utxo_pubkey,btc_sig): #TODO the next 2 lines are a little inefficient. btc_key = self.maker.wallet.get_key_from_addr(self.cj_addr) btc_pub = btc.privtopub(btc_key) - btc_sig = btc.ecdsa_sign(self.maker.enc_kp.hex_pk(),btc_key) - - self.maker.privmsg(nick, command_prefix + 'auth ' + str(','.join(self.utxos)) + ' ' + \ - btc_pub + ' ' + self.change_addr + ' ' + btc_sig) + btc_sig = btc.ecdsa_sign(self.kp.hex_pk(),btc_key) + authmsg = str(','.join(self.utxos)) + ' ' + \ + btc_pub + ' ' + self.change_addr + ' ' + btc_sig + self.send_priv(nick, '!auth', authmsg, True) return True def recv_tx(self, nick, b64tx): @@ -74,6 +73,7 @@ def recv_tx(self, nick, b64tx): if not goodtx: debug('not a good tx, reason=' + errmsg) self.maker.send_error(nick, errmsg) + #TODO: the above 3 errors should be encrypted, but it's a bit messy. debug('goodtx') sigs = [] for index, ins in enumerate(self.tx['ins']): @@ -87,9 +87,8 @@ def recv_tx(self, nick, b64tx): add_addr_notify(self.change_addr, self.unconfirm_callback, self.confirm_callback) debug('sending sigs ' + str(sigs)) - self.maker.privmsg(nick, ''.join([command_prefix + 'sig ' + s for s in sigs])) - #once signature is sent, close encrypted channel to this taker. - self.maker.end_encryption(nick) + for s in sigs: + self.send_priv(nick, '!sig', s, True) self.maker.active_orders[nick] = None def unconfirm_callback(self, balance): @@ -141,7 +140,6 @@ class CJMakerOrderError(StandardError): class Maker(irclib.IRCClient): def __init__(self, wallet): - self.init_encryption() self.active_orders = {} self.wallet = wallet self.nextoid = -1 @@ -168,7 +166,7 @@ def send_error(self, nick, errmsg): def on_welcome(self): self.privmsg_all_orders(CHANNEL) - + def on_privmsg(self, nick, message): if message[0] != command_prefix: return @@ -184,9 +182,11 @@ def on_privmsg(self, nick, message): if nick not in self.active_orders or self.active_orders[nick] == None: self.send_error(nick, 'No open order from this nick') cjorder = self.active_orders[nick] + encmsg = enc_wrapper.decode_decrypt(chunks[1],self.active_orders[nick].crypto_box) + encrypted_chunks = encmsg.split(" ") try: - i_utxo_pubkey = chunks[1] - btc_sig = chunks[2] + i_utxo_pubkey = encrypted_chunks[0] + btc_sig = encrypted_chunks[1] except (ValueError,IndexError) as e: self.send_error(nick, str(e)) self.active_orders[nick].auth_counterparty(nick, i_utxo_pubkey, btc_sig) @@ -205,8 +205,8 @@ def on_privmsg(self, nick, message): elif chunks[0] == 'tx': if nick not in self.active_orders or self.active_orders[nick] == None: self.send_error(nick, 'No open order from this nick') - b64tx = chunks[1] - self.active_orders[nick].recv_tx(nick, b64tx) + encb64tx = chunks[1] + self.active_orders[nick].recv_tx(nick, enc_wrapper.decode_decrypt(encb64tx,self.active_orders[nick].crypto_box)) except CJMakerOrderError: self.active_orders[nick] = None continue @@ -229,7 +229,6 @@ def on_set_topic(self, newtopic): print '=' * 60 def on_leave(self, nick): - self.end_encryption(nick) self.active_orders[nick] = None def modify_orders(self, to_cancel, to_announce): diff --git a/taker.py b/taker.py index 4396efa9..01f51a35 100644 --- a/taker.py +++ b/taker.py @@ -1,6 +1,7 @@ #! /usr/bin/env python from common import * +import enc_wrapper import irclib import bitcoin as btc @@ -27,14 +28,55 @@ def __init__(self, taker, cj_amount, orders, my_utxos, my_cj_addr, self.my_change_addr = my_change_addr self.cjfee_total = 0 self.latest_tx = None + #create DH keypair on the fly for this Tx object + self.kp = enc_wrapper.init_keypair() + self.crypto_boxes = {} #find the btc pubkey of the first utxo being used self.signing_btc_add = taker.wallet.unspent[self.my_utxos[0]]['address'] self.signing_btc_pub = btc.privtopub(taker.wallet.get_key_from_addr(self.signing_btc_add)) for c, oid in orders.iteritems(): - taker.privmsg(c, command_prefix + 'fill ' + \ - str(oid) + ' ' + str(cj_amount) + ' ' + taker.enc_kp.hex_pk()) + cmd = command_prefix + 'fill' + omsg = \ + str(oid) + ' ' + str(cj_amount) + ' ' + self.kp.hex_pk() + self.send_priv(c, cmd, omsg) - def recv_txio(self, nick, utxo_list, cj_pub, change_addr): + def send_priv(self, nick, cmd, msg, enc=False): + if enc: + self.taker.privmsg(nick, cmd + ' ' + enc_wrapper.encrypt_encode(msg, self.crypto_boxes[nick][1])) + else: + self.taker.privmsg(nick, cmd + ' ' + msg) + + def start_encryption(self, nick, maker_pk): + if nick not in self.active_orders.keys(): + raise Exception("Counterparty not part of this transaction.") + self.crypto_boxes[nick] = [maker_pk,enc_wrapper.as_init_encryption(self.kp, enc_wrapper.init_pubkey(maker_pk))] + #send authorisation request + my_btc_priv = self.taker.wallet.get_key_from_addr(self.taker.wallet.unspent[self.my_utxos[0]]['address']) + my_btc_pub = btc.privtopub(my_btc_priv) + my_btc_sig = btc.ecdsa_sign(self.kp.hex_pk(), my_btc_priv) + message = my_btc_pub + ' ' + my_btc_sig + self.send_priv(nick, '!auth', message, True) + + def auth_counterparty(self, nick, btc_sig, cj_pub): + '''Validate the counterpartys claim to own the btc + address/pubkey that will be used for coinjoining + with an ecdsa verification.''' + if not btc.ecdsa_verify(self.crypto_boxes[nick][0], btc_sig, cj_pub): + print 'signature didnt match pubkey and message' + return False + return True + + def recv_txio(self, nick, msg): + decrypted = enc_wrapper.decode_decrypt(msg, self.crypto_boxes[nick][1]) + chunks = decrypted.split(' ') + utxo_list = chunks[0].split(',') + cj_pub = chunks[1] + change_addr = chunks[2] + btc_sig = chunks[3] + if not self.auth_counterparty(nick, btc_sig, cj_pub): + print 'Authenticated encryption with counterparty: ' + nick + \ + ' not established. TODO: send rejection message' + return cj_addr = btc.pubtoaddr(cj_pub,get_addr_vbyte()) if nick not in self.nonrespondants: debug('nick(' + nick + ') not in nonrespondants ' + str(self.nonrespondants)) @@ -78,7 +120,7 @@ def recv_txio(self, nick, utxo_list, cj_pub, change_addr): debug('obtained tx\n' + pprint.pformat(btc.deserialize(tx))) txb64 = base64.b64encode(tx.decode('hex')) for nickk in self.active_orders.keys(): - self.taker.privmsg(nickk, command_prefix + 'tx ' + txb64) + self.send_priv(nickk, command_prefix + 'tx',txb64, True) #now sign it ourselves here for index, ins in enumerate(btc.deserialize(tx)['ins']): @@ -91,7 +133,8 @@ def recv_txio(self, nick, utxo_list, cj_pub, change_addr): tx = btc.sign(tx, index, self.taker.wallet.get_key_from_addr(addr)) self.latest_tx = btc.deserialize(tx) - def add_signature(self, sigb64): + def add_signature(self, nick, encsig): + sigb64 = enc_wrapper.decode_decrypt(encsig,self.crypto_boxes[nick][1]) sig = base64.b64decode(sigb64).encode('hex') inserted_sig = False @@ -120,8 +163,7 @@ def add_signature(self, sigb64): if not tx_signed: return debug('the entire tx is signed, ready to pushtx()') - #end encryption channel with all counterparties - self.taker.end_all_encryption() + print btc.serialize(self.latest_tx) ret = btc.blockr_pushtx(btc.serialize(self.latest_tx), get_network()) debug('pushed tx ' + str(ret)) @@ -204,16 +246,6 @@ def __init__(self): #TODO have a list of maker's nick we're coinjoining with, so # that some other guy doesnt send you confusing stuff #maybe a start_cj_tx() method is needed - self.init_encryption() - - def auth_counterparty(self,nick,btc_sig,cj_pub): - '''Validate the counterpartys claim to own the btc - address/pubkey that will be used for coinjoining - with an ecdsa verification.''' - if not btc.ecdsa_verify(self.maker_pks[nick],btc_sig,cj_pub): - print 'signature didnt match pubkey and message' - return False - return True def on_privmsg(self, nick, message): OrderbookWatch.on_privmsg(self, nick, message) @@ -224,28 +256,12 @@ def on_privmsg(self, nick, message): chunks = command.split(" ") if chunks[0] == 'pubkey': maker_pk = chunks[1] - #store the declared pubkeys in a dict indexed by maker nick - self.maker_pks[nick] = maker_pk - self.start_encryption(nick, self.maker_pks[nick]) - #send authorisation request - my_btc_priv = self.wallet.get_key_from_addr(self.wallet.unspent[self.cjtx.my_utxos[0]]['address']) - my_btc_pub = btc.privtopub(my_btc_priv) - my_btc_sig = btc.ecdsa_sign(self.enc_kp.hex_pk(),my_btc_priv) - message = '!auth ' + my_btc_pub + ' ' + my_btc_sig - self.privmsg(nick,message) #note: we do this *before* starting encryption + self.cjtx.start_encryption(nick, maker_pk) if chunks[0] == 'auth': - utxo_list = chunks[1].split(',') - cj_pub = chunks[2] - change_addr = chunks[3] - btc_sig = chunks[4] - if not self.auth_counterparty(nick,btc_sig,cj_pub): - print 'Authenticated encryption with counterparty: ' + nick + \ - ' not established. TODO: send rejection message' - continue - self.cjtx.recv_txio(nick, utxo_list, cj_pub, change_addr) + self.cjtx.recv_txio(nick, chunks[1]) elif chunks[0] == 'sig': sig = chunks[1] - self.cjtx.add_signature(sig) + self.cjtx.add_signature(nick,sig) my_tx_fee = 10000 From e96606d4acd24f732daa9b0b1b8c435c5e110db7 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sat, 31 Jan 2015 20:01:40 +0000 Subject: [PATCH 103/409] silly bugfixes --- maker.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/maker.py b/maker.py index 176b3edd..a6ccaf1a 100644 --- a/maker.py +++ b/maker.py @@ -80,21 +80,21 @@ def recv_tx(self, nick, b64tx): self.maker.active_orders[nick] = None def unconfirm_callback(self, balance): - self.wallet_unspent_lock.acquire() + self.maker.wallet_unspent_lock.acquire() try: removed_utxos = self.maker.wallet.remove_old_utxos(self.tx) finally: - self.wallet_unspent_lock.release() + self.maker.wallet_unspent_lock.release() debug('saw tx on network, removed_utxos=\n' + pprint.pformat(removed_utxos)) to_cancel, to_announce = self.maker.on_tx_unconfirmed(self, balance, removed_utxos) self.maker.modify_orders(to_cancel, to_announce) def confirm_callback(self, confirmations, txid, balance): - self.wallet_unspent_lock.acquire() + self.maker.wallet_unspent_lock.acquire() try: added_utxos = self.maker.wallet.add_new_utxos(self.tx, txid) finally: - self.wallet_unspent_lock.release() + self.maker.wallet_unspent_lock.release() debug('tx in a block, added_utxos=\n' + pprint.pformat(added_utxos)) to_cancel, to_announce = self.maker.on_tx_confirmed(self, confirmations, txid, balance, added_utxos) @@ -102,11 +102,11 @@ def confirm_callback(self, confirmations, txid, balance): def verify_unsigned_tx(self, txd): tx_utxo_set = set([ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) for ins in txd['ins']]) - my_uxto_set = set(self.utxos) - wallet_uxtos = set(self.wallet.unspent) + my_utxo_set = set(self.utxos) + wallet_utxos = set(self.maker.wallet.unspent) if not tx_utxo_set.issuperset(my_utxo_set): return False, 'my utxos are not contained' - if not wallet.utxos.issuperset(my_utxo_set): + if not wallet_utxos.issuperset(my_utxo_set): return False, 'my utxos already spent' my_total_in = 0 for u in self.utxos: From e427070c8b9eaa7c4bd536d6af04906e936b1203 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Sat, 31 Jan 2015 22:43:15 +0200 Subject: [PATCH 104/409] final adjustments to merge --- common.py | 2 +- maker.py | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/common.py b/common.py index fe23c65a..27dc06fa 100644 --- a/common.py +++ b/common.py @@ -16,7 +16,7 @@ MAX_PRIVMSG_LEN = 400 ordername_list = ["absorder", "relorder"] -valid_commands = ["absorder", "relorder", "fill", "pubkey", "auth", "tx", "sig", "error", "orderbook"] + def debug(msg): print datetime.datetime.now().strftime("[%Y/%m/%d %H:%M:%S] ") + msg diff --git a/maker.py b/maker.py index d29e1693..b792faef 100644 --- a/maker.py +++ b/maker.py @@ -190,10 +190,7 @@ def on_privmsg(self, nick, message): try: if len(chunks) < 2: self.send_error(nick, 'Not enough arguments') - encmsg = enc_wrapper.decode_decrypt(chunks[1],self.active_orders[nick].crypto_box) - encrypted_chunks = encmsg.split(" ") - i_utxo_pubkey = encrypted_chunks[0] - btc_sig = encrypted_chunks[1] + if chunks[0] == 'fill': if nick in self.active_orders and self.active_orders[nick] != None: self.active_orders[nick] = None @@ -213,9 +210,11 @@ def on_privmsg(self, nick, message): if nick not in self.active_orders or self.active_orders[nick] == None: self.send_error(nick, 'No open order from this nick') cjorder = self.active_orders[nick] + encmsg = enc_wrapper.decode_decrypt(chunks[1],self.active_orders[nick].crypto_box) + encrypted_chunks = encmsg.split(" ") try: - i_utxo_pubkey = chunks[1] - btc_sig = chunks[2] + i_utxo_pubkey = encrypted_chunks[0] + btc_sig = encrypted_chunks[1] except (ValueError,IndexError) as e: self.send_error(nick, str(e)) self.active_orders[nick].auth_counterparty(nick, i_utxo_pubkey, btc_sig) @@ -226,7 +225,7 @@ def on_privmsg(self, nick, message): encb64tx = chunks[1] self.wallet_unspent_lock.acquire() try: - self.active_orders[nick].recv_tx(nick, enc_wrapper.decode_decrypt(encb64tx,self.active_orders[nick].crypto_box)) + self.active_orders[nick].recv_tx(nick, enc_wrapper.decode_decrypt(encb64tx, self.active_orders[nick].crypto_box)) finally: self.wallet_unspent_lock.release() except CJMakerOrderError: From 57e6fa8a571d5084e4eef5ab54d280d71364d1c7 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Sat, 31 Jan 2015 22:49:46 +0200 Subject: [PATCH 105/409] channel name fix --- common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common.py b/common.py index 27dc06fa..fb94d729 100644 --- a/common.py +++ b/common.py @@ -6,7 +6,7 @@ import threading HOST = 'irc.freenode.net' -CHANNEL = '#joinmarket-pit-test2' +CHANNEL = '#joinmarket-pit-test' PORT = 6667 #for the mainnet its #joinmarket-pit From 46124a230a0fa75c79dfb0a6c42ebacf2bee668a Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Sun, 1 Feb 2015 16:16:36 +0200 Subject: [PATCH 106/409] removed old encryption vars from irclib --- irclib.py | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/irclib.py b/irclib.py index be5ad925..48beca19 100644 --- a/irclib.py +++ b/irclib.py @@ -1,7 +1,6 @@ import socket, threading, time from common import debug, chunks -import enc_wrapper import base64, os PING_INTERVAL = 40 @@ -70,18 +69,12 @@ def pubmsg(self, message): self.send_raw("PRIVMSG " + self.channel + " :" + message) def privmsg(self, nick, message): - clearmsg = message - will_encrypt = False - if nick in self.encrypting.keys() and self.encrypting[nick]: - message = self.encrypt_encode(message, nick) - will_encrypt = True - debug('>>privmsg ' + ('enc ' if will_encrypt else '') + 'nick=' + nick + ' msg=' + clearmsg) + debug('>>privmsg ' + 'nick=' + nick + ' msg=' + message) if len(message) > 350: message_chunks = chunks(message, 350) else: message_chunks = [message] - #print "We are going to send these chunks: " - #print message_chunks + for m in message_chunks: trailer = ' ~' if m==message_chunks[-1] else ' ;' self.send_raw("PRIVMSG " + nick + " :" + m + trailer) @@ -110,9 +103,7 @@ def __handle_privmsg(self, source, target, message): self.waiting[nick]=True elif message[-1]=='~': self.waiting[nick]=False - if nick in self.encrypting.keys() and self.encrypting[nick]: - parsed = self.decode_decrypt(self.built_privmsg[nick],nick) - else: parsed = self.built_privmsg[nick] + parsed = self.built_privmsg[nick] #wipe the message buffer waiting for the next one self.built_privmsg[nick]='' debug("< some coinjoin tools we use today were broken -<> one allowed people to use a mix of uncompressed and compressed keys, so it was obvious which party was which. - -TODO -probably a good idea to have a debug.log where loads of information is dumped - -#TODO add random delays to the orderbook stuff so there isnt such a traffic spike when a new bot joins -#two options, random delay !orderbook for ones which dont mind, !orderbook without delay for bots -# which need the orders asap - -TODO -code something that extends orderbookwatch and creates graphs - those graphs can be posted to a bitcointalk thread (like the bitstamp wall watch thread) - and could be a nice historical record and guide to pricing - -TODO -code something that analyzes the blockchain, detects coinjoin tx likely made by joinmarket - and calculates the paid fee, therefore is a guide to pricing - -TODO -the add_addr_notify() stuff doesnt work, so if theres several CoinJoinOrder's open it will start a few - threads to do the notifying, they could race condition or other multithreaded errors -i suggest to create a single thread that sorts out all the stuff - -#TODO make an ordertype where maker publishes the utxo he will use -# this is a way to auction off the use of a desirable coin, maybe a -# very newly mined coin or one which hasnt been moved for years From b592f07128ff9e15d44d014db75d91e8fdb6cd64 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 1 Feb 2015 20:34:48 +0000 Subject: [PATCH 108/409] Revert "moved all TODOs to github issues", shouldnt be in msgchan branch This reverts commit 400fac10170d6fca0490a444facfac212d097307. --- TODO | 114 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 TODO diff --git a/TODO b/TODO new file mode 100644 index 00000000..265c3f8f --- /dev/null +++ b/TODO @@ -0,0 +1,114 @@ + +#TODO +#ask people on the testnet stuff to code up a few trading algos to see if the interface/protocol that +# iv invented is general enough +a few algos: +fees proportional to how many utxos used, since the marginal cost is unrelated to your cj amount, only to + the amount of utxos you use up + +#TODO dont always pick the lowest cost order, instead have an exponentially decaying +# distribution, so most of the time you pick the lowest and sometimes you take higher ones +# this represents your uncertainty in sybil attackers, the cheapest may not always be the best +#i.e. randomly chosen makers, weighted by the price they offer + +#TODO on nickname change, change also the counterparty variable in any open orders + +#TODO use electrum json_rpc instead of the pybitcointools stuff +# problem, i dont think that supports testnet +# bitcoind json_rpc obviously supports testnet, but someone else can download +# the blockchain, actually it seems you cant replace pybitcointools with bitcoind +# cant look up any txid or address +# could use a websocket api for learning when new blocks/tx appear +# could use python-bitcoinlib to be a node in the p2p network + +#TODO option for how many blocks deep to wait before using a utxo for more mixing +# 1 confirm is probably enough +TODO +have the taker enforce this, look up the txhash of the maker's utxo and make sure + it is already in a block + +TODO implement rate limiting for irc.privmsg to stop the bot being killed due to flood +i suggest creating a thread that only dispatches/writes to the irc socket + +TODO have an option for sendpayment.py take advantage of the small orders posted +by patientsendpayment.py + +TODO patientsendpayment.py needs a different algo +if someone fills the entire order all at once, they pay very little + +TODO sort out the nick = nick + '_' stuff in irclib +its not a good way of doing it + +TODO +bug in sweep, doesnt work for N=1 (just one maker) + +TODO +implement sasl in the irc code, required for freenode over tor and better than sending nickserv :identify + +TODO +robustness in the taker code +for instance if the maker being joined with quits halfway through + +#TODO implement something against dust +# e.g. where the change address ends up having an output of value 1000 satoshis + +#TODO completely abstract away the irc stuff, so it can be switched to something else +# e.g. twitter but more likely darkwallet obelisk and/or electrum server + +TODO combine the taker and maker code into one file where you can make different kinds of + bot which combine both roles +e.g. tumbler.py repeatedly takes orders on the same coins again and again in an effort + to improve privacy and break the link between them, make sure to split up and combine them again + in random amounts, because the yield-generator will also be splitting and combining coins + random intervals between blocks included might be worth it too, since yield-generator.py + will appear to have coins which dont get mixed again for a while +e.g. patient-tumbler.py which waits a while being a maker, then just starts to take orders + after a time limit for people who want to mix coins but dont mind waiting until a fixed upper time limit +e.g. yield-generator.py which acts as a maker solely for the purpose of making money + might need to take orders at some point, for very small outputs which have a small probability of being filled +e.g. single-tx.py which takes a single order, using it to send coins to some address + typically as a payment, so this is what the electrum plugin would look like +e.g. patient-single-tx.py which does the above but doesnt mind waiting up to a limit +e.g. gui-taker.py has a gui which shows the user the orderbook and they can easily fill and order + and see other statistics, could be easily done by opening a http port and sending a html form and graphics + +TODO +implement this the thing that gmaxwell wrote about in the original coinjoin post, as a kind of tumbler +"Isn't the anonymity set size limited by how many parties you can get in a single transaction?" + +"Not quite. The anonymity set size of a single transaction is limited by the number of parties in it, obviously. And transaction size limits as well as failure (retry) risk mean that really huge joint transactions would not be wise. But because these transactions are cheap, there is no limit to the number of transactions you can cascade. + +In particular, if you have can build transactions with m participants per transaction you can create a sequence of m*3 transactions which form a three-stage switching network that permits any of m^2 final outputs to have come from any of m^2 original inputs (e.g. using three stages of 32 transactions with 32 inputs each 1024 users can be joined with a total of 96 transactions). This allows the anonymity set to be any size, limited only by participation." +https://en.wikipedia.org/wiki/Clos_network +Not sure if it will actually be possible in this liquidity maker/taker system + +TODO need to move onto the bip44 structure of HD wallets + +TODO think about this +<> some coinjoin tools we use today were broken +<> one allowed people to use a mix of uncompressed and compressed keys, so it was obvious which party was which. + +TODO +probably a good idea to have a debug.log where loads of information is dumped + +#TODO add random delays to the orderbook stuff so there isnt such a traffic spike when a new bot joins +#two options, random delay !orderbook for ones which dont mind, !orderbook without delay for bots +# which need the orders asap + +TODO +code something that extends orderbookwatch and creates graphs + those graphs can be posted to a bitcointalk thread (like the bitstamp wall watch thread) + and could be a nice historical record and guide to pricing + +TODO +code something that analyzes the blockchain, detects coinjoin tx likely made by joinmarket + and calculates the paid fee, therefore is a guide to pricing + +TODO +the add_addr_notify() stuff doesnt work, so if theres several CoinJoinOrder's open it will start a few + threads to do the notifying, they could race condition or other multithreaded errors +i suggest to create a single thread that sorts out all the stuff + +#TODO make an ordertype where maker publishes the utxo he will use +# this is a way to auction off the use of a desirable coin, maybe a +# very newly mined coin or one which hasnt been moved for years From d4a8491cfbd2d7670a91853ceb2a265a07008ae6 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 1 Feb 2015 20:36:27 +0000 Subject: [PATCH 109/409] moved TODOs to github issues --- TODO | 114 ----------------------------------------------------------- 1 file changed, 114 deletions(-) delete mode 100644 TODO diff --git a/TODO b/TODO deleted file mode 100644 index 265c3f8f..00000000 --- a/TODO +++ /dev/null @@ -1,114 +0,0 @@ - -#TODO -#ask people on the testnet stuff to code up a few trading algos to see if the interface/protocol that -# iv invented is general enough -a few algos: -fees proportional to how many utxos used, since the marginal cost is unrelated to your cj amount, only to - the amount of utxos you use up - -#TODO dont always pick the lowest cost order, instead have an exponentially decaying -# distribution, so most of the time you pick the lowest and sometimes you take higher ones -# this represents your uncertainty in sybil attackers, the cheapest may not always be the best -#i.e. randomly chosen makers, weighted by the price they offer - -#TODO on nickname change, change also the counterparty variable in any open orders - -#TODO use electrum json_rpc instead of the pybitcointools stuff -# problem, i dont think that supports testnet -# bitcoind json_rpc obviously supports testnet, but someone else can download -# the blockchain, actually it seems you cant replace pybitcointools with bitcoind -# cant look up any txid or address -# could use a websocket api for learning when new blocks/tx appear -# could use python-bitcoinlib to be a node in the p2p network - -#TODO option for how many blocks deep to wait before using a utxo for more mixing -# 1 confirm is probably enough -TODO -have the taker enforce this, look up the txhash of the maker's utxo and make sure - it is already in a block - -TODO implement rate limiting for irc.privmsg to stop the bot being killed due to flood -i suggest creating a thread that only dispatches/writes to the irc socket - -TODO have an option for sendpayment.py take advantage of the small orders posted -by patientsendpayment.py - -TODO patientsendpayment.py needs a different algo -if someone fills the entire order all at once, they pay very little - -TODO sort out the nick = nick + '_' stuff in irclib -its not a good way of doing it - -TODO -bug in sweep, doesnt work for N=1 (just one maker) - -TODO -implement sasl in the irc code, required for freenode over tor and better than sending nickserv :identify - -TODO -robustness in the taker code -for instance if the maker being joined with quits halfway through - -#TODO implement something against dust -# e.g. where the change address ends up having an output of value 1000 satoshis - -#TODO completely abstract away the irc stuff, so it can be switched to something else -# e.g. twitter but more likely darkwallet obelisk and/or electrum server - -TODO combine the taker and maker code into one file where you can make different kinds of - bot which combine both roles -e.g. tumbler.py repeatedly takes orders on the same coins again and again in an effort - to improve privacy and break the link between them, make sure to split up and combine them again - in random amounts, because the yield-generator will also be splitting and combining coins - random intervals between blocks included might be worth it too, since yield-generator.py - will appear to have coins which dont get mixed again for a while -e.g. patient-tumbler.py which waits a while being a maker, then just starts to take orders - after a time limit for people who want to mix coins but dont mind waiting until a fixed upper time limit -e.g. yield-generator.py which acts as a maker solely for the purpose of making money - might need to take orders at some point, for very small outputs which have a small probability of being filled -e.g. single-tx.py which takes a single order, using it to send coins to some address - typically as a payment, so this is what the electrum plugin would look like -e.g. patient-single-tx.py which does the above but doesnt mind waiting up to a limit -e.g. gui-taker.py has a gui which shows the user the orderbook and they can easily fill and order - and see other statistics, could be easily done by opening a http port and sending a html form and graphics - -TODO -implement this the thing that gmaxwell wrote about in the original coinjoin post, as a kind of tumbler -"Isn't the anonymity set size limited by how many parties you can get in a single transaction?" - -"Not quite. The anonymity set size of a single transaction is limited by the number of parties in it, obviously. And transaction size limits as well as failure (retry) risk mean that really huge joint transactions would not be wise. But because these transactions are cheap, there is no limit to the number of transactions you can cascade. - -In particular, if you have can build transactions with m participants per transaction you can create a sequence of m*3 transactions which form a three-stage switching network that permits any of m^2 final outputs to have come from any of m^2 original inputs (e.g. using three stages of 32 transactions with 32 inputs each 1024 users can be joined with a total of 96 transactions). This allows the anonymity set to be any size, limited only by participation." -https://en.wikipedia.org/wiki/Clos_network -Not sure if it will actually be possible in this liquidity maker/taker system - -TODO need to move onto the bip44 structure of HD wallets - -TODO think about this -<> some coinjoin tools we use today were broken -<> one allowed people to use a mix of uncompressed and compressed keys, so it was obvious which party was which. - -TODO -probably a good idea to have a debug.log where loads of information is dumped - -#TODO add random delays to the orderbook stuff so there isnt such a traffic spike when a new bot joins -#two options, random delay !orderbook for ones which dont mind, !orderbook without delay for bots -# which need the orders asap - -TODO -code something that extends orderbookwatch and creates graphs - those graphs can be posted to a bitcointalk thread (like the bitstamp wall watch thread) - and could be a nice historical record and guide to pricing - -TODO -code something that analyzes the blockchain, detects coinjoin tx likely made by joinmarket - and calculates the paid fee, therefore is a guide to pricing - -TODO -the add_addr_notify() stuff doesnt work, so if theres several CoinJoinOrder's open it will start a few - threads to do the notifying, they could race condition or other multithreaded errors -i suggest to create a single thread that sorts out all the stuff - -#TODO make an ordertype where maker publishes the utxo he will use -# this is a way to auction off the use of a desirable coin, maybe a -# very newly mined coin or one which hasnt been moved for years From 7a0f00b599be16a4bb3464d63bd340a9d64caa3c Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Mon, 2 Feb 2015 23:43:22 +0200 Subject: [PATCH 110/409] move encryption out of maker and taker classes --- common.py | 2 ++ irclib.py | 63 +++++++++++++++++++++++++++++++++++++++++++++---------- maker.py | 36 +++++++++++-------------------- taker.py | 51 +++++++++++++++++++------------------------- 4 files changed, 87 insertions(+), 65 deletions(-) diff --git a/common.py b/common.py index fb94d729..f80017b1 100644 --- a/common.py +++ b/common.py @@ -16,6 +16,8 @@ MAX_PRIVMSG_LEN = 400 ordername_list = ["absorder", "relorder"] +encrypted_commands = ["auth", "ioauth", "tx", "sig"] +plaintext_commands = ["fill", "error", "pubkey", "orderbook", "relorder", "absorder"] def debug(msg): print datetime.datetime.now().strftime("[%Y/%m/%d %H:%M:%S] ") + msg diff --git a/irclib.py b/irclib.py index 48beca19..6bdbdae9 100644 --- a/irclib.py +++ b/irclib.py @@ -1,7 +1,8 @@ import socket, threading, time -from common import debug, chunks +from common import * import base64, os +import enc_wrapper PING_INTERVAL = 40 PING_TIMEOUT = 10 @@ -54,6 +55,29 @@ def on_disconnect(self): pass def on_connect(self): pass #TODO implement on_nick_change + def encrypting(self, cmd, nick, sending=False): + '''Establish whether the message is to be + encrypted/decrypted based on the command string. + If so, retrieve the appropriate crypto_box object + and return. Sending/receiving flag enables us + to check which command strings correspond to which + type of object (maker/taker).''' + + if cmd in plaintext_commands: + return None + elif cmd not in encrypted_commands: + raise Exception("Invalid command type: " + cmd) + + maker_strings = ['tx','auth'] if not sending else ['ioauth','sig'] + taker_strings = ['ioauth','sig'] if not sending else ['tx','auth'] + + if cmd in maker_strings: + return self.active_orders[nick].crypto_box + elif cmd in taker_strings: + return self.cjtx.crypto_boxes[nick][1] + else: + raise Exception("Invalid command type: " + cmd) + def close(self): try: self.send_raw("QUIT") @@ -68,8 +92,14 @@ def pubmsg(self, message): debug('>>pubmsg ' + message) self.send_raw("PRIVMSG " + self.channel + " :" + message) - def privmsg(self, nick, message): - debug('>>privmsg ' + 'nick=' + nick + ' msg=' + message) + def privmsg(self, nick, cmd, message): + debug('>>privmsg ' + 'nick=' + nick + 'cmd=' + cmd + ' msg=' + message) + #should we encrypt? + box = self.encrypting(cmd, nick, sending=True) + #encrypt before chunking + if box: + message = enc_wrapper.encrypt_encode(message,box) + if len(message) > 350: message_chunks = chunks(message, 350) else: @@ -77,7 +107,9 @@ def privmsg(self, nick, message): for m in message_chunks: trailer = ' ~' if m==message_chunks[-1] else ' ;' - self.send_raw("PRIVMSG " + nick + " :" + m + trailer) + header = "PRIVMSG " + nick + " :" + if m==message_chunks[0]: header += '!'+cmd + ' ' + self.send_raw(header + m + trailer) def send_raw(self, line): #if not line.startswith('PING LAG'): @@ -95,17 +127,26 @@ def __handle_privmsg(self, source, target, message): #TODO ctcp version here, since some servers dont let you get on without if target == self.nick: - if nick not in self.built_privmsg or self.built_privmsg[nick]=='': - self.built_privmsg[nick] = message[:-2] + if nick not in self.built_privmsg: + #new message starting + cmd_string = ''.join(message.split(' ')[0][1:]) + self.built_privmsg[nick] = [cmd_string, message[:-2]] else: - self.built_privmsg[nick] += message[:-2] + self.built_privmsg[nick][1] += message[:-2] + box = self.encrypting(self.built_privmsg[nick][0], nick) if message[-1]==';': self.waiting[nick]=True elif message[-1]=='~': - self.waiting[nick]=False - parsed = self.built_privmsg[nick] + self.waiting[nick]=False + if box: + #need to decrypt everything after the command string + to_decrypt = ''.join(self.built_privmsg[nick][1].split(' ')[1]) + decrypted = enc_wrapper.decode_decrypt(to_decrypt, box) + parsed = self.built_privmsg[nick][1].split(' ')[0] + ' ' + decrypted + else: + parsed = self.built_privmsg[nick][1] #wipe the message buffer waiting for the next one - self.built_privmsg[nick]='' + del self.built_privmsg[nick] debug("< MAX_PRIVMSG_LEN: - self.privmsg(target, orderline) - orderline = '' - if len(orderline) > 0: - self.privmsg(target, orderline) + self.privmsg(target,order['ordertype'],' '.join(elem_list)) def send_error(self, nick, errmsg): debug('error<%s> : %s' % (nick, errmsg)) - self.privmsg(nick, command_prefix + 'error ' + errmsg) + self.privmsg(nick,'error', errmsg) raise CJMakerOrderError() def on_welcome(self): @@ -210,11 +200,9 @@ def on_privmsg(self, nick, message): if nick not in self.active_orders or self.active_orders[nick] == None: self.send_error(nick, 'No open order from this nick') cjorder = self.active_orders[nick] - encmsg = enc_wrapper.decode_decrypt(chunks[1],self.active_orders[nick].crypto_box) - encrypted_chunks = encmsg.split(" ") try: - i_utxo_pubkey = encrypted_chunks[0] - btc_sig = encrypted_chunks[1] + i_utxo_pubkey = chunks[1] + btc_sig = chunks[2] except (ValueError,IndexError) as e: self.send_error(nick, str(e)) self.active_orders[nick].auth_counterparty(nick, i_utxo_pubkey, btc_sig) @@ -222,10 +210,10 @@ def on_privmsg(self, nick, message): elif chunks[0] == 'tx': if nick not in self.active_orders or self.active_orders[nick] == None: self.send_error(nick, 'No open order from this nick') - encb64tx = chunks[1] + b64tx = chunks[1] self.wallet_unspent_lock.acquire() try: - self.active_orders[nick].recv_tx(nick, enc_wrapper.decode_decrypt(encb64tx, self.active_orders[nick].crypto_box)) + self.active_orders[nick].recv_tx(nick, b64tx) finally: self.wallet_unspent_lock.release() except CJMakerOrderError: diff --git a/taker.py b/taker.py index 01f51a35..7eb078da 100644 --- a/taker.py +++ b/taker.py @@ -35,27 +35,22 @@ def __init__(self, taker, cj_amount, orders, my_utxos, my_cj_addr, self.signing_btc_add = taker.wallet.unspent[self.my_utxos[0]]['address'] self.signing_btc_pub = btc.privtopub(taker.wallet.get_key_from_addr(self.signing_btc_add)) for c, oid in orders.iteritems(): - cmd = command_prefix + 'fill' - omsg = \ - str(oid) + ' ' + str(cj_amount) + ' ' + self.kp.hex_pk() - self.send_priv(c, cmd, omsg) - - def send_priv(self, nick, cmd, msg, enc=False): - if enc: - self.taker.privmsg(nick, cmd + ' ' + enc_wrapper.encrypt_encode(msg, self.crypto_boxes[nick][1])) - else: - self.taker.privmsg(nick, cmd + ' ' + msg) + cmd = 'fill' + omsg = str(oid) + ' ' + str(cj_amount) + ' ' + self.kp.hex_pk() + self.taker.privmsg(c, cmd, omsg) def start_encryption(self, nick, maker_pk): if nick not in self.active_orders.keys(): raise Exception("Counterparty not part of this transaction.") - self.crypto_boxes[nick] = [maker_pk,enc_wrapper.as_init_encryption(self.kp, enc_wrapper.init_pubkey(maker_pk))] + self.crypto_boxes[nick] = [maker_pk,enc_wrapper.as_init_encryption(\ + self.kp, enc_wrapper.init_pubkey(maker_pk))] #send authorisation request - my_btc_priv = self.taker.wallet.get_key_from_addr(self.taker.wallet.unspent[self.my_utxos[0]]['address']) + my_btc_priv = self.taker.wallet.get_key_from_addr(\ + self.taker.wallet.unspent[self.my_utxos[0]]['address']) my_btc_pub = btc.privtopub(my_btc_priv) my_btc_sig = btc.ecdsa_sign(self.kp.hex_pk(), my_btc_priv) message = my_btc_pub + ' ' + my_btc_sig - self.send_priv(nick, '!auth', message, True) + self.taker.privmsg(nick, 'auth', message) def auth_counterparty(self, nick, btc_sig, cj_pub): '''Validate the counterpartys claim to own the btc @@ -66,17 +61,7 @@ def auth_counterparty(self, nick, btc_sig, cj_pub): return False return True - def recv_txio(self, nick, msg): - decrypted = enc_wrapper.decode_decrypt(msg, self.crypto_boxes[nick][1]) - chunks = decrypted.split(' ') - utxo_list = chunks[0].split(',') - cj_pub = chunks[1] - change_addr = chunks[2] - btc_sig = chunks[3] - if not self.auth_counterparty(nick, btc_sig, cj_pub): - print 'Authenticated encryption with counterparty: ' + nick + \ - ' not established. TODO: send rejection message' - return + def recv_txio(self, nick, utxo_list, cj_pub, change_addr): cj_addr = btc.pubtoaddr(cj_pub,get_addr_vbyte()) if nick not in self.nonrespondants: debug('nick(' + nick + ') not in nonrespondants ' + str(self.nonrespondants)) @@ -120,7 +105,7 @@ def recv_txio(self, nick, msg): debug('obtained tx\n' + pprint.pformat(btc.deserialize(tx))) txb64 = base64.b64encode(tx.decode('hex')) for nickk in self.active_orders.keys(): - self.send_priv(nickk, command_prefix + 'tx',txb64, True) + self.taker.privmsg(nickk, 'tx',txb64) #now sign it ourselves here for index, ins in enumerate(btc.deserialize(tx)['ins']): @@ -133,10 +118,8 @@ def recv_txio(self, nick, msg): tx = btc.sign(tx, index, self.taker.wallet.get_key_from_addr(addr)) self.latest_tx = btc.deserialize(tx) - def add_signature(self, nick, encsig): - sigb64 = enc_wrapper.decode_decrypt(encsig,self.crypto_boxes[nick][1]) + def add_signature(self, nick, sigb64): sig = base64.b64decode(sigb64).encode('hex') - inserted_sig = False tx = btc.serialize(self.latest_tx) for index, ins in enumerate(self.latest_tx['ins']): @@ -257,8 +240,16 @@ def on_privmsg(self, nick, message): if chunks[0] == 'pubkey': maker_pk = chunks[1] self.cjtx.start_encryption(nick, maker_pk) - if chunks[0] == 'auth': - self.cjtx.recv_txio(nick, chunks[1]) + if chunks[0] == 'ioauth': + utxo_list = chunks[1].split(',') + cj_pub = chunks[2] + change_addr = chunks[3] + btc_sig = chunks[4] + if not self.cjtx.auth_counterparty(nick, btc_sig, cj_pub): + print 'Authenticated encryption with counterparty: ' + nick + \ + ' not established. TODO: send rejection message' + return + self.cjtx.recv_txio(nick, utxo_list, cj_pub, change_addr) elif chunks[0] == 'sig': sig = chunks[1] self.cjtx.add_signature(nick,sig) From f7c60d84443f78a94b41ad400002e80b8519c904 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Wed, 4 Feb 2015 21:11:09 +0200 Subject: [PATCH 111/409] dont use utxos in existing unconfirmed txs dont use unconfirmed utxos in spends --- common.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/common.py b/common.py index fb94d729..bba8f29d 100644 --- a/common.py +++ b/common.py @@ -229,13 +229,15 @@ def find_unspent_addresses(self): blockr_url = 'http://tbtc.blockr.io/api/v1/address/unspent/' elif network == 'btc': blockr_url = 'http://btc.blockr.io/api/v1/address/unspent/' - res = btc.make_request(blockr_url+','.join(req)) + blockr_url += ','.join(req) + '?unconfirmed=1' + res = btc.make_request(blockr_url) data = json.loads(res)['data'] if 'unspent' in data: data = [data] for dat in data: for u in dat['unspent']: - self.unspent[u['tx']+':'+str(u['n'])] = {'address': + if u['confirmations'] != 0: + self.unspent[u['tx']+':'+str(u['n'])] = {'address': dat['address'], 'value': int(u['amount'].replace('.', ''))} def print_debug_wallet_info(self): From 027d28f395705efc6ea335d1fa9cb8c5d4c46519 Mon Sep 17 00:00:00 2001 From: Belcher Date: Thu, 5 Feb 2015 00:31:17 +0000 Subject: [PATCH 112/409] very slight edits, added spaces after commas --- enc_wrapper.py | 6 +++--- irclib.py | 2 +- taker.py | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/enc_wrapper.py b/enc_wrapper.py index aa21cf67..42a934ed 100644 --- a/enc_wrapper.py +++ b/enc_wrapper.py @@ -60,11 +60,11 @@ def as_init_encryption(kp, c_pk): #bitcoin signatures so it isn't necessary. #encoding for passing over the wire -def encrypt_encode(msg,box): +def encrypt_encode(msg, box): encrypted = box.encrypt(msg) return base64.b64encode(encrypted) -def decode_decrypt(msg,box): +def decode_decrypt(msg, box): decoded = base64.b64decode(msg) return box.decrypt(decoded) @@ -107,4 +107,4 @@ def test_case(case_name, alice_box, bob_box, ab_message, ba_message, num_iterati longb642 = base64.b64encode(''.join(random.choice(string.ascii_letters) for x in range(5000))) test_case("long b64", alice_box, bob_box, longb641, longb642,5) - print "All test cases passed - encryption and decryption should work correctly." \ No newline at end of file + print "All test cases passed - encryption and decryption should work correctly." diff --git a/irclib.py b/irclib.py index 6bdbdae9..38185ab2 100644 --- a/irclib.py +++ b/irclib.py @@ -93,7 +93,7 @@ def pubmsg(self, message): self.send_raw("PRIVMSG " + self.channel + " :" + message) def privmsg(self, nick, cmd, message): - debug('>>privmsg ' + 'nick=' + nick + 'cmd=' + cmd + ' msg=' + message) + debug('>>privmsg ' + 'nick=' + nick + ' cmd=' + cmd + ' msg=' + message) #should we encrypt? box = self.encrypting(cmd, nick, sending=True) #encrypt before chunking diff --git a/taker.py b/taker.py index 7eb078da..3eca2aab 100644 --- a/taker.py +++ b/taker.py @@ -105,7 +105,7 @@ def recv_txio(self, nick, utxo_list, cj_pub, change_addr): debug('obtained tx\n' + pprint.pformat(btc.deserialize(tx))) txb64 = base64.b64encode(tx.decode('hex')) for nickk in self.active_orders.keys(): - self.taker.privmsg(nickk, 'tx',txb64) + self.taker.privmsg(nickk, 'tx', txb64) #now sign it ourselves here for index, ins in enumerate(btc.deserialize(tx)['ins']): @@ -118,7 +118,7 @@ def recv_txio(self, nick, utxo_list, cj_pub, change_addr): tx = btc.sign(tx, index, self.taker.wallet.get_key_from_addr(addr)) self.latest_tx = btc.deserialize(tx) - def add_signature(self, nick, sigb64): + def add_signature(self, sigb64): sig = base64.b64decode(sigb64).encode('hex') inserted_sig = False tx = btc.serialize(self.latest_tx) @@ -252,7 +252,7 @@ def on_privmsg(self, nick, message): self.cjtx.recv_txio(nick, utxo_list, cj_pub, change_addr) elif chunks[0] == 'sig': sig = chunks[1] - self.cjtx.add_signature(nick,sig) + self.cjtx.add_signature(sig) my_tx_fee = 10000 From a4b64c864e4e5a8d917fc8c512c999bc5822e501 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Thu, 5 Feb 2015 20:00:21 +0200 Subject: [PATCH 113/409] complete authentication by checking btc pubkey used is valid, also optimise blockchain query code --- common.py | 58 +++++++++++++++++++++++------------------ encryption_protocol.txt | 3 ++- maker.py | 14 +++++++--- 3 files changed, 44 insertions(+), 31 deletions(-) diff --git a/common.py b/common.py index 96536e8b..1b3ba2d8 100644 --- a/common.py +++ b/common.py @@ -56,7 +56,31 @@ def debug_dump_object(obj, skip_fields=[]): else: print v - +def get_addr_from_utxo(txhash, index): + '''return the bitcoin address of the outpoint at + the specified index for the transaction with specified hash. + Return None if no such index existed for that transaction.''' + data = get_blockchain_data('txinfo',csv_params=[txhash]) + for a in data['vouts']: + if a['n']==index: + return a['address'] + return None + +def get_blockchain_data(body, source='blockr', csv_params=[], + query_params=[], network='test', output_key='data'): + '''A first step towards encapsulating blockchain queries.''' + if source != 'blockr': raise Exception ("source not yet implemented") + stem = 'http://btc.blockr.io/api/v1/' + if network=='test': stem = stem[:7]+'t'+stem[7:] + elif network != 'main': raise Exception("unrecognised bitcoin network type") + bodies = {'addrtx':'address/txs/','txinfo':'tx/info/','addrunspent':'address/unspent/', + 'addrbalance':'address/balance/'} + url = stem + bodies[body] + ','.join(csv_params) + if query_params: + url += '?'+','.join(query_params) + res = btc.make_request(url) + return json.loads(res)[output_key] + class Wallet(object): def __init__(self, seed, max_mix_depth=2): self.max_mix_depth = max_mix_depth @@ -179,12 +203,7 @@ def download_wallet_history(self, gaplimit=6): #TODO send a pull request to pybitcointools # because this surely should be possible with a function from it - if get_network() == 'testnet': - blockr_url = 'http://tbtc.blockr.io/api/v1/address/txs/' - elif network == 'btc': - blockr_url = 'http://btc.blockr.io/api/v1/address/txs/' - res = btc.make_request(blockr_url+','.join(addrs)) - data = json.loads(res)['data'] + data = get_blockchain_data('addrtx', csv_params=addrs) for dat in data: if dat['nb_txs'] != 0: last_used_addr = dat['address'] @@ -227,13 +246,8 @@ def find_unspent_addresses(self): #TODO send a pull request to pybitcointools # unspent() doesnt tell you which address, you get a bunch of utxos # but dont know which privkey to sign with - if get_network() == 'testnet': - blockr_url = 'http://tbtc.blockr.io/api/v1/address/unspent/' - elif network == 'btc': - blockr_url = 'http://btc.blockr.io/api/v1/address/unspent/' - blockr_url += ','.join(req) + '?unconfirmed=1' - res = btc.make_request(blockr_url) - data = json.loads(res)['data'] + data = get_blockchain_data('addrunspent', csv_params=req, + query_params=['unconfirmed=1']) if 'unspent' in data: data = [data] for dat in data: @@ -278,12 +292,8 @@ def run(self): unconfirmtimeoutfun() debug('checking for unconfirmed tx timed out') return - if get_network() == 'testnet': - blockr_url = 'http://tbtc.blockr.io/api/v1/address/balance/' - else: - blockr_url = 'http://btc.blockr.io/api/v1/address/balance/' - res = btc.make_request(blockr_url + self.address + '?confirmations=0') - data = json.loads(res)['data'] + data = get_blockchain_data('addrbalance',csv_params=[self.address], + query_params=['confirmations=0']) if data['balance'] > 0: break self.unconfirmfun(data['balance']*1e8) @@ -295,12 +305,8 @@ def run(self): confirmtimeoutfun() debug('checking for confirmed tx timed out') return - if get_network() == 'testnet': - blockr_url = 'http://tbtc.blockr.io/api/v1/address/txs/' - else: - blockr_url = 'http://btc.blockr.io/api/v1/address/txs/' - res = btc.make_request(blockr_url + self.address + '?confirmations=0') - data = json.loads(res)['data'] + data = get_blockchain_data('addrtx',csv_params=[self.address], + query_params=['confirmations=0']) if data['nb_txs'] == 0: continue if data['txs'][0]['confirmations'] >= 1: #confirmation threshold diff --git a/encryption_protocol.txt b/encryption_protocol.txt index bef83d0b..fb07259b 100644 --- a/encryption_protocol.txt +++ b/encryption_protocol.txt @@ -16,7 +16,7 @@ Encrypted TAK: !auth (Maker verifies the btc sig; if not valid, connection is dropped - send REJECT message) -MAK: !auth +MAK: !ioauth (Taker verifies the btc sig; if not valid, as for previous) Because the !auth messages are under encryption, there is no privacy leak of bitcoin pubkeys or output addresses. @@ -25,6 +25,7 @@ If both verifications pass, the remainder of the messages exchanged between the Specifically, these message types will be encrypted: !auth +!ioauth !tx !sig diff --git a/maker.py b/maker.py index 4159dccf..5846b212 100644 --- a/maker.py +++ b/maker.py @@ -36,14 +36,14 @@ def __init__(self, maker, nick, oid, amount, taker_pk): self.maker.privmsg(nick, 'pubkey', self.kp.hex_pk()) def auth_counterparty(self,nick,i_utxo_pubkey,btc_sig): - #TODO: add check that the pubkey's address is part of the order. self.i_utxo_pubkey = i_utxo_pubkey if not btc.ecdsa_verify(self.taker_pk,btc_sig,self.i_utxo_pubkey): print 'signature didnt match pubkey and message' return False - #authorisation of taker passed - #send auth request to taker + #authorisation of taker passed + #(but input utxo pubkey is checked in verify_unsigned_tx). + #Send auth request to taker #TODO the next 2 lines are a little inefficient. btc_key = self.maker.wallet.get_key_from_addr(self.cj_addr) btc_pub = btc.privtopub(btc_key) @@ -107,7 +107,13 @@ def confirm_callback(self, confirmations, txid, balance): self.maker.modify_orders(to_cancel, to_announce) def verify_unsigned_tx(self, txd): - tx_utxo_set = set([ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) for ins in txd['ins']]) + tx_utxo_set = set([ins['outpoint']['hash'] + ':' \ + + str(ins['outpoint']['index']) for ins in txd['ins']]) + #complete authentication: check the tx input uses the authing pubkey + if not btc.pubtoaddr(self.i_utxo_pubkey,get_addr_vbyte()) \ + in [get_addr_from_utxo(i['outpoint']['hash'],i['outpoint']['index']) \ + for i in txd['ins']]: + return False, "authenticating bitcoin address is not contained" my_utxo_set = set(self.utxos) wallet_utxos = set(self.maker.wallet.unspent) if not tx_utxo_set.issuperset(my_utxo_set): From a841c14c5a02d38026904d2fa65fd80552867e50 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Thu, 5 Feb 2015 21:11:16 +0200 Subject: [PATCH 114/409] spaces after commas --- common.py | 6 +++--- enc_wrapper.py | 10 +++++----- irclib.py | 2 +- maker.py | 16 ++++++++-------- taker.py | 10 +++++----- wallet-tool.py | 2 +- 6 files changed, 23 insertions(+), 23 deletions(-) diff --git a/common.py b/common.py index 1b3ba2d8..c2ad611c 100644 --- a/common.py +++ b/common.py @@ -60,7 +60,7 @@ def get_addr_from_utxo(txhash, index): '''return the bitcoin address of the outpoint at the specified index for the transaction with specified hash. Return None if no such index existed for that transaction.''' - data = get_blockchain_data('txinfo',csv_params=[txhash]) + data = get_blockchain_data('txinfo', csv_params=[txhash]) for a in data['vouts']: if a['n']==index: return a['address'] @@ -292,7 +292,7 @@ def run(self): unconfirmtimeoutfun() debug('checking for unconfirmed tx timed out') return - data = get_blockchain_data('addrbalance',csv_params=[self.address], + data = get_blockchain_data('addrbalance', csv_params=[self.address], query_params=['confirmations=0']) if data['balance'] > 0: break @@ -305,7 +305,7 @@ def run(self): confirmtimeoutfun() debug('checking for confirmed tx timed out') return - data = get_blockchain_data('addrtx',csv_params=[self.address], + data = get_blockchain_data('addrtx', csv_params=[self.address], query_params=['confirmations=0']) if data['nb_txs'] == 0: continue diff --git a/enc_wrapper.py b/enc_wrapper.py index 42a934ed..bc346944 100644 --- a/enc_wrapper.py +++ b/enc_wrapper.py @@ -43,7 +43,7 @@ def as_init_encryption(kp, c_pk): pubkey c_pk, create a Box ready for encryption/decryption. ''' - return libnacl.public.Box(kp.sk,c_pk) + return libnacl.public.Box(kp.sk, c_pk) ''' After initialisation, it's possible to use the box object returned from @@ -88,13 +88,13 @@ def test_case(case_name, alice_box, bob_box, ab_message, ba_message, num_iterati bob_kp = init_keypair() #this is the DH key exchange part - bob_otwpk = get_pubkey(bob_kp,True) - alice_otwpk = get_pubkey(alice_kp,True) + bob_otwpk = get_pubkey(bob_kp, True) + alice_otwpk = get_pubkey(alice_kp, True) bob_pk = init_pubkey(bob_otwpk) - alice_box = as_init_encryption(alice_kp,bob_pk) + alice_box = as_init_encryption(alice_kp, bob_pk) alice_pk = init_pubkey(alice_otwpk) - bob_box = as_init_encryption(bob_kp,alice_pk) + bob_box = as_init_encryption(bob_kp, alice_pk) #now Alice and Bob can use their 'box' #constructs (both of which utilise the same diff --git a/irclib.py b/irclib.py index 38185ab2..510384c7 100644 --- a/irclib.py +++ b/irclib.py @@ -98,7 +98,7 @@ def privmsg(self, nick, cmd, message): box = self.encrypting(cmd, nick, sending=True) #encrypt before chunking if box: - message = enc_wrapper.encrypt_encode(message,box) + message = enc_wrapper.encrypt_encode(message, box) if len(message) > 350: message_chunks = chunks(message, 350) diff --git a/maker.py b/maker.py index 5846b212..1518dc9b 100644 --- a/maker.py +++ b/maker.py @@ -35,10 +35,10 @@ def __init__(self, maker, nick, oid, amount, taker_pk): # orders to find out which addresses you use self.maker.privmsg(nick, 'pubkey', self.kp.hex_pk()) - def auth_counterparty(self,nick,i_utxo_pubkey,btc_sig): + def auth_counterparty(self, nick, i_utxo_pubkey, btc_sig): self.i_utxo_pubkey = i_utxo_pubkey - if not btc.ecdsa_verify(self.taker_pk,btc_sig,self.i_utxo_pubkey): + if not btc.ecdsa_verify(self.taker_pk, btc_sig, self.i_utxo_pubkey): print 'signature didnt match pubkey and message' return False #authorisation of taker passed @@ -47,7 +47,7 @@ def auth_counterparty(self,nick,i_utxo_pubkey,btc_sig): #TODO the next 2 lines are a little inefficient. btc_key = self.maker.wallet.get_key_from_addr(self.cj_addr) btc_pub = btc.privtopub(btc_key) - btc_sig = btc.ecdsa_sign(self.kp.hex_pk(),btc_key) + btc_sig = btc.ecdsa_sign(self.kp.hex_pk(), btc_key) authmsg = str(','.join(self.utxos)) + ' ' + \ btc_pub + ' ' + self.change_addr + ' ' + btc_sig self.maker.privmsg(nick, 'ioauth', authmsg) @@ -110,8 +110,8 @@ def verify_unsigned_tx(self, txd): tx_utxo_set = set([ins['outpoint']['hash'] + ':' \ + str(ins['outpoint']['index']) for ins in txd['ins']]) #complete authentication: check the tx input uses the authing pubkey - if not btc.pubtoaddr(self.i_utxo_pubkey,get_addr_vbyte()) \ - in [get_addr_from_utxo(i['outpoint']['hash'],i['outpoint']['index']) \ + if not btc.pubtoaddr(self.i_utxo_pubkey, get_addr_vbyte()) \ + in [get_addr_from_utxo(i['outpoint']['hash'], i['outpoint']['index']) \ for i in txd['ins']]: return False, "authenticating bitcoin address is not contained" my_utxo_set = set(self.utxos) @@ -165,7 +165,7 @@ def privmsg_all_orders(self, target, orderlist=None): orderline = '' for order in orderlist: elem_list = [str(order[k]) for k in order_keys] - self.privmsg(target,order['ordertype'],' '.join(elem_list)) + self.privmsg(target, order['ordertype'],' '.join(elem_list)) def send_error(self, nick, errmsg): debug('error<%s> : %s' % (nick, errmsg)) @@ -209,7 +209,7 @@ def on_privmsg(self, nick, message): try: i_utxo_pubkey = chunks[1] btc_sig = chunks[2] - except (ValueError,IndexError) as e: + except (ValueError, IndexError) as e: self.send_error(nick, str(e)) self.active_orders[nick].auth_counterparty(nick, i_utxo_pubkey, btc_sig) @@ -349,7 +349,7 @@ def main(): import sys seed = sys.argv[1] #btc.sha256('dont use brainwallets except for holding testnet coins') - wallet = Wallet(seed,max_mix_depth=5) + wallet = Wallet(seed, max_mix_depth=5) wallet.sync_wallet() maker = Maker(wallet) diff --git a/taker.py b/taker.py index 3eca2aab..2f2d4c48 100644 --- a/taker.py +++ b/taker.py @@ -42,7 +42,7 @@ def __init__(self, taker, cj_amount, orders, my_utxos, my_cj_addr, def start_encryption(self, nick, maker_pk): if nick not in self.active_orders.keys(): raise Exception("Counterparty not part of this transaction.") - self.crypto_boxes[nick] = [maker_pk,enc_wrapper.as_init_encryption(\ + self.crypto_boxes[nick] = [maker_pk, enc_wrapper.as_init_encryption(\ self.kp, enc_wrapper.init_pubkey(maker_pk))] #send authorisation request my_btc_priv = self.taker.wallet.get_key_from_addr(\ @@ -62,7 +62,7 @@ def auth_counterparty(self, nick, btc_sig, cj_pub): return True def recv_txio(self, nick, utxo_list, cj_pub, change_addr): - cj_addr = btc.pubtoaddr(cj_pub,get_addr_vbyte()) + cj_addr = btc.pubtoaddr(cj_pub, get_addr_vbyte()) if nick not in self.nonrespondants: debug('nick(' + nick + ') not in nonrespondants ' + str(self.nonrespondants)) return @@ -291,7 +291,7 @@ def on_pubmsg(self, nick, message): print 'making cjtx' self.cjtx = CoinJoinTX(self, int(amt), {cp: oid}, utxos, self.wallet.get_receive_addr(mixing_depth=1), - self.wallet.get_change_addr(mixing_depth=0), my_tx_fee,self.finish_callback) + self.wallet.get_change_addr(mixing_depth=0), my_tx_fee, self.finish_callback) elif chunks[0] == '%unspent': from pprint import pprint pprint(self.wallet.unspent) @@ -304,7 +304,7 @@ def on_pubmsg(self, nick, message): print 'making cjtx' self.cjtx = CoinJoinTX(self, int(amount), {counterparty: oid}, [my_utxo], self.wallet.get_receive_addr(mixing_depth=1), - self.wallet.get_change_addr(mixing_depth=0), my_tx_fee,self.finish_callback) + self.wallet.get_change_addr(mixing_depth=0), my_tx_fee, self.finish_callback) elif chunks[0] == '%2fill': #!2fill [amount] [utxo] [counterparty1] [oid1] [counterparty2] [oid2] amount = int(chunks[1]) @@ -316,7 +316,7 @@ def on_pubmsg(self, nick, message): print 'creating cjtx' self.cjtx = CoinJoinTX(self, amount, {cp1: oid1, cp2: oid2}, [my_utxo], self.wallet.get_receive_addr(mixing_depth=1), - self.wallet.get_change_addr(mixing_depth=0), my_tx_fee,self.finish_callback) + self.wallet.get_change_addr(mixing_depth=0), my_tx_fee, self.finish_callback) def main(): import sys diff --git a/wallet-tool.py b/wallet-tool.py index f28b9870..f74e428f 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -105,7 +105,7 @@ ins = [] outs = [] balance = 0 - for utxo,addrvalue in wallet.unspent.iteritems(): + for utxo, addrvalue in wallet.unspent.iteritems(): ins.append({'output': utxo}) balance += addrvalue['value'] destaddr = wallet.get_addr(0,0,0) From b01e7db0c06c2e212dce12464e6b04d0ac274145 Mon Sep 17 00:00:00 2001 From: Emilian Ursu Date: Sat, 7 Feb 2015 01:12:11 -0500 Subject: [PATCH 115/409] Capitalizing command_prefix --- common.py | 3 +-- maker.py | 8 ++++---- taker.py | 18 +++++++++--------- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/common.py b/common.py index 96536e8b..b2f76f70 100644 --- a/common.py +++ b/common.py @@ -11,8 +11,7 @@ #for the mainnet its #joinmarket-pit -#TODO make this var all in caps -command_prefix = '!' +COMMAND_PREFIX = '!' MAX_PRIVMSG_LEN = 400 ordername_list = ["absorder", "relorder"] diff --git a/maker.py b/maker.py index 4159dccf..0f650883 100644 --- a/maker.py +++ b/maker.py @@ -170,9 +170,9 @@ def on_welcome(self): self.privmsg_all_orders(CHANNEL) def on_privmsg(self, nick, message): - if message[0] != command_prefix: + if message[0] != COMMAND_PREFIX: return - command_lines = message.split(command_prefix) + command_lines = message.split(COMMAND_PREFIX) for command_line in command_lines: if len(command_line) == 0: continue @@ -224,7 +224,7 @@ def on_privmsg(self, nick, message): # using the same id again overwrites it, they'll be plenty of times when an order # has to be modified and its better to just have !order rather than !cancelorder then !order def on_pubmsg(self, nick, message): - if message[0] == command_prefix: + if message[0] == COMMAND_PREFIX: chunks = message[1:].split(" ") if chunks[0] == 'orderbook': self.privmsg_all_orders(nick) @@ -246,7 +246,7 @@ def modify_orders(self, to_cancel, to_announce): order = [o for o in self.orderlist if o['oid'] == oid][0] self.orderlist.remove(order) if len(to_cancel) > 0: - clines = [command_prefix + 'cancel ' + str(oid) for oid in to_cancel] + clines = [COMMAND_PREFIX + 'cancel ' + str(oid) for oid in to_cancel] self.pubmsg(''.join(clines)) if len(to_announce) > 0: self.privmsg_all_orders(CHANNEL, to_announce) diff --git a/taker.py b/taker.py index 3eca2aab..c9807357 100644 --- a/taker.py +++ b/taker.py @@ -168,10 +168,10 @@ def add_order(self, nick, chunks): (nick, chunks[1], chunks[0], chunks[2], chunks[3], chunks[4], chunks[5])) def on_privmsg(self, nick, message): - if message[0] != command_prefix: + if message[0] != COMMAND_PREFIX: return - for command in message[1:].split(command_prefix): + for command in message[1:].split(COMMAND_PREFIX): chunks = command.split(" ") if chunks[0] in ordername_list: self.add_order(nick, chunks) @@ -180,9 +180,9 @@ def on_privmsg(self, nick, message): # using the same id again overwrites it, they'll be plenty of times when an order # has to be modified and its better to just have !order rather than !cancelorder then !order def on_pubmsg(self, nick, message): - if message[0] != command_prefix: + if message[0] != COMMAND_PREFIX: return - for command in message[1:].split(command_prefix): + for command in message[1:].split(COMMAND_PREFIX): #commands starting with % are for testing and will be removed in the final version chunks = command.split(" ") if chunks[0] == 'cancel': @@ -204,7 +204,7 @@ def on_pubmsg(self, nick, message): print('done') def on_welcome(self): - self.pubmsg(command_prefix + 'orderbook') + self.pubmsg(COMMAND_PREFIX + 'orderbook') def on_set_topic(self, newtopic): chunks = newtopic.split('|') @@ -233,9 +233,9 @@ def __init__(self): def on_privmsg(self, nick, message): OrderbookWatch.on_privmsg(self, nick, message) #debug("privmsg nick=%s message=%s" % (nick, message)) - if message[0] != command_prefix: + if message[0] != COMMAND_PREFIX: return - for command in message[1:].split(command_prefix): + for command in message[1:].split(COMMAND_PREFIX): chunks = command.split(" ") if chunks[0] == 'pubkey': maker_pk = chunks[1] @@ -269,9 +269,9 @@ def finish_callback(self): def on_pubmsg(self, nick, message): Taker.on_pubmsg(self, nick, message) - if message[0] != command_prefix: + if message[0] != COMMAND_PREFIX: return - for command in message[1:].split(command_prefix): + for command in message[1:].split(COMMAND_PREFIX): #commands starting with % are for testing and will be removed in the final version chunks = command.split(" ") if chunks[0] == '%go': From 104e543361899614270f72b3f79ac68769e4f0ef Mon Sep 17 00:00:00 2001 From: Emilian Ursu Date: Sat, 7 Feb 2015 01:48:15 -0500 Subject: [PATCH 116/409] Changed get_mix_utxo_list to get_utxo_list_by_mixdepth --- common.py | 11 +++++------ patientsendpayment.py | 4 ++-- sendpayment.py | 2 +- taker.py | 2 +- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/common.py b/common.py index 96536e8b..1ea29025 100644 --- a/common.py +++ b/common.py @@ -110,7 +110,7 @@ def remove_old_utxos(self, tx): continue removed_utxos[utxo] = self.unspent[utxo] del self.unspent[utxo] - debug('removed utxos, wallet now is \n' + pprint.pformat(self.get_mix_utxo_list())) + debug('removed utxos, wallet now is \n' + pprint.pformat(self.get_utxo_list_by_mixdepth())) return removed_utxos def add_new_utxos(self, tx, txid): @@ -123,11 +123,10 @@ def add_new_utxos(self, tx, txid): utxo = txid + ':' + str(index) added_utxos[utxo] = addrdict self.unspent[utxo] = addrdict - debug('added utxos, wallet now is \n' + pprint.pformat(self.get_mix_utxo_list())) + debug('added utxos, wallet now is \n' + pprint.pformat(self.get_utxo_list_by_mixdepth())) return added_utxos - #TODO change the name of this to get_utxo_list_by_mixdepth - def get_mix_utxo_list(self): + def get_utxo_list_by_mixdepth(self): ''' returns a list of utxos sorted by different mix levels ''' @@ -140,7 +139,7 @@ def get_mix_utxo_list(self): return mix_utxo_list def get_balance_by_mixdepth(self): - mix_utxo_list = self.get_mix_utxo_list() + mix_utxo_list = self.get_utxo_list_by_mixdepth() mix_balance = {} for mixdepth, utxo_list in mix_utxo_list.iteritems(): total_value = 0 @@ -150,7 +149,7 @@ def get_balance_by_mixdepth(self): return mix_balance def select_utxos(self, mixdepth, amount): - utxo_list = self.get_mix_utxo_list()[mixdepth] + utxo_list = self.get_utxo_list_by_mixdepth()[mixdepth] unspent = [{'utxo': utxo, 'value': self.unspent[utxo]['value']} for utxo in utxo_list] inputs = btc.select(unspent, amount) diff --git a/patientsendpayment.py b/patientsendpayment.py index ed7cef64..5e8ff934 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -85,7 +85,7 @@ def on_tx_unconfirmed(self, cjorder, balance, removed_utxos): self.takerthread.finished = True print 'finished sending, exiting..' self.shutdown() - utxo_list = self.wallet.get_mix_utxo_list()[self.mixdepth] + utxo_list = self.wallet.get_utxo_list_by_mixdepth()[self.mixdepth] available_balance = 0 for utxo in utxo_list: available_balance = self.wallet.unspent[utxo]['value'] @@ -139,7 +139,7 @@ def main(): wallet = Wallet(seed, options.mixdepth + 1) wallet.sync_wallet() - utxo_list = wallet.get_mix_utxo_list()[options.mixdepth] + utxo_list = wallet.get_utxo_list_by_mixdepth()[options.mixdepth] available_balance = 0 for utxo in utxo_list: available_balance += wallet.unspent[utxo]['value'] diff --git a/sendpayment.py b/sendpayment.py index 95e87219..01b8845a 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -31,7 +31,7 @@ def run(self): if self.taker.amount == 0: total_value = 0 - utxo_list = self.taker.wallet.get_mix_utxo_list()[self.taker.mixdepth] + utxo_list = self.taker.wallet.get_utxo_list_by_mixdepth()[self.taker.mixdepth] for utxo in utxo_list: total_value += self.taker.wallet.unspent[utxo]['value'] orders, cjamount = choose_sweep_order(self.taker.db, total_value, self.taker.txfee, self.taker.makercount) diff --git a/taker.py b/taker.py index 3eca2aab..76bd3069 100644 --- a/taker.py +++ b/taker.py @@ -282,7 +282,7 @@ def on_pubmsg(self, nick, message): #this testing command implements a very dumb algorithm. #just take 1 utxo from anywhere and output it to a level 1 #change address. - utxo_dict = self.wallet.get_mix_utxo_list() + utxo_dict = self.wallet.get_utxo_list_by_mixdepth() utxo_list = [x for v in utxo_dict.itervalues() for x in v] unspent = [{'utxo': utxo, 'value': self.wallet.unspent[utxo]['value']} \ for utxo in utxo_list] From d08e3000ffcb6e9be86265d2373fc2e0edd32135 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Wed, 11 Feb 2015 14:50:32 +0200 Subject: [PATCH 117/409] first regtest changes --- blockchaininterface.py | 156 +++++++++++++++++++++++++++++++++++++++++ common.py | 47 +++++++++++-- testing/regtesttest.py | 34 +++++++++ 3 files changed, 230 insertions(+), 7 deletions(-) create mode 100644 blockchaininterface.py create mode 100644 testing/regtesttest.py diff --git a/blockchaininterface.py b/blockchaininterface.py new file mode 100644 index 00000000..089823cb --- /dev/null +++ b/blockchaininterface.py @@ -0,0 +1,156 @@ +#from joinmarket import * +import subprocess +import unittest +import json +import abc +from decimal import Decimal + +class BlockChainInterface(object): + __metaclass__ = abc.ABCMeta + def __init__(self): + #TODO: pass in network type (main/test) + pass + @abc.abstractmethod + def get_utxos_from_addr(self, address): + '''Given an address, return a list of utxos + in format txid:vout''' + pass + def get_balance_at_addr(self,address): + '''Given an address, return a balance in satoshis''' + pass + + @abc.abstractmethod + def send_tx(self, tx_hex): + '''Given raw txhex, push to network and return result in form: TODO''' + print 'got herer' + pass + def get_net_info(self): + pass + def get_addr_from_utxo(self, txhash, index): + '''Given utxo in form txhash, index, return the address + owning the utxo and the amount in satoshis in form (addr,amt)''' + pass + + +#class for regtest chain access +#running on local daemon. Only +#to be instantiated after network is up +#with > 100 blocks. +class RegTestImp(BlockChainInterface): + def __init__(self, client_location): + self.command_params = [client_location,'-regtest'] + #quick check that it's up else quit + res = self.rpc(['getbalance']) + try: + self.current_balance = int(Decimal(res)) + print "Instantiated interface to regtest, wallet balance is: "+str(self.current_balance) +" bitcoins." + if not self.current_balance > 0: + raise Exception("Regtest network not properly initialised.") + except Exception as e: + print e + + def rpc(self,args,accept_failure=[]): + try: + res = subprocess.check_output(self.command_params+args) + except subprocess.CalledProcessError, e: + if e.returncode in accept_failure: + return '' + raise + return res + + def send_tx(self, tx_hex): + res = self.rpc(['sendrawtransaction',tx_hex]) + #TODO parse return string + print res + return True + + def get_utxos_from_addr(self,address): + res = json.loads(self.rpc(['listunspent','1','9999999','[\"'+address+'\"]'])) + utxos = [] + for r in res: + utxos.append(r['txid']+':'+str(r['vout'])) + return utxos + + def get_txs_from_addr(self,addresses): + #use listtransactions and then filter + #e.g.: -regtest listtransactions 'watchonly' 1000 0 true + #to get the last 1000 transactions TODO 1000 is arbitrary + + res = json.loads(self.rpc(['listtransactions','watchonly','1000','0','true'])) + result=[] + for address in addresses: + nbtxs = 0 + txs=[] + for a in res: + if a['address'] != address: + continue + nbtxs += 1 + txs.append({'confirmations':a['confirmations'],'tx':a['txid'],'amount':a['amount']}) + result.append({'nb_txs':nbtxs,'address':address,'txs':txs}) + return {'data':result} + + def get_balance_at_addr(self, address): + #NB This will NOT return coinbase coins (but wont matter in our use case). + #In order to have the Bitcoin RPC read balances at addresses + #it doesn't own, we must import the addresses as watch-only + #Note that this is a 0.10 feature; won't work with older bitcoin clients. + #TODO : there can be a performance issue with rescanning here. + + #allow importaddress to fail in case the address is already in the wallet + self.rpc(['importaddress',address,'watchonly'],[4]) + return int(Decimal(1e8) * Decimal(self.rpc(['getreceivedbyaddress',address]))) + + def tick_forward_chain(self, n): + '''Special method for regtest only; + instruct to mine n blocks.''' + self.rpc(['setgenerate','true',str(n)]) + + def grab_coins(self,receiving_addr,amt=50): + ''' + NOTE! amt is passed in Coins, not Satoshis! + Special method for regtest only: + take coins from bitcoind's own wallet + and put them in the receiving addr. + Return the txid. + ''' + if amt > 500: + raise Exception("Either you forgot to pass the amount in Bitcoins, or you\'re too greedy") + if amt > self.current_balance: + #mine enough to get to the reqd amt + reqd = int(amt - self.current_balance) + reqd_blocks = str(int(reqd/50) +1) + if self.rpc(['setgenerate','true',reqd_blocks]): + raise Exception("Something went wrong") + #now we do a custom create transaction and push to the receiver + txid = self.rpc(['sendtoaddress',receiving_addr,str(amt)]) + if not txid: + raise Exception("Failed to broadcast transaction") + #confirm + self.tick_forward_chain(1) + return txid + + def get_addr_from_utxo(self, txhash, index): + #get the transaction details + res = json.loads(self.rpc(['gettxout',txhash, str(index)])) + amt = int(Decimal(1e8)*Decimal(res['value'])) + address = res('addresses')[0] + return (address,amt) + +def main(): + bitcointoolsdir = '/home/adam/DevRepos/bitcoin/src/' + btc_client = bitcointoolsdir + 'bitcoin-cli' + myBCI = RegTestImp(btc_client) + #myBCI.send_tx('stuff') + print myBCI.get_utxos_from_addr("n4EjHhGVS4Rod8ociyviR3FH442XYMWweD") + print myBCI.get_balance_at_addr("n4EjHhGVS4Rod8ociyviR3FH442XYMWweD") + txid = myBCI.grab_coins('mioPqzCoFWcMTiWcYKo2Py6kwEp5vPzL2X',23) + print txid + print myBCI.get_balance_at_addr('mioPqzCoFWcMTiWcYKo2Py6kwEp5vPzL2X') + print myBCI.get_utxos_from_addr('mioPqzCoFWcMTiWcYKo2Py6kwEp5vPzL2X') + +if __name__ == '__main__': + main() + + + + diff --git a/common.py b/common.py index c2ad611c..77a70114 100644 --- a/common.py +++ b/common.py @@ -4,6 +4,7 @@ from math import factorial import sys, datetime, json, time, pprint import threading +import blockchaininterface HOST = 'irc.freenode.net' CHANNEL = '#joinmarket-pit-test' @@ -14,7 +15,7 @@ #TODO make this var all in caps command_prefix = '!' MAX_PRIVMSG_LEN = 400 - +blockchain_source = 'regtest' ordername_list = ["absorder", "relorder"] encrypted_commands = ["auth", "ioauth", "tx", "sig"] plaintext_commands = ["fill", "error", "pubkey", "orderbook", "relorder", "absorder"] @@ -66,20 +67,52 @@ def get_addr_from_utxo(txhash, index): return a['address'] return None -def get_blockchain_data(body, source='blockr', csv_params=[], +def get_blockchain_data(body, csv_params=[], query_params=[], network='test', output_key='data'): '''A first step towards encapsulating blockchain queries.''' - if source != 'blockr': raise Exception ("source not yet implemented") - stem = 'http://btc.blockr.io/api/v1/' - if network=='test': stem = stem[:7]+'t'+stem[7:] - elif network != 'main': raise Exception("unrecognised bitcoin network type") + if blockchain_source=='regtest': + stem = 'regtest:' + elif blockchain_source=='blockr': + stem = 'http://btc.blockr.io/api/v1/' + if network=='test': + stem = stem[:7]+'t'+stem[7:] + elif network != 'main': + raise Exception("unrecognised bitcoin network type") + else: + raise Exception("Unrecognised blockchain source") + bodies = {'addrtx':'address/txs/','txinfo':'tx/info/','addrunspent':'address/unspent/', 'addrbalance':'address/balance/'} url = stem + bodies[body] + ','.join(csv_params) if query_params: url += '?'+','.join(query_params) - res = btc.make_request(url) + if blockchain_source=='blockr': + res = get_blockr_data(url) + elif blockchain_source=='regtest': + res = get_regtest_data(url) + else: + raise Exception("Unrecognised blockchain source" + "") return json.loads(res)[output_key] + +def get_blockr_data(req): + return btc.make_request(req) + +def get_regtest_data(req): + bitcointoolsdir = '/home/adam/DevRepos/bitcoin/src/' + btc_client = bitcointoolsdir + 'bitcoin-cli' + myBCI = blockchaininterface.RegTestImp(btc_client) + if not req.startswith('regtest'): + raise Exception("Invalid request to regtest") + req = ''.join(req.split(':')[1:]).split('/') + if req[0]=='address' and req[1]=='txs': + addrs = req[2].split(',') + #NB: we don't allow unconfirmeds in regtest + #for now; TODO + if 'unconfirmed' in addrs[-1]: + addrs = addrs[:-1] + return myBCI.get_txs_from_addr(addrs) + class Wallet(object): def __init__(self, seed, max_mix_depth=2): diff --git a/testing/regtesttest.py b/testing/regtesttest.py new file mode 100644 index 00000000..a9e5d9d5 --- /dev/null +++ b/testing/regtesttest.py @@ -0,0 +1,34 @@ +import sys +import os +import subprocess +import unittest + +'''Expectations +1. Any bot should run indefinitely irrespective of the input +messages it receives, except: +a. Bots which perform a finite action +b. When there is a network failure, the bot should quit gracefully. + +2. A bot must never spend an unacceptably high transaction fee. + +3. A bot must explicitly reject interactions with another bot not +respecting the JoinMarket protocol for its version. + + +''' +bitcointoolsdir = '/home/adam/bitcoin/bitcoin-0.9.1-linux/bin/64/' +btc_client = bitcointoolsdir + 'bitcoin-cli' +btc_client_flags = '-regtest' + +class FooTests(unittest.TestCase): + def testFoo(self): + self.failUnless(False) + #subprocess.Popen([btc_client,btc_client_flags,'listunspent']) + +def main(): + unittest.main() + +if __name__ == '__main__': + main() + + From c77a65b61d1b459338af6d6133c07092c6e7af61 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Thu, 12 Feb 2015 20:35:35 +0200 Subject: [PATCH 118/409] further steps --- blockchaininterface.py | 19 +++++++++++++++++-- common.py | 5 +++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/blockchaininterface.py b/blockchaininterface.py index 089823cb..9b1bac5e 100644 --- a/blockchaininterface.py +++ b/blockchaininterface.py @@ -4,6 +4,8 @@ import json import abc from decimal import Decimal +import bitcoin as btc + class BlockChainInterface(object): __metaclass__ = abc.ABCMeta @@ -71,7 +73,7 @@ def get_utxos_from_addr(self,address): utxos.append(r['txid']+':'+str(r['vout'])) return utxos - def get_txs_from_addr(self,addresses): + def get_txs_from_addr(self, addresses): #use listtransactions and then filter #e.g.: -regtest listtransactions 'watchonly' 1000 0 true #to get the last 1000 transactions TODO 1000 is arbitrary @@ -87,7 +89,20 @@ def get_txs_from_addr(self,addresses): nbtxs += 1 txs.append({'confirmations':a['confirmations'],'tx':a['txid'],'amount':a['amount']}) result.append({'nb_txs':nbtxs,'address':address,'txs':txs}) - return {'data':result} + return {'data':result} + + def get_tx_info(self, txhash): + res = json.loads(self.rpc(['gettransaction',txhash,'true'])) + tx = btc.deserialize(res['hex']) + #build vout list + vouts = [] + n=0 + for o in tx['outs']: + vouts.append({'n':n,'amount':o['value'],'address':btc.script_to_address(o['script'])}) + n+=1 + + return {'data':{'vouts':vouts}} + def get_balance_at_addr(self, address): #NB This will NOT return coinbase coins (but wont matter in our use case). diff --git a/common.py b/common.py index 77a70114..82653911 100644 --- a/common.py +++ b/common.py @@ -112,6 +112,11 @@ def get_regtest_data(req): if 'unconfirmed' in addrs[-1]: addrs = addrs[:-1] return myBCI.get_txs_from_addr(addrs) + elif req[0]=='tx' and req[1]=='info': + txhash = req[2] #TODO currently only allowing one tx + return myBCI.get_tx_info(txhash) + elif req[0]=='addr' and req[1] == 'balance': + class Wallet(object): From fdd2d895b1f18c3176e18e9a67ac6ee93400eca7 Mon Sep 17 00:00:00 2001 From: chris belcher Date: Thu, 12 Feb 2015 20:32:29 +0000 Subject: [PATCH 119/409] started modifying irclib and taker --- irclib.py | 47 ++++++++++++++++++++++++++++++++++++++-------- message_channel.py | 29 ++++++++++++++++++++++++---- taker.py | 21 ++++++++++++++++++--- 3 files changed, 82 insertions(+), 15 deletions(-) diff --git a/irclib.py b/irclib.py index 510384c7..da5bbfad 100644 --- a/irclib.py +++ b/irclib.py @@ -1,6 +1,8 @@ -import socket, threading, time from common import * +from message_channel import MessageChannel + +import socket, threading, time import base64, os import enc_wrapper @@ -44,7 +46,30 @@ def run(self): debug('ended ping thread') #handle one channel at a time -class IRCClient(object): +class IRCMessageChannel(MessageChannel): + #def run(self): pass + #def shutdown(self): pass + def request_orderbook(self): pass + def fill_order(self, nick, oid, cj_amount, taker_pubkey): pass + def send_auth(self, nick, pubkey, sig): pass + def send_tx(self, nick, txhex): pass + def announce_orders(self, orderlist, nick=None): pass #nick=None means announce publicly + def cancel_orders(self, oid_list): pass + def send_error(self, nick, errormsg): pass + def send_pubkey(self, nick, pubkey): pass + def send_ioauth(self, nick, utxo_list, cj_pubkey, change_addr, sig): pass + def send_sigs(self, nick, sig_list): pass + + ''' + def register_channel_callbacks(self, on_welcome=None, on_set_topic=None, + on_connect=None, on_disconnect=None, on_nick_leave=None, on_nick_change=None): + def register_orderbookwatch_callbacks(self, on_orders_seen=None, + on_orders_cancel=None): + def register_taker_callbacks(self, on_error=None, on_pubkey=None, on_ioauth=None, + on_sigs=None): + def register_maker_callbacks(self, on_orderbook_requested=None, on_order_filled=None, + on_seen_auth=None, on_seen_tx=None): + ''' def on_privmsg(self, nick, message): pass def on_pubmsg(self, nick, message): pass @@ -77,7 +102,8 @@ def encrypting(self, cmd, nick, sending=False): return self.cjtx.crypto_boxes[nick][1] else: raise Exception("Invalid command type: " + cmd) - + + #close implies it will attempt to reconnect def close(self): try: self.send_raw("QUIT") @@ -206,10 +232,14 @@ def __handle_line(self, line): elif chunks[1] == '251': self.motd_fd.close() ''' - - def run(self, server, port, nick, channel, username='username', realname='realname'): + + def __init__(self, server, port, nick, channel, username='username', realname='realname'): + self.serverport = (server, port) self.nick = nick self.channel = channel + self.userrealname = (username, realname) + + def run(self): self.connect_attempts = 0 self.waiting = {} self.built_privmsg = {} @@ -222,9 +252,9 @@ def run(self, server, port, nick, channel, username='username', realname='realna try: debug('connecting') self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.sock.connect((server, port)) + self.sock.connect(self.serverport) self.fd = self.sock.makefile() - self.send_raw('USER %s b c :%s' % (username, realname)) + self.send_raw('USER %s b c :%s' % userrealname) self.send_raw('NICK ' + nick) while 1: try: @@ -243,7 +273,8 @@ def run(self, server, port, nick, channel, username='username', realname='realna finally: self.fd.close() self.sock.close() - self.on_disconnect() + if self.on_disconnect != None: + self.on_disconnect() print 'disconnected irc' time.sleep(10) self.connect_attempts += 1 diff --git a/message_channel.py b/message_channel.py index 90ef0b41..0b3c2af7 100644 --- a/message_channel.py +++ b/message_channel.py @@ -7,21 +7,42 @@ class MessageChannel(object): def run(self): pass def shutdown(self): pass + #callbacks for everyone + #some of these many not have meaning in a future channel, like bitmessage + def register_channel_callbacks(self, on_welcome=None, on_set_topic=None, + on_connect=None, on_disconnect=None, on_nick_leave=None, on_nick_change=None): + self.on_welcome = on_welcome + self.on_set_topic = on_set_topic + self.on_connect = on_connect + self.on_disconnect = on_disconnect + self.on_nick_leave = on_nick_leave + self.on_nick_change = on_nick_change + #orderbook watcher commands - def register_orderbookwatch_callbacks(self, on_order_seen=None, - on_order_cancel=None): pass + def register_orderbookwatch_callbacks(self, on_orders_seen=None, + on_orders_cancel=None): + self.on_orders_seen = on_orders_seen + self.on_orders_cancel = on_orders_cancel def request_orderbook(self): pass #taker commands def register_taker_callbacks(self, on_error=None, on_pubkey=None, on_ioauth=None, - on_sigs=None): pass + on_sigs=None): + self.on_error = on_error + self.on_pubkey = on_pubkey + self.on_ioauth = on_ioauth + self.on_sigs = on_sigs def fill_order(self, nick, oid, cj_amount, taker_pubkey): pass def send_auth(self, nick, pubkey, sig): pass def send_tx(self, nick, txhex): pass #maker commands def register_maker_callbacks(self, on_orderbook_requested=None, on_order_filled=None, - on_seen_auth=None, on_seen_tx=None): pass + on_seen_auth=None, on_seen_tx=None): + self.on_orderbook_requested = on_orderbook_requested + self.on_order_filled = on_order_filled + self.on_seen_auth = on_seen_auth + self.on_seen_tx = on_seen_tx def announce_orders(self, orderlist, nick=None): pass #nick=None means announce publicly def cancel_orders(self, oid_list): pass def send_error(self, nick, errormsg): pass diff --git a/taker.py b/taker.py index b4ef7c2f..de7eacd7 100644 --- a/taker.py +++ b/taker.py @@ -152,15 +152,30 @@ def add_signature(self, sigb64): debug('pushed tx ' + str(ret)) if self.finishcallback != None: self.finishcallback() - -class OrderbookWatch(irclib.IRCClient): - def __init__(self): + +class CoinJoinerPeer(object): + pass + +class OrderbookWatch(CoinJoinerPeer): + def __init__(self, msgchan): + self.msgchan = msgchan + self.msgchan.register_orderbookwatch_callbacks(on_orders_seen, + on_orders_cancel) + con = sqlite3.connect(":memory:", check_same_thread=False) con.row_factory = sqlite3.Row self.db = con.cursor() self.db.execute("CREATE TABLE orderbook(counterparty TEXT, oid INTEGER, ordertype TEXT, " + "minsize INTEGER, maxsize INTEGER, txfee INTEGER, cjfee TEXT);") + #accept a list + def on_orders_seen(self): + pass + + def on_orders_cancel(self, nick, oid): + pass + + def add_order(self, nick, chunks): self.db.execute("DELETE FROM orderbook WHERE counterparty=? AND oid=?;", (nick, chunks[1])) From 5cda43d02b054faae581e895afa91c6bc9164a21 Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 13 Feb 2015 00:20:12 +0000 Subject: [PATCH 120/409] more --- irclib.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/irclib.py b/irclib.py index da5bbfad..52b53dda 100644 --- a/irclib.py +++ b/irclib.py @@ -60,7 +60,6 @@ def send_pubkey(self, nick, pubkey): pass def send_ioauth(self, nick, utxo_list, cj_pubkey, change_addr, sig): pass def send_sigs(self, nick, sig_list): pass - ''' def register_channel_callbacks(self, on_welcome=None, on_set_topic=None, on_connect=None, on_disconnect=None, on_nick_leave=None, on_nick_change=None): def register_orderbookwatch_callbacks(self, on_orders_seen=None, @@ -69,7 +68,6 @@ def register_taker_callbacks(self, on_error=None, on_pubkey=None, on_ioauth=None on_sigs=None): def register_maker_callbacks(self, on_orderbook_requested=None, on_order_filled=None, on_seen_auth=None, on_seen_tx=None): - ''' def on_privmsg(self, nick, message): pass def on_pubmsg(self, nick, message): pass @@ -197,14 +195,16 @@ def __handle_line(self, line): self.lockcond.notify() self.lockcond.release() elif chunks[1] == '376': #end of motd - self.on_connect() + if self.on_connect: + self.on_connect() self.send_raw('JOIN ' + self.channel) elif chunks[1] == '433': #nick in use self.nick += '_' self.send_raw('NICK ' + self.nick) elif chunks[1] == '366': #end of names list self.connect_attempts = 0 - self.on_welcome() + if self.on_welcome: + self.on_welcome() elif chunks[1] == '332' or chunks[1] == 'TOPIC': #channel topic topic = get_irc_text(line) self.on_set_topic(topic) @@ -213,14 +213,17 @@ def __handle_line(self, line): if nick == self.nick: raise IOError('we quit') else: - self.on_leave(nick) + if self.on_nick_leave: + self.on_nick_leave(nick) elif chunks[1] == 'KICK': target = chunks[3] nick = get_irc_nick(chunks[0]) - self.on_leave(nick) + if self.on_nick_leave: + self.on_nick_leave(nick) elif chunks[1] == 'PART': nick = get_irc_nick(chunks[0]) - self.on_leave(nick) + if self.on_nick_leave: + self.on_nick_leave(nick) elif chunks[1] == 'JOIN': channel = chunks[2][1:] nick = get_irc_nick(chunks[0]) @@ -273,7 +276,7 @@ def run(self): finally: self.fd.close() self.sock.close() - if self.on_disconnect != None: + if self.on_disconnect: self.on_disconnect() print 'disconnected irc' time.sleep(10) From 4e2a256eb764196d1cc2aea2900d2c6b8f313754 Mon Sep 17 00:00:00 2001 From: chris belcher Date: Fri, 13 Feb 2015 12:24:06 +0000 Subject: [PATCH 121/409] guitaker works with message channel --- gui-taker.py | 10 ++-- irclib.py | 132 +++++++++++++++++++++++++++++++-------------- message_channel.py | 8 +-- taker.py | 70 ++++++------------------ 4 files changed, 118 insertions(+), 102 deletions(-) diff --git a/gui-taker.py b/gui-taker.py index a017dc36..b7a63eee 100644 --- a/gui-taker.py +++ b/gui-taker.py @@ -1,5 +1,6 @@ import taker +from irclib import IRCMessageChannel from common import * import BaseHTTPServer, SimpleHTTPServer, threading @@ -131,7 +132,7 @@ def do_GET(self): def do_POST(self): if self.path == '/shutdown': - self.taker.shutdown() + self.taker.msgchan.shutdown() self.send_response(200) self.send_header('Content-Type', 'text/html') self.send_header('Content-Length', len(shutdownpage)) @@ -162,11 +163,10 @@ def main(): import bitcoin as btc nickname = 'guitaker-' + btc.sha256(gethostname())[:6] + irc = IRCMessageChannel(nickname) + taker = GUITaker(irc) print 'starting irc' - taker = GUITaker() - taker.run(HOST, PORT, nickname, CHANNEL) - - #create_depth_graph() + irc.run() if __name__ == "__main__": main() diff --git a/irclib.py b/irclib.py index 52b53dda..1f0f7c86 100644 --- a/irclib.py +++ b/irclib.py @@ -49,7 +49,7 @@ def run(self): class IRCMessageChannel(MessageChannel): #def run(self): pass #def shutdown(self): pass - def request_orderbook(self): pass + #def request_orderbook(self): pass def fill_order(self, nick, oid, cj_amount, taker_pubkey): pass def send_auth(self, nick, pubkey, sig): pass def send_tx(self, nick, txhex): pass @@ -60,47 +60,27 @@ def send_pubkey(self, nick, pubkey): pass def send_ioauth(self, nick, utxo_list, cj_pubkey, change_addr, sig): pass def send_sigs(self, nick, sig_list): pass - def register_channel_callbacks(self, on_welcome=None, on_set_topic=None, - on_connect=None, on_disconnect=None, on_nick_leave=None, on_nick_change=None): - def register_orderbookwatch_callbacks(self, on_orders_seen=None, - on_orders_cancel=None): + #def register_channel_callbacks(self, on_welcome=None, on_set_topic=None, + # on_connect=None, on_disconnect=None, on_nick_leave=None, on_nick_change=None): + #def register_orderbookwatch_callbacks(self, on_order_seen=None, + # on_order_cancel=None): + ''' def register_taker_callbacks(self, on_error=None, on_pubkey=None, on_ioauth=None, on_sigs=None): def register_maker_callbacks(self, on_orderbook_requested=None, on_order_filled=None, on_seen_auth=None, on_seen_tx=None): + ''' - def on_privmsg(self, nick, message): pass - def on_pubmsg(self, nick, message): pass + #def on_privmsg(self, nick, message): pass + #def on_pubmsg(self, nick, message): pass + ''' def on_welcome(self): pass def on_set_topic(self, newtopic): pass def on_leave(self, nick): pass def on_disconnect(self): pass def on_connect(self): pass #TODO implement on_nick_change - - def encrypting(self, cmd, nick, sending=False): - '''Establish whether the message is to be - encrypted/decrypted based on the command string. - If so, retrieve the appropriate crypto_box object - and return. Sending/receiving flag enables us - to check which command strings correspond to which - type of object (maker/taker).''' - - if cmd in plaintext_commands: - return None - elif cmd not in encrypted_commands: - raise Exception("Invalid command type: " + cmd) - - maker_strings = ['tx','auth'] if not sending else ['ioauth','sig'] - taker_strings = ['ioauth','sig'] if not sending else ['tx','auth'] - - if cmd in maker_strings: - return self.active_orders[nick].crypto_box - elif cmd in taker_strings: - return self.cjtx.crypto_boxes[nick][1] - else: - raise Exception("Invalid command type: " + cmd) - + ''' #close implies it will attempt to reconnect def close(self): try: @@ -112,6 +92,10 @@ def shutdown(self): self.close() self.give_up = True + def request_orderbook(self): + self.pubmsg(COMMAND_PREFIX + 'orderbook') + + def pubmsg(self, message): debug('>>pubmsg ' + message) self.send_raw("PRIVMSG " + self.channel + " :" + message) @@ -119,7 +103,7 @@ def pubmsg(self, message): def privmsg(self, nick, cmd, message): debug('>>privmsg ' + 'nick=' + nick + ' cmd=' + cmd + ' msg=' + message) #should we encrypt? - box = self.encrypting(cmd, nick, sending=True) + box = self.__encrypting(cmd, nick, sending=True) #encrypt before chunking if box: message = enc_wrapper.encrypt_encode(message, box) @@ -140,6 +124,76 @@ def send_raw(self, line): # debug('sendraw ' + line) self.sock.sendall(line + '\r\n') + def check_for_orders(self, nick, chunks): + if chunks[0] in ordername_list: + try: + counterparty = nick + oid = chunks[1] + ordertype = chunks[0] + minsize = chunks[2] + maxsize = chunks[3] + txfee = chunks[4] + cjfee = chunks[5] + if self.on_order_seen: + self.on_order_seen(counterparty, oid, + ordertype, minsize, maxsize, txfee, cjfee) + return True + except IndexError as e: + debug('index error parsing chunks') + #TODO what now? just ignore iirc + return False + + def __on_privmsg(self, nick, message): + '''handles the case when a private message is received''' + if message[0] != COMMAND_PREFIX: + return + for command in message[1:].split(COMMAND_PREFIX): + chunks = command.split(" ") + if self.check_for_orders(nick, chunks): + pass + + + def __on_pubmsg(self, nick, message): + if message[0] != COMMAND_PREFIX: + return + for command in message[1:].split(COMMAND_PREFIX): + chunks = command.split(" ") + if self.check_for_orders(nick, chunks): + pass + elif chunks[0] == 'cancel': + #!cancel [oid] + try: + oid = int(chunks[1]) + if self.on_order_cancel: + self.on_order_cancel(nick, oid) + except ValueError as e: + debug("!cancel " + repr(e)) + return + + def __encrypting(self, cmd, nick, sending=False): + '''Establish whether the message is to be + encrypted/decrypted based on the command string. + If so, retrieve the appropriate crypto_box object + and return. Sending/receiving flag enables us + to check which command strings correspond to which + type of object (maker/taker).''' + + if cmd in plaintext_commands: + return None + elif cmd not in encrypted_commands: + raise Exception("Invalid command type: " + cmd) + + maker_strings = ['tx','auth'] if not sending else ['ioauth','sig'] + taker_strings = ['ioauth','sig'] if not sending else ['tx','auth'] + + if cmd in maker_strings: + return self.active_orders[nick].crypto_box + elif cmd in taker_strings: + return self.cjtx.crypto_boxes[nick][1] + else: + raise Exception("Invalid command type: " + cmd) + + def __handle_privmsg(self, source, target, message): nick = get_irc_nick(source) if message[0] == '\x01': @@ -157,7 +211,7 @@ def __handle_privmsg(self, source, target, message): self.built_privmsg[nick] = [cmd_string, message[:-2]] else: self.built_privmsg[nick][1] += message[:-2] - box = self.encrypting(self.built_privmsg[nick][0], nick) + box = self.__encrypting(self.built_privmsg[nick][0], nick) if message[-1]==';': self.waiting[nick]=True elif message[-1]=='~': @@ -172,12 +226,12 @@ def __handle_privmsg(self, source, target, message): #wipe the message buffer waiting for the next one del self.built_privmsg[nick] debug("< Date: Fri, 13 Feb 2015 18:08:41 +0000 Subject: [PATCH 122/409] renamed irclib.py --- gui-taker.py | 2 +- irclib.py => irc.py | 700 ++++++++++++++++++++++---------------------- taker.py | 1 - 3 files changed, 351 insertions(+), 352 deletions(-) rename irclib.py => irc.py (96%) diff --git a/gui-taker.py b/gui-taker.py index b7a63eee..41cecd64 100644 --- a/gui-taker.py +++ b/gui-taker.py @@ -1,6 +1,6 @@ import taker -from irclib import IRCMessageChannel +from irc import IRCMessageChannel from common import * import BaseHTTPServer, SimpleHTTPServer, threading diff --git a/irclib.py b/irc.py similarity index 96% rename from irclib.py rename to irc.py index 1f0f7c86..c4425670 100644 --- a/irclib.py +++ b/irc.py @@ -1,350 +1,350 @@ - -from common import * -from message_channel import MessageChannel - -import socket, threading, time -import base64, os -import enc_wrapper - -PING_INTERVAL = 40 -PING_TIMEOUT = 10 - -def get_irc_text(line): - return line[line[1:].find(':') + 2:] - -def get_irc_nick(source): - return source[1:source.find('!')] - -class PingThread(threading.Thread): - def __init__(self, irc): - threading.Thread.__init__(self) - self.daemon = True - self.irc = irc - def run(self): - debug('starting ping thread') - while not self.irc.give_up: - time.sleep(PING_INTERVAL) - try: - self.irc.ping_reply = False - #maybe use this to calculate the lag one day - self.irc.lockcond.acquire() - self.irc.send_raw('PING LAG' + str(int(time.time() * 1000))) - self.irc.lockcond.wait(PING_TIMEOUT) - self.irc.lockcond.release() - if not self.irc.ping_reply: - debug('irc ping timed out') - try: self.irc.close() - except IOError: pass - try: self.irc.fd.close() - except IOError: pass - try: - self.irc.sock.shutdown(socket.SHUT_RDWR) - self.irc.sock.close() - except IOError: pass - except IOError as e: - debug('ping thread: ' + repr(e)) - debug('ended ping thread') - -#handle one channel at a time -class IRCMessageChannel(MessageChannel): - #def run(self): pass - #def shutdown(self): pass - #def request_orderbook(self): pass - def fill_order(self, nick, oid, cj_amount, taker_pubkey): pass - def send_auth(self, nick, pubkey, sig): pass - def send_tx(self, nick, txhex): pass - def announce_orders(self, orderlist, nick=None): pass #nick=None means announce publicly - def cancel_orders(self, oid_list): pass - def send_error(self, nick, errormsg): pass - def send_pubkey(self, nick, pubkey): pass - def send_ioauth(self, nick, utxo_list, cj_pubkey, change_addr, sig): pass - def send_sigs(self, nick, sig_list): pass - - #def register_channel_callbacks(self, on_welcome=None, on_set_topic=None, - # on_connect=None, on_disconnect=None, on_nick_leave=None, on_nick_change=None): - #def register_orderbookwatch_callbacks(self, on_order_seen=None, - # on_order_cancel=None): - ''' - def register_taker_callbacks(self, on_error=None, on_pubkey=None, on_ioauth=None, - on_sigs=None): - def register_maker_callbacks(self, on_orderbook_requested=None, on_order_filled=None, - on_seen_auth=None, on_seen_tx=None): - ''' - - #def on_privmsg(self, nick, message): pass - #def on_pubmsg(self, nick, message): pass - ''' - def on_welcome(self): pass - def on_set_topic(self, newtopic): pass - def on_leave(self, nick): pass - def on_disconnect(self): pass - def on_connect(self): pass - #TODO implement on_nick_change - ''' - #close implies it will attempt to reconnect - def close(self): - try: - self.send_raw("QUIT") - except IOError as e: - debug('errored while trying to quit: ' + repr(e)) - - def shutdown(self): - self.close() - self.give_up = True - - def request_orderbook(self): - self.pubmsg(COMMAND_PREFIX + 'orderbook') - - - def pubmsg(self, message): - debug('>>pubmsg ' + message) - self.send_raw("PRIVMSG " + self.channel + " :" + message) - - def privmsg(self, nick, cmd, message): - debug('>>privmsg ' + 'nick=' + nick + ' cmd=' + cmd + ' msg=' + message) - #should we encrypt? - box = self.__encrypting(cmd, nick, sending=True) - #encrypt before chunking - if box: - message = enc_wrapper.encrypt_encode(message, box) - - if len(message) > 350: - message_chunks = chunks(message, 350) - else: - message_chunks = [message] - - for m in message_chunks: - trailer = ' ~' if m==message_chunks[-1] else ' ;' - header = "PRIVMSG " + nick + " :" - if m==message_chunks[0]: header += '!'+cmd + ' ' - self.send_raw(header + m + trailer) - - def send_raw(self, line): - #if not line.startswith('PING LAG'): - # debug('sendraw ' + line) - self.sock.sendall(line + '\r\n') - - def check_for_orders(self, nick, chunks): - if chunks[0] in ordername_list: - try: - counterparty = nick - oid = chunks[1] - ordertype = chunks[0] - minsize = chunks[2] - maxsize = chunks[3] - txfee = chunks[4] - cjfee = chunks[5] - if self.on_order_seen: - self.on_order_seen(counterparty, oid, - ordertype, minsize, maxsize, txfee, cjfee) - return True - except IndexError as e: - debug('index error parsing chunks') - #TODO what now? just ignore iirc - return False - - def __on_privmsg(self, nick, message): - '''handles the case when a private message is received''' - if message[0] != COMMAND_PREFIX: - return - for command in message[1:].split(COMMAND_PREFIX): - chunks = command.split(" ") - if self.check_for_orders(nick, chunks): - pass - - - def __on_pubmsg(self, nick, message): - if message[0] != COMMAND_PREFIX: - return - for command in message[1:].split(COMMAND_PREFIX): - chunks = command.split(" ") - if self.check_for_orders(nick, chunks): - pass - elif chunks[0] == 'cancel': - #!cancel [oid] - try: - oid = int(chunks[1]) - if self.on_order_cancel: - self.on_order_cancel(nick, oid) - except ValueError as e: - debug("!cancel " + repr(e)) - return - - def __encrypting(self, cmd, nick, sending=False): - '''Establish whether the message is to be - encrypted/decrypted based on the command string. - If so, retrieve the appropriate crypto_box object - and return. Sending/receiving flag enables us - to check which command strings correspond to which - type of object (maker/taker).''' - - if cmd in plaintext_commands: - return None - elif cmd not in encrypted_commands: - raise Exception("Invalid command type: " + cmd) - - maker_strings = ['tx','auth'] if not sending else ['ioauth','sig'] - taker_strings = ['ioauth','sig'] if not sending else ['tx','auth'] - - if cmd in maker_strings: - return self.active_orders[nick].crypto_box - elif cmd in taker_strings: - return self.cjtx.crypto_boxes[nick][1] - else: - raise Exception("Invalid command type: " + cmd) - - - def __handle_privmsg(self, source, target, message): - nick = get_irc_nick(source) - if message[0] == '\x01': - endindex = message[1:].find('\x01') - if endindex == -1: - return - ctcp = message[1:endindex + 1] - #self.send_raw('PRIVMSG ' + nick + ' :\x01VERSION - #TODO ctcp version here, since some servers dont let you get on without - - if target == self.nick: - if nick not in self.built_privmsg: - #new message starting - cmd_string = ''.join(message.split(' ')[0][1:]) - self.built_privmsg[nick] = [cmd_string, message[:-2]] - else: - self.built_privmsg[nick][1] += message[:-2] - box = self.__encrypting(self.built_privmsg[nick][0], nick) - if message[-1]==';': - self.waiting[nick]=True - elif message[-1]=='~': - self.waiting[nick]=False - if box: - #need to decrypt everything after the command string - to_decrypt = ''.join(self.built_privmsg[nick][1].split(' ')[1]) - decrypted = enc_wrapper.decode_decrypt(to_decrypt, box) - parsed = self.built_privmsg[nick][1].split(' ')[0] + ' ' + decrypted - else: - parsed = self.built_privmsg[nick][1] - #wipe the message buffer waiting for the next one - del self.built_privmsg[nick] - debug("< MAX_PRIVMSG_LEN: - irc.privmsg(target, prefix + line) - line = '' - if len(line) > 0: - irc.privmsg(target, prefix + line) - + +from common import * +from message_channel import MessageChannel + +import socket, threading, time +import base64, os +import enc_wrapper + +PING_INTERVAL = 40 +PING_TIMEOUT = 10 + +def get_irc_text(line): + return line[line[1:].find(':') + 2:] + +def get_irc_nick(source): + return source[1:source.find('!')] + +class PingThread(threading.Thread): + def __init__(self, irc): + threading.Thread.__init__(self) + self.daemon = True + self.irc = irc + def run(self): + debug('starting ping thread') + while not self.irc.give_up: + time.sleep(PING_INTERVAL) + try: + self.irc.ping_reply = False + #maybe use this to calculate the lag one day + self.irc.lockcond.acquire() + self.irc.send_raw('PING LAG' + str(int(time.time() * 1000))) + self.irc.lockcond.wait(PING_TIMEOUT) + self.irc.lockcond.release() + if not self.irc.ping_reply: + debug('irc ping timed out') + try: self.irc.close() + except IOError: pass + try: self.irc.fd.close() + except IOError: pass + try: + self.irc.sock.shutdown(socket.SHUT_RDWR) + self.irc.sock.close() + except IOError: pass + except IOError as e: + debug('ping thread: ' + repr(e)) + debug('ended ping thread') + +#handle one channel at a time +class IRCMessageChannel(MessageChannel): + #def run(self): pass + #def shutdown(self): pass + #def request_orderbook(self): pass + def fill_order(self, nick, oid, cj_amount, taker_pubkey): pass + def send_auth(self, nick, pubkey, sig): pass + def send_tx(self, nick, txhex): pass + def announce_orders(self, orderlist, nick=None): pass #nick=None means announce publicly + def cancel_orders(self, oid_list): pass + def send_error(self, nick, errormsg): pass + def send_pubkey(self, nick, pubkey): pass + def send_ioauth(self, nick, utxo_list, cj_pubkey, change_addr, sig): pass + def send_sigs(self, nick, sig_list): pass + + #def register_channel_callbacks(self, on_welcome=None, on_set_topic=None, + # on_connect=None, on_disconnect=None, on_nick_leave=None, on_nick_change=None): + #def register_orderbookwatch_callbacks(self, on_order_seen=None, + # on_order_cancel=None): + ''' + def register_taker_callbacks(self, on_error=None, on_pubkey=None, on_ioauth=None, + on_sigs=None): + def register_maker_callbacks(self, on_orderbook_requested=None, on_order_filled=None, + on_seen_auth=None, on_seen_tx=None): + ''' + + #def on_privmsg(self, nick, message): pass + #def on_pubmsg(self, nick, message): pass + ''' + def on_welcome(self): pass + def on_set_topic(self, newtopic): pass + def on_leave(self, nick): pass + def on_disconnect(self): pass + def on_connect(self): pass + #TODO implement on_nick_change + ''' + #close implies it will attempt to reconnect + def close(self): + try: + self.send_raw("QUIT") + except IOError as e: + debug('errored while trying to quit: ' + repr(e)) + + def shutdown(self): + self.close() + self.give_up = True + + def request_orderbook(self): + self.pubmsg(COMMAND_PREFIX + 'orderbook') + + + def pubmsg(self, message): + debug('>>pubmsg ' + message) + self.send_raw("PRIVMSG " + self.channel + " :" + message) + + def privmsg(self, nick, cmd, message): + debug('>>privmsg ' + 'nick=' + nick + ' cmd=' + cmd + ' msg=' + message) + #should we encrypt? + box = self.__encrypting(cmd, nick, sending=True) + #encrypt before chunking + if box: + message = enc_wrapper.encrypt_encode(message, box) + + if len(message) > 350: + message_chunks = chunks(message, 350) + else: + message_chunks = [message] + + for m in message_chunks: + trailer = ' ~' if m==message_chunks[-1] else ' ;' + header = "PRIVMSG " + nick + " :" + if m==message_chunks[0]: header += '!'+cmd + ' ' + self.send_raw(header + m + trailer) + + def send_raw(self, line): + #if not line.startswith('PING LAG'): + # debug('sendraw ' + line) + self.sock.sendall(line + '\r\n') + + def check_for_orders(self, nick, chunks): + if chunks[0] in ordername_list: + try: + counterparty = nick + oid = chunks[1] + ordertype = chunks[0] + minsize = chunks[2] + maxsize = chunks[3] + txfee = chunks[4] + cjfee = chunks[5] + if self.on_order_seen: + self.on_order_seen(counterparty, oid, + ordertype, minsize, maxsize, txfee, cjfee) + return True + except IndexError as e: + debug('index error parsing chunks') + #TODO what now? just ignore iirc + return False + + def __on_privmsg(self, nick, message): + '''handles the case when a private message is received''' + if message[0] != COMMAND_PREFIX: + return + for command in message[1:].split(COMMAND_PREFIX): + chunks = command.split(" ") + if self.check_for_orders(nick, chunks): + pass + + + def __on_pubmsg(self, nick, message): + if message[0] != COMMAND_PREFIX: + return + for command in message[1:].split(COMMAND_PREFIX): + chunks = command.split(" ") + if self.check_for_orders(nick, chunks): + pass + elif chunks[0] == 'cancel': + #!cancel [oid] + try: + oid = int(chunks[1]) + if self.on_order_cancel: + self.on_order_cancel(nick, oid) + except ValueError as e: + debug("!cancel " + repr(e)) + return + + def __encrypting(self, cmd, nick, sending=False): + '''Establish whether the message is to be + encrypted/decrypted based on the command string. + If so, retrieve the appropriate crypto_box object + and return. Sending/receiving flag enables us + to check which command strings correspond to which + type of object (maker/taker).''' + + if cmd in plaintext_commands: + return None + elif cmd not in encrypted_commands: + raise Exception("Invalid command type: " + cmd) + + maker_strings = ['tx','auth'] if not sending else ['ioauth','sig'] + taker_strings = ['ioauth','sig'] if not sending else ['tx','auth'] + + if cmd in maker_strings: + return self.active_orders[nick].crypto_box + elif cmd in taker_strings: + return self.cjtx.crypto_boxes[nick][1] + else: + raise Exception("Invalid command type: " + cmd) + + + def __handle_privmsg(self, source, target, message): + nick = get_irc_nick(source) + if message[0] == '\x01': + endindex = message[1:].find('\x01') + if endindex == -1: + return + ctcp = message[1:endindex + 1] + #self.send_raw('PRIVMSG ' + nick + ' :\x01VERSION + #TODO ctcp version here, since some servers dont let you get on without + + if target == self.nick: + if nick not in self.built_privmsg: + #new message starting + cmd_string = ''.join(message.split(' ')[0][1:]) + self.built_privmsg[nick] = [cmd_string, message[:-2]] + else: + self.built_privmsg[nick][1] += message[:-2] + box = self.__encrypting(self.built_privmsg[nick][0], nick) + if message[-1]==';': + self.waiting[nick]=True + elif message[-1]=='~': + self.waiting[nick]=False + if box: + #need to decrypt everything after the command string + to_decrypt = ''.join(self.built_privmsg[nick][1].split(' ')[1]) + decrypted = enc_wrapper.decode_decrypt(to_decrypt, box) + parsed = self.built_privmsg[nick][1].split(' ')[0] + ' ' + decrypted + else: + parsed = self.built_privmsg[nick][1] + #wipe the message buffer waiting for the next one + del self.built_privmsg[nick] + debug("< MAX_PRIVMSG_LEN: + irc.privmsg(target, prefix + line) + line = '' + if len(line) > 0: + irc.privmsg(target, prefix + line) + diff --git a/taker.py b/taker.py index 5551a895..77795209 100644 --- a/taker.py +++ b/taker.py @@ -2,7 +2,6 @@ from common import * import enc_wrapper -import irclib import bitcoin as btc import sqlite3, base64, threading, time, random From 79c228e5060c4dcce80c7187ff7610631aa95319 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Fri, 13 Feb 2015 20:18:24 +0200 Subject: [PATCH 123/409] improvements, nearly working --- blockchaininterface.py | 45 +++++++++++++++++++++++++++++------------- common.py | 24 ++++++++++++++++------ 2 files changed, 49 insertions(+), 20 deletions(-) diff --git a/blockchaininterface.py b/blockchaininterface.py index 9b1bac5e..fb93b229 100644 --- a/blockchaininterface.py +++ b/blockchaininterface.py @@ -66,12 +66,21 @@ def send_tx(self, tx_hex): print res return True - def get_utxos_from_addr(self,address): - res = json.loads(self.rpc(['listunspent','1','9999999','[\"'+address+'\"]'])) - utxos = [] - for r in res: - utxos.append(r['txid']+':'+str(r['vout'])) - return utxos + def get_utxos_from_addr(self,addresses): + for address in addresses: + res = json.loads(self.rpc(['listunspent','1','9999999','[\"'+address+'\"]'])) + #utxos = [] + #for r in res: + # utxos.append(r['txid']+':'+str(r['vout'])) + #return utxos + r = [] + for address in addresses: + utxos = [x for x in res if x['address']==address] + unspents=[] + for u in utxos: + unspents.append({'tx':u['txid'],'n':u['vout'],'amount':str(u['amount']),'address':address,'confirmations':u['confirmations']}) + r.append({'address':address,'unspent':unspents}) + return {'data':r} def get_txs_from_addr(self, addresses): #use listtransactions and then filter @@ -79,6 +88,9 @@ def get_txs_from_addr(self, addresses): #to get the last 1000 transactions TODO 1000 is arbitrary res = json.loads(self.rpc(['listtransactions','watchonly','1000','0','true'])) + #print "Got this res: " + #print res + result=[] for address in addresses: nbtxs = 0 @@ -89,6 +101,8 @@ def get_txs_from_addr(self, addresses): nbtxs += 1 txs.append({'confirmations':a['confirmations'],'tx':a['txid'],'amount':a['amount']}) result.append({'nb_txs':nbtxs,'address':address,'txs':txs}) + #print "Returning this data: " + #print result return {'data':result} def get_tx_info(self, txhash): @@ -104,7 +118,7 @@ def get_tx_info(self, txhash): return {'data':{'vouts':vouts}} - def get_balance_at_addr(self, address): + def get_balance_at_addr(self, addresses): #NB This will NOT return coinbase coins (but wont matter in our use case). #In order to have the Bitcoin RPC read balances at addresses #it doesn't own, we must import the addresses as watch-only @@ -112,8 +126,11 @@ def get_balance_at_addr(self, address): #TODO : there can be a performance issue with rescanning here. #allow importaddress to fail in case the address is already in the wallet - self.rpc(['importaddress',address,'watchonly'],[4]) - return int(Decimal(1e8) * Decimal(self.rpc(['getreceivedbyaddress',address]))) + res = [] + for address in addresses: + self.rpc(['importaddress',address,'watchonly'],[4]) + res.append({'address':address,'balance':int(Decimal(1e8) * Decimal(self.rpc(['getreceivedbyaddress',address])))}) + return {'data':res} def tick_forward_chain(self, n): '''Special method for regtest only; @@ -156,12 +173,12 @@ def main(): btc_client = bitcointoolsdir + 'bitcoin-cli' myBCI = RegTestImp(btc_client) #myBCI.send_tx('stuff') - print myBCI.get_utxos_from_addr("n4EjHhGVS4Rod8ociyviR3FH442XYMWweD") - print myBCI.get_balance_at_addr("n4EjHhGVS4Rod8ociyviR3FH442XYMWweD") - txid = myBCI.grab_coins('mioPqzCoFWcMTiWcYKo2Py6kwEp5vPzL2X',23) + print myBCI.get_utxos_from_addr(["n4EjHhGVS4Rod8ociyviR3FH442XYMWweD"]) + print myBCI.get_balance_at_addr(["n4EjHhGVS4Rod8ociyviR3FH442XYMWweD"]) + txid = myBCI.grab_coins('mtc6UaPPp2x1Fabugi8JG4BNouFo9rADNb',23) print txid - print myBCI.get_balance_at_addr('mioPqzCoFWcMTiWcYKo2Py6kwEp5vPzL2X') - print myBCI.get_utxos_from_addr('mioPqzCoFWcMTiWcYKo2Py6kwEp5vPzL2X') + print myBCI.get_balance_at_addr(['mtc6UaPPp2x1Fabugi8JG4BNouFo9rADNb']) + print myBCI.get_utxos_from_addr(['mtc6UaPPp2x1Fabugi8JG4BNouFo9rADNb']) if __name__ == '__main__': main() diff --git a/common.py b/common.py index 82653911..6cc1bfda 100644 --- a/common.py +++ b/common.py @@ -87,13 +87,13 @@ def get_blockchain_data(body, csv_params=[], if query_params: url += '?'+','.join(query_params) if blockchain_source=='blockr': - res = get_blockr_data(url) + res = json.loads(get_blockr_data(url)) elif blockchain_source=='regtest': res = get_regtest_data(url) else: - raise Exception("Unrecognised blockchain source" - "") - return json.loads(res)[output_key] + raise Exception("Unrecognised blockchain source") + + return res[output_key] def get_blockr_data(req): return btc.make_request(req) @@ -116,7 +116,16 @@ def get_regtest_data(req): txhash = req[2] #TODO currently only allowing one tx return myBCI.get_tx_info(txhash) elif req[0]=='addr' and req[1] == 'balance': - + addrs = req[2].split(',') + if 'unconfirmed' in addrs[-1]: + addrs = addrs[-1] + return myBCI.get_balance_at_addr(addrs) + elif req[0]=='address' and req[1] == 'unspent': + if '?' in req[2]: req[2] = req[2].split('?')[0] + addrs = req[2].split(',') + return myBCI.get_utxos_from_addr(addrs) + else: + raise Exception ("Unrecognized call to regtest blockchain interface") class Wallet(object): @@ -286,13 +295,16 @@ def find_unspent_addresses(self): # but dont know which privkey to sign with data = get_blockchain_data('addrunspent', csv_params=req, query_params=['unconfirmed=1']) + print 'got this addressunspent data: ' + print data if 'unspent' in data: data = [data] for dat in data: for u in dat['unspent']: if u['confirmations'] != 0: + u['amount'] = int(Decimal(1e8) * Decimal(u['amount'])) self.unspent[u['tx']+':'+str(u['n'])] = {'address': - dat['address'], 'value': int(u['amount'].replace('.', ''))} + dat['address'], 'value': u['amount']} def print_debug_wallet_info(self): debug('printing debug wallet information') From a4eb5b1a2db1d88f764d3fd44a54c9e433aeefb5 Mon Sep 17 00:00:00 2001 From: chris belcher Date: Fri, 13 Feb 2015 20:52:18 +0000 Subject: [PATCH 124/409] started implementing msgchan for taker, needs irc.py __encrypting() fix --- irc.py | 53 +++++++++++++++++++++++++++++++------ message_channel.py | 8 +++--- sendpayment.py | 22 +++++++++------- taker.py | 66 ++++++++++++++++++++-------------------------- 4 files changed, 90 insertions(+), 59 deletions(-) diff --git a/irc.py b/irc.py index c4425670..3c7045c8 100644 --- a/irc.py +++ b/irc.py @@ -50,9 +50,9 @@ class IRCMessageChannel(MessageChannel): #def run(self): pass #def shutdown(self): pass #def request_orderbook(self): pass - def fill_order(self, nick, oid, cj_amount, taker_pubkey): pass - def send_auth(self, nick, pubkey, sig): pass - def send_tx(self, nick, txhex): pass + #def fill_orders(self, nickoid_dict, cj_amount, taker_pubkey): pass + #def send_auth(self, nick, pubkey, sig): pass + #def send_tx(self, nick_list, txhex): pass def announce_orders(self, orderlist, nick=None): pass #nick=None means announce publicly def cancel_orders(self, oid_list): pass def send_error(self, nick, errormsg): pass @@ -66,7 +66,7 @@ def send_sigs(self, nick, sig_list): pass # on_order_cancel=None): ''' def register_taker_callbacks(self, on_error=None, on_pubkey=None, on_ioauth=None, - on_sigs=None): + on_sig=None): def register_maker_callbacks(self, on_orderbook_requested=None, on_order_filled=None, on_seen_auth=None, on_seen_tx=None): ''' @@ -92,15 +92,32 @@ def shutdown(self): self.close() self.give_up = True + #OrderbookWatch callback def request_orderbook(self): - self.pubmsg(COMMAND_PREFIX + 'orderbook') + self.__pubmsg(COMMAND_PREFIX + 'orderbook') + #Taker callbacks + def fill_orders(self, nickoid_dict, cj_amount, taker_pubkey): + for c, oid in nickoid_dict.iteritems(): + msg = str(oid) + ' ' + str(cj_amount) + ' ' + taker_pubkey + self.__privmsg(c, 'fill', msg) - def pubmsg(self, message): + def send_auth(self, nick, pubkey, sig): + message = pubkey + ' ' + sig + self.__privmsg(nick, 'auth', message) + + def send_tx(self, nick_list, txhex): + txb64 = base64.b64encode(txhex.decode('hex')) + for nick in nick_list: + self.__privmsg(nick, 'tx', txb64) + + + + def __pubmsg(self, message): debug('>>pubmsg ' + message) self.send_raw("PRIVMSG " + self.channel + " :" + message) - def privmsg(self, nick, cmd, message): + def __privmsg(self, nick, cmd, message): debug('>>privmsg ' + 'nick=' + nick + ' cmd=' + cmd + ' msg=' + message) #should we encrypt? box = self.__encrypting(cmd, nick, sending=True) @@ -137,10 +154,11 @@ def check_for_orders(self, nick, chunks): if self.on_order_seen: self.on_order_seen(counterparty, oid, ordertype, minsize, maxsize, txfee, cjfee) - return True except IndexError as e: debug('index error parsing chunks') #TODO what now? just ignore iirc + finally: + return True return False def __on_privmsg(self, nick, message): @@ -149,9 +167,28 @@ def __on_privmsg(self, nick, message): return for command in message[1:].split(COMMAND_PREFIX): chunks = command.split(" ") + + #orderbook watch commands if self.check_for_orders(nick, chunks): pass + #taker commands + elif chunks[0] == 'pubkey': + maker_pk = chunks[1] + if self.on_pubkey: + self.on_pubkey(nick, maker_pk) + elif chunks[0] == 'ioauth': + utxo_list = chunks[1].split(',') + cj_pub = chunks[2] + change_addr = chunks[3] + btc_sig = chunks[4] + if self.on_ioauth: + self.on_ioauth(nick, utxo_list, cj_pub, change_addr, btc_sig) + elif chunks[0] == 'sig': + sig = chunks[1] + if self.on_sig: + self.on_sig(sig) + def __on_pubmsg(self, nick, message): if message[0] != COMMAND_PREFIX: diff --git a/message_channel.py b/message_channel.py index 49841265..f51d0dbc 100644 --- a/message_channel.py +++ b/message_channel.py @@ -27,14 +27,14 @@ def request_orderbook(self): pass #taker commands def register_taker_callbacks(self, on_error=None, on_pubkey=None, on_ioauth=None, - on_sigs=None): + on_sig=None): self.on_error = on_error self.on_pubkey = on_pubkey self.on_ioauth = on_ioauth - self.on_sigs = on_sigs - def fill_order(self, nick, oid, cj_amount, taker_pubkey): pass + self.on_sig = on_sig + def fill_orders(self, nickoid_dict, cj_amount, taker_pubkey): pass def send_auth(self, nick, pubkey, sig): pass - def send_tx(self, nick, txhex): pass + def send_tx(self, nick_list, txhex): pass #maker commands def register_maker_callbacks(self, on_orderbook_requested=None, on_order_filled=None, diff --git a/sendpayment.py b/sendpayment.py index 01b8845a..28c4f2c7 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -2,6 +2,7 @@ from common import * import taker as takermodule +from irc import IRCMessageChannel import bitcoin as btc from optparse import OptionParser @@ -16,7 +17,7 @@ def __init__(self, taker): self.taker = taker def finishcallback(self): - self.taker.shutdown() + self.taker.msgchan.shutdown() def run(self): print 'waiting for all orders to certainly arrive' @@ -26,7 +27,7 @@ def run(self): counterparty_count = crow['COUNT(DISTINCT counterparty)'] if counterparty_count < self.taker.makercount: print 'not enough counterparties to fill order, ending' - self.taker.shutdown() + self.taker.msgchan.shutdown() return if self.taker.amount == 0: @@ -44,14 +45,14 @@ def run(self): print 'total amount spent = ' + str(total_amount) utxos = self.taker.wallet.select_utxos(self.taker.mixdepth, total_amount) - self.taker.cjtx = takermodule.CoinJoinTX(self.taker, self.taker.amount, + self.taker.cjtx = takermodule.CoinJoinTX(self.taker.msgchan, self.taker, self.taker.amount, orders, utxos, self.taker.destaddr, self.taker.wallet.get_change_addr(self.taker.mixdepth), self.taker.txfee, self.finishcallback) class SendPayment(takermodule.Taker): - def __init__(self, wallet, destaddr, amount, makercount, txfee, waittime, mixdepth): - takermodule.Taker.__init__(self) + def __init__(self, msgchan, wallet, destaddr, amount, makercount, txfee, waittime, mixdepth): + takermodule.Taker.__init__(self, msgchan) self.wallet = wallet self.destaddr = destaddr self.amount = amount @@ -92,16 +93,19 @@ def main(): wallet = Wallet(seed, options.mixdepth + 1) wallet.sync_wallet() - print 'starting irc' - taker = SendPayment(wallet, destaddr, amount, options.makercount, options.txfee, + irc = IRCMessageChannel(nickname) + taker = SendPayment(irc, wallet, destaddr, amount, options.makercount, options.txfee, options.waittime, options.mixdepth) try: - taker.run(HOST, PORT, nickname, CHANNEL) - finally: + print 'starting irc' + irc.run() + except: debug('CRASHING, DUMPING EVERYTHING') debug('wallet seed = ' + seed) debug_dump_object(wallet, ['addr_cache']) debug_dump_object(taker) + import traceback + traceback.print_exc() if __name__ == "__main__": main() diff --git a/taker.py b/taker.py index 77795209..8fba6ed0 100644 --- a/taker.py +++ b/taker.py @@ -7,7 +7,8 @@ import sqlite3, base64, threading, time, random class CoinJoinTX(object): - def __init__(self, taker, cj_amount, orders, my_utxos, my_cj_addr, + #soon the taker argument will be removed and just be replaced by wallet or some other interface + def __init__(self, msgchan, taker, cj_amount, orders, my_utxos, my_cj_addr, my_change_addr, my_txfee, finishcallback=None): ''' if my_change is None then there wont be a change address @@ -15,12 +16,13 @@ def __init__(self, taker, cj_amount, orders, my_utxos, my_cj_addr, orders is the orders you want to fill {'counterpartynick': oid, 'cp2': oid2} ''' debug('starting cj to ' + my_cj_addr + ' with change at ' + str(my_change_addr)) + self.msgchan = msgchan self.taker = taker self.cj_amount = cj_amount self.active_orders = dict(orders) self.nonrespondants = list(orders.keys()) self.my_utxos = my_utxos - self.utxos = {taker.nick: my_utxos} + self.utxos = {None: my_utxos} #None means they belong to me self.finishcallback = finishcallback self.my_txfee = my_txfee self.outputs = [{'address': my_cj_addr, 'value': self.cj_amount}] @@ -33,11 +35,8 @@ def __init__(self, taker, cj_amount, orders, my_utxos, my_cj_addr, #find the btc pubkey of the first utxo being used self.signing_btc_add = taker.wallet.unspent[self.my_utxos[0]]['address'] self.signing_btc_pub = btc.privtopub(taker.wallet.get_key_from_addr(self.signing_btc_add)) - for c, oid in orders.iteritems(): - cmd = 'fill' - omsg = str(oid) + ' ' + str(cj_amount) + ' ' + self.kp.hex_pk() - self.taker.privmsg(c, cmd, omsg) - + self.msgchan.fill_orders(orders, cj_amount, self.kp.hex_pk()) + def start_encryption(self, nick, maker_pk): if nick not in self.active_orders.keys(): raise Exception("Counterparty not part of this transaction.") @@ -48,8 +47,7 @@ def start_encryption(self, nick, maker_pk): self.taker.wallet.unspent[self.my_utxos[0]]['address']) my_btc_pub = btc.privtopub(my_btc_priv) my_btc_sig = btc.ecdsa_sign(self.kp.hex_pk(), my_btc_priv) - message = my_btc_pub + ' ' + my_btc_sig - self.taker.privmsg(nick, 'auth', message) + self.msgchan.send_auth(nick, my_btc_pub, my_btc_sig) def auth_counterparty(self, nick, btc_sig, cj_pub): '''Validate the counterpartys claim to own the btc @@ -102,9 +100,7 @@ def recv_txio(self, nick, utxo_list, cj_pub, change_addr): tx = btc.mktx(utxo_tx, self.outputs) import pprint debug('obtained tx\n' + pprint.pformat(btc.deserialize(tx))) - txb64 = base64.b64encode(tx.decode('hex')) - for nickk in self.active_orders.keys(): - self.taker.privmsg(nickk, 'tx', txb64) + self.msgchan.send_tx(self.active_orders.keys(), tx) #now sign it ourselves here for index, ins in enumerate(btc.deserialize(tx)['ins']): @@ -198,37 +194,31 @@ def on_set_topic(self, newtopic): #assume this only has one open cj tx at a time class Taker(OrderbookWatch): - def __init__(self): - OrderbookWatch.__init__(self) + def __init__(self, msgchan): + OrderbookWatch.__init__(self, msgchan) + msgchan.register_taker_callbacks(self.on_error, self.on_pubkey, + self.on_ioauth, self.on_sig) self.cjtx = None self.maker_pks = {} #TODO have a list of maker's nick we're coinjoining with, so # that some other guy doesnt send you confusing stuff #maybe a start_cj_tx() method is needed - - def on_privmsg(self, nick, message): - OrderbookWatch.on_privmsg(self, nick, message) - #debug("privmsg nick=%s message=%s" % (nick, message)) - if message[0] != COMMAND_PREFIX: - return - for command in message[1:].split(COMMAND_PREFIX): - chunks = command.split(" ") - if chunks[0] == 'pubkey': - maker_pk = chunks[1] - self.cjtx.start_encryption(nick, maker_pk) - if chunks[0] == 'ioauth': - utxo_list = chunks[1].split(',') - cj_pub = chunks[2] - change_addr = chunks[3] - btc_sig = chunks[4] - if not self.cjtx.auth_counterparty(nick, btc_sig, cj_pub): - print 'Authenticated encryption with counterparty: ' + nick + \ - ' not established. TODO: send rejection message' - return - self.cjtx.recv_txio(nick, utxo_list, cj_pub, change_addr) - elif chunks[0] == 'sig': - sig = chunks[1] - self.cjtx.add_signature(sig) + + def on_error(self): + pass #TODO implement + + def on_pubkey(self, nick, maker_pubkey): + self.cjtx.start_encryption(nick, maker_pubkey) + + def on_ioauth(self, nick, utxo_list, cj_pub, change_addr, btc_sig): + if not self.cjtx.auth_counterparty(nick, btc_sig, cj_pub): + print 'Authenticated encryption with counterparty: ' + nick + \ + ' not established. TODO: send rejection message' + return + self.cjtx.recv_txio(nick, utxo_list, cj_pub, change_addr) + + def on_sig(self, nick, sig): + self.cjtx.add_signature(sig) my_tx_fee = 10000 From 91104dfd2b865dbb55c70cad1de7509570013a49 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 15 Feb 2015 12:06:16 +0000 Subject: [PATCH 125/409] implemented __encryption() for message channel --- irc.py | 7 ++++--- taker.py | 16 ++++++++++++---- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/irc.py b/irc.py index 3c7045c8..842e7f3e 100644 --- a/irc.py +++ b/irc.py @@ -187,7 +187,7 @@ def __on_privmsg(self, nick, message): elif chunks[0] == 'sig': sig = chunks[1] if self.on_sig: - self.on_sig(sig) + self.on_sig(nick, sig) def __on_pubmsg(self, nick, message): @@ -219,7 +219,8 @@ def __encrypting(self, cmd, nick, sending=False): return None elif cmd not in encrypted_commands: raise Exception("Invalid command type: " + cmd) - + return self.cjpeer.get_crypto_box_from_nick(nick) + ''' maker_strings = ['tx','auth'] if not sending else ['ioauth','sig'] taker_strings = ['ioauth','sig'] if not sending else ['tx','auth'] @@ -229,7 +230,7 @@ def __encrypting(self, cmd, nick, sending=False): return self.cjtx.crypto_boxes[nick][1] else: raise Exception("Invalid command type: " + cmd) - + ''' def __handle_privmsg(self, source, target, message): nick = get_irc_nick(source) diff --git a/taker.py b/taker.py index 8fba6ed0..1283949a 100644 --- a/taker.py +++ b/taker.py @@ -143,17 +143,21 @@ def add_signature(self, sigb64): debug('the entire tx is signed, ready to pushtx()') print btc.serialize(self.latest_tx) - ret = btc.blockr_pushtx(btc.serialize(self.latest_tx), get_network()) - debug('pushed tx ' + str(ret)) + #ret = btc.blockr_pushtx(btc.serialize(self.latest_tx), get_network()) + #debug('pushed tx ' + str(ret)) if self.finishcallback != None: self.finishcallback() class CoinJoinerPeer(object): - pass + def __init__(self, msgchan): + self.msgchan = msgchan + + def get_crypto_box_from_nick(self, nick): + raise Exception() class OrderbookWatch(CoinJoinerPeer): def __init__(self, msgchan): - self.msgchan = msgchan + CoinJoinerPeer.__init__(self, msgchan) self.msgchan.register_orderbookwatch_callbacks(self.on_order_seen, self.on_order_cancel) self.msgchan.register_channel_callbacks(self.on_welcome, self.on_set_topic, @@ -198,12 +202,16 @@ def __init__(self, msgchan): OrderbookWatch.__init__(self, msgchan) msgchan.register_taker_callbacks(self.on_error, self.on_pubkey, self.on_ioauth, self.on_sig) + msgchan.cjpeer = self self.cjtx = None self.maker_pks = {} #TODO have a list of maker's nick we're coinjoining with, so # that some other guy doesnt send you confusing stuff #maybe a start_cj_tx() method is needed + def get_crypto_box_from_nick(self, nick): + return self.cjtx.crypto_boxes[nick][1] + def on_error(self): pass #TODO implement From 46c85479ce4034d12289e3df4a8441ffe837fce6 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 15 Feb 2015 14:23:07 +0000 Subject: [PATCH 126/409] converted maker/yieldgen to use MessageChannel but the encryption is broken --- irc.py | 158 +++++++++++++++++++++++++++++---------------- maker.py | 142 ++++++++++++++-------------------------- message_channel.py | 9 ++- yield-generator.py | 18 ++++-- 4 files changed, 169 insertions(+), 158 deletions(-) diff --git a/irc.py b/irc.py index 842e7f3e..b2caade2 100644 --- a/irc.py +++ b/irc.py @@ -1,6 +1,7 @@ from common import * from message_channel import MessageChannel +from message_channel import CJPeerError import socket, threading, time import base64, os @@ -47,40 +48,7 @@ def run(self): #handle one channel at a time class IRCMessageChannel(MessageChannel): - #def run(self): pass - #def shutdown(self): pass - #def request_orderbook(self): pass - #def fill_orders(self, nickoid_dict, cj_amount, taker_pubkey): pass - #def send_auth(self, nick, pubkey, sig): pass - #def send_tx(self, nick_list, txhex): pass - def announce_orders(self, orderlist, nick=None): pass #nick=None means announce publicly - def cancel_orders(self, oid_list): pass - def send_error(self, nick, errormsg): pass - def send_pubkey(self, nick, pubkey): pass - def send_ioauth(self, nick, utxo_list, cj_pubkey, change_addr, sig): pass - def send_sigs(self, nick, sig_list): pass - - #def register_channel_callbacks(self, on_welcome=None, on_set_topic=None, - # on_connect=None, on_disconnect=None, on_nick_leave=None, on_nick_change=None): - #def register_orderbookwatch_callbacks(self, on_order_seen=None, - # on_order_cancel=None): - ''' - def register_taker_callbacks(self, on_error=None, on_pubkey=None, on_ioauth=None, - on_sig=None): - def register_maker_callbacks(self, on_orderbook_requested=None, on_order_filled=None, - on_seen_auth=None, on_seen_tx=None): - ''' - - #def on_privmsg(self, nick, message): pass - #def on_pubmsg(self, nick, message): pass - ''' - def on_welcome(self): pass - def on_set_topic(self, newtopic): pass - def on_leave(self, nick): pass - def on_disconnect(self): pass - def on_connect(self): pass - #TODO implement on_nick_change - ''' + #close implies it will attempt to reconnect def close(self): try: @@ -92,6 +60,11 @@ def shutdown(self): self.close() self.give_up = True + def send_error(self, nick, errormsg): + debug('error<%s> : %s' % (nick, errormsg)) + self.__privmsg(nick, 'error', errormsg) + raise CJPeerError() + #OrderbookWatch callback def request_orderbook(self): self.__pubmsg(COMMAND_PREFIX + 'orderbook') @@ -111,7 +84,35 @@ def send_tx(self, nick_list, txhex): for nick in nick_list: self.__privmsg(nick, 'tx', txb64) + #Maker callbacks + def announce_orders(self, orderlist, nick=None): + #nick=None means announce publicly + order_keys = ['oid', 'minsize', 'maxsize', 'txfee', 'cjfee'] + orderline = '' + for order in orderlist: + #TODO send all the orders on one line + elem_list = [str(order[k]) for k in order_keys] + if nick: + self.__privmsg(nick, order['ordertype'], ' '.join(elem_list)) + else: + self.__pubmsg(COMMAND_PREFIX + order['ordertype'] + ' ' + ' '.join(elem_list)) + + def cancel_orders(self, oid_list): + clines = [COMMAND_PREFIX + 'cancel ' + str(oid) for oid in oid_list] + self.pubmsg(''.join(clines)) + def send_pubkey(self, nick, pubkey): + self.__privmsg(nick, 'pubkey', pubkey) + + def send_ioauth(self, nick, utxo_list, cj_pubkey, change_addr, sig): + authmsg = (','.join(utxo_list) + ' ' + + cj_pubkey + ' ' + change_addr + ' ' + sig) + self.__privmsg(nick, 'ioauth', authmsg) + + def send_sigs(self, nick, sig_list): + #TODO make it send the sigs on one line if there's space + for s in sigs_list: + self.__privmsg(nick, 'sig', s) def __pubmsg(self, message): debug('>>pubmsg ' + message) @@ -124,6 +125,7 @@ def __privmsg(self, nick, cmd, message): #encrypt before chunking if box: message = enc_wrapper.encrypt_encode(message, box) + print 'emsg=' + message if len(message) > 350: message_chunks = chunks(message, 350) @@ -134,6 +136,7 @@ def __privmsg(self, nick, cmd, message): trailer = ' ~' if m==message_chunks[-1] else ' ;' header = "PRIVMSG " + nick + " :" if m==message_chunks[0]: header += '!'+cmd + ' ' + print 'sendraw ' + header + m + trailer self.send_raw(header + m + trailer) def send_raw(self, line): @@ -167,28 +170,61 @@ def __on_privmsg(self, nick, message): return for command in message[1:].split(COMMAND_PREFIX): chunks = command.split(" ") - - #orderbook watch commands - if self.check_for_orders(nick, chunks): - pass - - #taker commands - elif chunks[0] == 'pubkey': - maker_pk = chunks[1] - if self.on_pubkey: - self.on_pubkey(nick, maker_pk) - elif chunks[0] == 'ioauth': - utxo_list = chunks[1].split(',') - cj_pub = chunks[2] - change_addr = chunks[3] - btc_sig = chunks[4] - if self.on_ioauth: - self.on_ioauth(nick, utxo_list, cj_pub, change_addr, btc_sig) - elif chunks[0] == 'sig': - sig = chunks[1] - if self.on_sig: - self.on_sig(nick, sig) + #looks like a very similar pattern for all of these + # check for a command name, parse arguments, call a function + # maybe we need some eval() trickery to do it better + try: + #orderbook watch commands + if self.check_for_orders(nick, chunks): + pass + + #taker commands + elif chunks[0] == 'pubkey': + maker_pk = chunks[1] + if self.on_pubkey: + self.on_pubkey(nick, maker_pk) + elif chunks[0] == 'ioauth': + utxo_list = chunks[1].split(',') + cj_pub = chunks[2] + change_addr = chunks[3] + btc_sig = chunks[4] + if self.on_ioauth: + self.on_ioauth(nick, utxo_list, cj_pub, change_addr, btc_sig) + elif chunks[0] == 'sig': + sig = chunks[1] + if self.on_sig: + self.on_sig(nick, sig) + + #maker commands + if chunks[0] == 'fill': + try: + oid = int(chunks[1]) + amount = int(chunks[2]) + taker_pk = chunks[3] + if self.on_order_fill: + self.on_order_fill(nick, oid, amount, taker_pk) + except (ValueError, IndexError) as e: + self.send_error(nick, str(e)) + elif chunks[0] == 'auth': + try: + i_utxo_pubkey = chunks[1] + btc_sig = chunks[2] + if self.on_seen_auth: + self.on_seen_auth(nick, i_utxo_pubkey, btc_sig) + except (ValueError, IndexError) as e: + self.send_error(nick, str(e)) + elif chunks[0] == 'tx': + b64tx = chunks[1] + try: + txhex = base64.b64decode(b64tx).encode('hex') + if self.on_seen_tx: + self.on_seen_tx(nick, txhex) + except TypeError as e: + self.send_error(nick, 'bad base64 tx. ' + repr(e)) + except CJPeerError: + #TODO proper error handling + continue def __on_pubmsg(self, nick, message): if message[0] != COMMAND_PREFIX: @@ -206,6 +242,9 @@ def __on_pubmsg(self, nick, message): except ValueError as e: debug("!cancel " + repr(e)) return + elif chunks[0] == 'orderbook': + if self.on_orderbook_requested: + self.on_orderbook_requested(nick) def __encrypting(self, cmd, nick, sending=False): '''Establish whether the message is to be @@ -219,6 +258,7 @@ def __encrypting(self, cmd, nick, sending=False): return None elif cmd not in encrypted_commands: raise Exception("Invalid command type: " + cmd) + return self.cjpeer.get_crypto_box_from_nick(nick) ''' maker_strings = ['tx','auth'] if not sending else ['ioauth','sig'] @@ -231,6 +271,7 @@ def __encrypting(self, cmd, nick, sending=False): else: raise Exception("Invalid command type: " + cmd) ''' + def __handle_privmsg(self, source, target, message): nick = get_irc_nick(source) @@ -249,7 +290,9 @@ def __handle_privmsg(self, source, target, message): self.built_privmsg[nick] = [cmd_string, message[:-2]] else: self.built_privmsg[nick][1] += message[:-2] - box = self.__encrypting(self.built_privmsg[nick][0], nick) + box = self.__encrypting(self.built_privmsg[nick][0], nick) + print 'cmd=' + self.built_privmsg[nick][0] + ' nick=' + nick + ' box=' + str(box) + print 'msg=' + message if message[-1]==';': self.waiting[nick]=True elif message[-1]=='~': @@ -329,6 +372,7 @@ def __handle_line(self, line): ''' def __init__(self, nick, server=HOST, port=PORT, channel=CHANNEL, username='username', realname='realname'): + self.cjpeer = None #subclasses have to set this to self self.nick = nick self.serverport = (server, port) self.channel = channel @@ -341,7 +385,7 @@ def run(self): self.give_up = False self.ping_reply = True self.lockcond = threading.Condition() - PingThread(self).start() + #PingThread(self).start() while self.connect_attempts < 10 and not self.give_up: try: diff --git a/maker.py b/maker.py index 152c9dcb..965e44d6 100644 --- a/maker.py +++ b/maker.py @@ -1,7 +1,7 @@ #! /usr/bin/env python from common import * -import irclib +from taker import CoinJoinerPeer import bitcoin as btc import base64, pprint, threading import enc_wrapper @@ -33,7 +33,7 @@ def __init__(self, maker, nick, oid, amount, taker_pk): #always a new address even if the order ends up never being # furfilled, you dont want someone pretending to fill all your # orders to find out which addresses you use - self.maker.privmsg(nick, 'pubkey', self.kp.hex_pk()) + self.maker.msgchan.send_pubkey(nick, self.kp.hex_pk()) def auth_counterparty(self, nick, i_utxo_pubkey, btc_sig): self.i_utxo_pubkey = i_utxo_pubkey @@ -48,16 +48,10 @@ def auth_counterparty(self, nick, i_utxo_pubkey, btc_sig): btc_key = self.maker.wallet.get_key_from_addr(self.cj_addr) btc_pub = btc.privtopub(btc_key) btc_sig = btc.ecdsa_sign(self.kp.hex_pk(), btc_key) - authmsg = str(','.join(self.utxos)) + ' ' + \ - btc_pub + ' ' + self.change_addr + ' ' + btc_sig - self.maker.privmsg(nick, 'ioauth', authmsg) + self.maker.msgchan.send_ioauth(nick, self.utxos, btc_pub, self.change_addr, btc_sig) return True - def recv_tx(self, nick, b64tx): - try: - txhex = base64.b64decode(b64tx).encode('hex') - except TypeError as e: - self.maker.send_error(nick, 'bad base64 tx. ' + repr(e)) + def recv_tx(self, nick, txhex): try: self.tx = btc.deserialize(txhex) except IndexError as e: @@ -81,8 +75,7 @@ def recv_tx(self, nick, b64tx): add_addr_notify(self.change_addr, self.unconfirm_callback, self.confirm_callback) debug('sending sigs ' + str(sigs)) - for s in sigs: - self.maker.privmsg(nick, 'sig', s) + self.maker.msgchan.send_sigs(nick, sigs) self.maker.active_orders[nick] = None def unconfirm_callback(self, balance): @@ -150,90 +143,56 @@ def verify_unsigned_tx(self, txd): class CJMakerOrderError(StandardError): pass -class Maker(irclib.IRCClient): - def __init__(self, wallet): +class Maker(CoinJoinerPeer): + def __init__(self, msgchan, wallet): + CoinJoinerPeer.__init__(self, msgchan) + self.msgchan.register_channel_callbacks(self.on_welcome, self.on_set_topic, + None, None, self.on_nick_leave, None) + msgchan.register_maker_callbacks(self.on_orderbook_requested, + self.on_order_fill, self.on_seen_auth, self.on_seen_tx) + msgchan.cjpeer = self + self.active_orders = {} self.wallet = wallet self.nextoid = -1 self.orderlist = self.create_my_orders() self.wallet_unspent_lock = threading.Lock() - def privmsg_all_orders(self, target, orderlist=None): - if orderlist == None: - orderlist = self.orderlist - order_keys = ['oid', 'minsize', 'maxsize', 'txfee', 'cjfee'] - orderline = '' - for order in orderlist: - elem_list = [str(order[k]) for k in order_keys] - self.privmsg(target, order['ordertype'],' '.join(elem_list)) - - def send_error(self, nick, errmsg): - debug('error<%s> : %s' % (nick, errmsg)) - self.privmsg(nick,'error', errmsg) - raise CJMakerOrderError() + def get_crypto_box_from_nick(self, nick): + return self.active_orders[nick].crypto_box - def on_welcome(self): - self.privmsg_all_orders(CHANNEL) - - def on_privmsg(self, nick, message): - if message[0] != COMMAND_PREFIX: - return - command_lines = message.split(COMMAND_PREFIX) - for command_line in command_lines: - if len(command_line) == 0: - continue - chunks = command_line.split(" ") - try: - if len(chunks) < 2: - self.send_error(nick, 'Not enough arguments') - - if chunks[0] == 'fill': - if nick in self.active_orders and self.active_orders[nick] != None: - self.active_orders[nick] = None - debug('had a partially filled order but starting over now') - try: - oid = int(chunks[1]) - amount = int(chunks[2]) - taker_pk = chunks[3] - except (ValueError, IndexError) as e: - self.send_error(nick, str(e)) - self.wallet_unspent_lock.acquire() - try: - self.active_orders[nick] = CoinJoinOrder(self, nick, oid, amount, taker_pk) - finally: - self.wallet_unspent_lock.release() - elif chunks[0] == 'auth': - if nick not in self.active_orders or self.active_orders[nick] == None: - self.send_error(nick, 'No open order from this nick') - cjorder = self.active_orders[nick] - try: - i_utxo_pubkey = chunks[1] - btc_sig = chunks[2] - except (ValueError, IndexError) as e: - self.send_error(nick, str(e)) - self.active_orders[nick].auth_counterparty(nick, i_utxo_pubkey, btc_sig) - - elif chunks[0] == 'tx': - if nick not in self.active_orders or self.active_orders[nick] == None: - self.send_error(nick, 'No open order from this nick') - b64tx = chunks[1] - self.wallet_unspent_lock.acquire() - try: - self.active_orders[nick].recv_tx(nick, b64tx) - finally: - self.wallet_unspent_lock.release() - except CJMakerOrderError: - self.active_orders[nick] = None - continue + def on_orderbook_requested(self, nick): + self.msgchan.announce_orders(self.orderlist, nick) + + def on_order_fill(self, nick, oid, amount, taker_pubkey): + if nick in self.active_orders and self.active_orders[nick] != None: + self.active_orders[nick] = None + debug('had a partially filled order but starting over now') + self.wallet_unspent_lock.acquire() + try: + self.active_orders[nick] = CoinJoinOrder(self, nick, oid, amount, taker_pubkey) + finally: + self.wallet_unspent_lock.release() + + def on_seen_auth(self, nick, pubkey, sig): + if nick not in self.active_orders or self.active_orders[nick] == None: + self.send_error(nick, 'No open order from this nick') + self.active_orders[nick].auth_counterparty(nick, pubkey, sig) + #TODO if auth_counterparty returns false, remove this order from active_orders + # and send an error + + def on_seen_tx(self, nick, txhex): + if nick not in self.active_orders or self.active_orders[nick] == None: + self.send_error(nick, 'No open order from this nick') + self.wallet_unspent_lock.acquire() + try: + self.active_orders[nick].recv_tx(nick, txhex) + finally: + self.wallet_unspent_lock.release() - #each order has an id for referencing to and looking up - # using the same id again overwrites it, they'll be plenty of times when an order - # has to be modified and its better to just have !order rather than !cancelorder then !order - def on_pubmsg(self, nick, message): - if message[0] == COMMAND_PREFIX: - chunks = message[1:].split(" ") - if chunks[0] == 'orderbook': - self.privmsg_all_orders(nick) + def on_welcome(self): + self.msgchan.announce_orders(self.orderlist) + self.active_orders = {} def on_set_topic(self, newtopic): chunks = newtopic.split('|') @@ -243,7 +202,7 @@ def on_set_topic(self, newtopic): print chunks[1].strip() print '=' * 60 - def on_leave(self, nick): + def on_nick_leave(self, nick): self.active_orders[nick] = None def modify_orders(self, to_cancel, to_announce): @@ -252,10 +211,9 @@ def modify_orders(self, to_cancel, to_announce): order = [o for o in self.orderlist if o['oid'] == oid][0] self.orderlist.remove(order) if len(to_cancel) > 0: - clines = [COMMAND_PREFIX + 'cancel ' + str(oid) for oid in to_cancel] - self.pubmsg(''.join(clines)) + self.msgchan.cancel_orders(to_cancel) if len(to_announce) > 0: - self.privmsg_all_orders(CHANNEL, to_announce) + self.msgchan.announce_orders(to_announce) for ann in to_announce: oldorder_s = [order for order in self.orderlist if order['oid'] == ann['oid']] if len(oldorder_s) > 0: diff --git a/message_channel.py b/message_channel.py index f51d0dbc..db3dfdf2 100644 --- a/message_channel.py +++ b/message_channel.py @@ -1,4 +1,7 @@ +class CJPeerError(StandardError): + pass + class MessageChannel(object): ''' Abstract class which implements a way for bots to communicate @@ -6,6 +9,7 @@ class MessageChannel(object): def run(self): pass def shutdown(self): pass + def send_error(self, nick, errormsg): pass #callbacks for everyone #some of these many not have meaning in a future channel, like bitmessage @@ -37,15 +41,14 @@ def send_auth(self, nick, pubkey, sig): pass def send_tx(self, nick_list, txhex): pass #maker commands - def register_maker_callbacks(self, on_orderbook_requested=None, on_order_filled=None, + def register_maker_callbacks(self, on_orderbook_requested=None, on_order_fill=None, on_seen_auth=None, on_seen_tx=None): self.on_orderbook_requested = on_orderbook_requested - self.on_order_filled = on_order_filled + self.on_order_fill = on_order_fill self.on_seen_auth = on_seen_auth self.on_seen_tx = on_seen_tx def announce_orders(self, orderlist, nick=None): pass #nick=None means announce publicly def cancel_orders(self, oid_list): pass - def send_error(self, nick, errormsg): pass def send_pubkey(self, nick, pubkey): pass def send_ioauth(self, nick, utxo_list, cj_pubkey, change_addr, sig): pass def send_sigs(self, nick, sig_list): pass diff --git a/yield-generator.py b/yield-generator.py index 1453a32b..1f95e2e5 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -1,6 +1,7 @@ #! /usr/bin/env python from maker import * +from irc import IRCMessageChannel import bitcoin as btc import time @@ -27,8 +28,10 @@ #spent from utxos that try to make the highest balance even higher # so try to keep coins concentrated in one mixing depth class YieldGenerator(Maker): - def __init__(self, wallet): - Maker.__init__(self, wallet) + def __init__(self, msgchan, wallet): + Maker.__init__(self, msgchan, wallet) + self.msgchan.register_channel_callbacks(self.on_welcome, self.on_set_topic, + self.on_connect, None, self.on_nick_leave, None) def on_connect(self): if len(nickserv_password) > 0: @@ -83,15 +86,18 @@ def main(): wallet = Wallet(seed, max_mix_depth = mix_levels) wallet.sync_wallet() - maker = YieldGenerator(wallet) - print 'connecting to irc' + irc = IRCMessageChannel(nickname) + maker = YieldGenerator(irc, wallet) try: - maker.run(HOST, PORT, nickname, CHANNEL) - finally: + print 'connecting to irc' + irc.run() + except: debug('CRASHING, DUMPING EVERYTHING') debug('wallet seed = ' + seed) debug_dump_object(wallet, ['addr_cache']) debug_dump_object(maker) + import traceback + traceback.print_exc() if __name__ == "__main__": main() From bc0efe7f7402dda2e2dda48001d2f53c987e3f15 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 15 Feb 2015 14:52:21 +0000 Subject: [PATCH 127/409] fixed bug in issue #26 --- common.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/common.py b/common.py index 78de40c7..5fa820e3 100644 --- a/common.py +++ b/common.py @@ -362,11 +362,12 @@ def create_combination(li, n): For example, combination(['apple', 'orange', 'pear'], 2) = [('apple', 'orange'), ('apple', 'pear'), ('orange', 'pear')] ''' - if n < 2: - raise ValueError('n must be >= 2') result = [] - if n == 2: - #creates a list oft + if n == 1: + result = [(l,) for l in li] #same thing but each order is a tuple + elif n == 2: + #this case could be removed and the function completely recurvsive + # but for n=2 this is slightly more efficent for i, e1 in enumerate(li): for e2 in li[i+1:]: result.append((e1, e2)) From 9047395ac38caea09835df65f35eb7d5fc79b2d4 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 15 Feb 2015 21:36:25 +0000 Subject: [PATCH 128/409] made TestTaker and testmaker work with Message Channel --- irc.py | 3 ++ maker.py | 12 ++++-- taker.py | 115 +++++++++++++++++++++++++++---------------------------- 3 files changed, 68 insertions(+), 62 deletions(-) diff --git a/irc.py b/irc.py index b2caade2..1d82abcb 100644 --- a/irc.py +++ b/irc.py @@ -245,6 +245,9 @@ def __on_pubmsg(self, nick, message): elif chunks[0] == 'orderbook': if self.on_orderbook_requested: self.on_orderbook_requested(nick) + else: + if self.debug_on_pubmsg_cmd: + self.debug_on_pubmsg_cmd(nick, chunks) def __encrypting(self, cmd, nick, sending=False): '''Establish whether the message is to be diff --git a/maker.py b/maker.py index 965e44d6..153f6034 100644 --- a/maker.py +++ b/maker.py @@ -310,15 +310,19 @@ def main(): wallet = Wallet(seed, max_mix_depth=5) wallet.sync_wallet() - maker = Maker(wallet) - print 'connecting to irc' + from irc import IRCMessageChannel + irc = IRCMessageChannel(nickname) + maker = Maker(irc, wallet) try: - maker.run(HOST, PORT, nickname, CHANNEL) - finally: + print 'connecting to irc' + irc.run() + except: debug('CRASHING, DUMPING EVERYTHING') debug('wallet seed = ' + seed) debug_dump_object(wallet, ['addr_cache']) debug_dump_object(maker) + import traceback + traceback.print_exc() if __name__ == "__main__": main() diff --git a/taker.py b/taker.py index 1283949a..41080bf1 100644 --- a/taker.py +++ b/taker.py @@ -231,66 +231,61 @@ def on_sig(self, nick, sig): my_tx_fee = 10000 class TestTaker(Taker): - def __init__(self, wallet): - Taker.__init__(self) - self.wallet = wallet + def __init__(self, msgchan, wallet): + Taker.__init__(self, msgchan) + self.wallet = wallet + self.msgchan.debug_on_pubmsg_cmd = self.debug_on_pubmsg_cmd def finish_callback(self): removed_utxos = self.wallet.remove_old_utxos(self.cjtx.latest_tx) added_utxos = self.wallet.add_new_utxos(self.cjtx.latest_tx, btc.txhash(btc.serialize(self.cjtx.latest_tx))) debug('tx published, added_utxos=\n' + pprint.pformat(added_utxos)) debug('removed_utxos=\n' + pprint.pformat(removed_utxos)) - - def on_pubmsg(self, nick, message): - Taker.on_pubmsg(self, nick, message) - if message[0] != COMMAND_PREFIX: - return - for command in message[1:].split(COMMAND_PREFIX): - #commands starting with % are for testing and will be removed in the final version - chunks = command.split(" ") - if chunks[0] == '%go': - #!%go [counterparty] [oid] [amount] - cp = chunks[1] - oid = chunks[2] - amt = chunks[3] - #this testing command implements a very dumb algorithm. - #just take 1 utxo from anywhere and output it to a level 1 - #change address. - utxo_dict = self.wallet.get_utxo_list_by_mixdepth() - utxo_list = [x for v in utxo_dict.itervalues() for x in v] - unspent = [{'utxo': utxo, 'value': self.wallet.unspent[utxo]['value']} \ - for utxo in utxo_list] - inputs = btc.select(unspent, amt) - utxos = [i['utxo'] for i in inputs] - print 'making cjtx' - self.cjtx = CoinJoinTX(self, int(amt), {cp: oid}, - utxos, self.wallet.get_receive_addr(mixing_depth=1), - self.wallet.get_change_addr(mixing_depth=0), my_tx_fee, self.finish_callback) - elif chunks[0] == '%unspent': - from pprint import pprint - pprint(self.wallet.unspent) - elif chunks[0] == '%fill': - #!fill [counterparty] [oid] [amount] [utxo] - counterparty = chunks[1] - oid = int(chunks[2]) - amount = chunks[3] - my_utxo = chunks[4] - print 'making cjtx' - self.cjtx = CoinJoinTX(self, int(amount), {counterparty: oid}, - [my_utxo], self.wallet.get_receive_addr(mixing_depth=1), - self.wallet.get_change_addr(mixing_depth=0), my_tx_fee, self.finish_callback) - elif chunks[0] == '%2fill': - #!2fill [amount] [utxo] [counterparty1] [oid1] [counterparty2] [oid2] - amount = int(chunks[1]) - my_utxo = chunks[2] - cp1 = chunks[3] - oid1 = int(chunks[4]) - cp2 = chunks[5] - oid2 = int(chunks[6]) - print 'creating cjtx' - self.cjtx = CoinJoinTX(self, amount, {cp1: oid1, cp2: oid2}, - [my_utxo], self.wallet.get_receive_addr(mixing_depth=1), - self.wallet.get_change_addr(mixing_depth=0), my_tx_fee, self.finish_callback) + + def debug_on_pubmsg_cmd(self, nick, chunks): + if chunks[0] == '%go': + #!%go [counterparty] [oid] [amount] + cp = chunks[1] + oid = chunks[2] + amt = chunks[3] + #this testing command implements a very dumb algorithm. + #just take 1 utxo from anywhere and output it to a level 1 + #change address. + utxo_dict = self.wallet.get_utxo_list_by_mixdepth() + utxo_list = [x for v in utxo_dict.itervalues() for x in v] + unspent = [{'utxo': utxo, 'value': self.wallet.unspent[utxo]['value']} \ + for utxo in utxo_list] + inputs = btc.select(unspent, amt) + utxos = [i['utxo'] for i in inputs] + print 'making cjtx' + self.cjtx = CoinJoinTX(self, int(amt), {cp: oid}, + utxos, self.wallet.get_receive_addr(mixing_depth=1), + self.wallet.get_change_addr(mixing_depth=0), my_tx_fee, self.finish_callback) + elif chunks[0] == '%unspent': + from pprint import pprint + pprint(self.wallet.unspent) + elif chunks[0] == '%fill': + #!fill [counterparty] [oid] [amount] [utxo] + counterparty = chunks[1] + oid = int(chunks[2]) + amount = chunks[3] + my_utxo = chunks[4] + print 'making cjtx' + self.cjtx = CoinJoinTX(self.msgchan, self, int(amount), {counterparty: oid}, + [my_utxo], self.wallet.get_receive_addr(mixing_depth=1), + self.wallet.get_change_addr(mixing_depth=0), my_tx_fee, self.finish_callback) + elif chunks[0] == '%2fill': + #!2fill [amount] [utxo] [counterparty1] [oid1] [counterparty2] [oid2] + amount = int(chunks[1]) + my_utxo = chunks[2] + cp1 = chunks[3] + oid1 = int(chunks[4]) + cp2 = chunks[5] + oid2 = int(chunks[6]) + print 'creating cjtx' + self.cjtx = CoinJoinTX(self.msgchan, self, amount, {cp1: oid1, cp2: oid2}, + [my_utxo], self.wallet.get_receive_addr(mixing_depth=1), + self.wallet.get_change_addr(mixing_depth=0), my_tx_fee, self.finish_callback) def main(): import sys @@ -301,15 +296,19 @@ def main(): wallet = Wallet(seed, max_mix_depth=5) wallet.sync_wallet() - print 'starting irc' - taker = TestTaker(wallet) + from irc import IRCMessageChannel + irc = IRCMessageChannel(nickname) + taker = TestTaker(irc, wallet) try: - taker.run(HOST, PORT, nickname, CHANNEL) - finally: + print 'connecting to irc' + irc.run() + except: debug('CRASHING, DUMPING EVERYTHING') debug('wallet seed = ' + seed) debug_dump_object(wallet, ['addr_cache']) debug_dump_object(taker) + import traceback + traceback.print_exc() if __name__ == "__main__": main() From a0c6cadbc529c51c9b2f5d4a9c99805158065d93 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 15 Feb 2015 21:53:55 +0000 Subject: [PATCH 129/409] slight bugfix --- irc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/irc.py b/irc.py index 1d82abcb..5f25f654 100644 --- a/irc.py +++ b/irc.py @@ -246,7 +246,7 @@ def __on_pubmsg(self, nick, message): if self.on_orderbook_requested: self.on_orderbook_requested(nick) else: - if self.debug_on_pubmsg_cmd: + if hasattr(self, 'debug_on_pubmsg_cmd'): self.debug_on_pubmsg_cmd(nick, chunks) def __encrypting(self, cmd, nick, sending=False): From d400b53a406822d7bcddb37084c1afb7524b5c8f Mon Sep 17 00:00:00 2001 From: Belcher Date: Mon, 16 Feb 2015 00:02:46 +0000 Subject: [PATCH 130/409] first start at tumbler.py --- tumbler.py | 187 +++++++++++++++++++++++++++-------------------------- 1 file changed, 95 insertions(+), 92 deletions(-) diff --git a/tumbler.py b/tumbler.py index caf92eab..9dd67310 100644 --- a/tumbler.py +++ b/tumbler.py @@ -1,107 +1,110 @@ +from optparse import OptionParser +import datetime +import numpy as np +from pprint import pprint -algo_thread = None +def int_lower_bounded(thelist, lowerbound): + return [int(l) if int(l) >= lowerbound else lowerbound for l in thelist] -#how long to wait for all the orders to arrive before starting to do coinjoins -ORDER_ARRIVAL_WAIT_TIME = 2 +def generate_tumbler_tx(destaddrs, options): + #sends the coins up through a few mixing depths + #send to the destination addresses from different mixing depths -def choose_order(cj_amount): - - sqlorders = db.execute('SELECT * FROM orderbook;').fetchall() - orders = [(o['counterparty'], o['oid'], calc_cj_fee(o['ordertype'], o['cjfee'], cj_amount)) - for o in sqlorders if cj_amount >= o['minsize'] or cj_amount <= o['maxsize']] - orders = sorted(orders, key=lambda k: k[2]) - print 'orders = ' + str(orders) - return orders[0] #choose the cheapest, later this will be chosen differently + #simple algo, move coins completely from one mixing depth to the next + # until you get to the end, then send to destaddrs -def choose_sweep_order(my_total_input, my_tx_fee): - ''' - choose an order given that we want to be left with no change - i.e. sweep an entire group of utxos + #txcounts for going completely from one mixdepth to the next + # follows a normal distribution + txcounts = np.random.normal(options.txcountparams[0], + options.txcountparams[1], options.mixdepthcount) + txcounts = int_lower_bounded(txcounts, 1) - solve for mychange = 0 - ABS FEE - mychange = totalin - cjamount - mytxfee - absfee - => cjamount = totalin - mytxfee - absfee - REL FEE - mychange = totalin - cjamount - mytxfee - relfee*cjamount - => 0 = totalin - mytxfee - cjamount*(1 + relfee) - => cjamount = (totalin - mytxfee) / (1 + relfee) - ''' - def calc_zero_change_cj_amount(ordertype, cjfee): - cj_amount = None - if ordertype == 'absorder': - cj_amount = my_total_input - my_tx_fee - cjfee - elif ordertype == 'relorder': - cj_amount = (my_total_input - my_tx_fee) / (Decimal(cjfee) + 1) - cj_amount = int(cj_amount.quantize(Decimal(1))) - else: - raise RuntimeError('unknown order type: ' + str(ordertype)) - return cj_amount + tx_list = [] + for m, txcount in enumerate(txcounts): + #assume that the sizes of outputs will follow a power law + amount_ratios = np.random.power(options.amountpower, txcount) + amount_ratios /= sum(amount_ratios) + #transaction times are uncorrelated and therefore follow poisson + blockheight_waits = np.random.poisson(options.timelambda, txcount) + #number of makers to use follows a normal distribution + makercounts = np.random.normal(options.makercountrange[0], options.makercountrange[1], txcount) + makercounts = int_lower_bounded(makercounts, 2) + for amount_ratio, blockheight_wait, makercount in zip(amount_ratios, blockheight_waits, makercounts): + tx = {'amount_ratio': amount_ratio, 'blockheight_wait': blockheight_wait, + 'srcmixdepth': m + options.mixdepthsrc, 'makercount': makercount} + tx_list.append(tx) + pprint(tx_list) + block_count = sum([tx['blockheight_wait'] for tx in tx_list]) + print 'requires ' + str(block_count) + ' blocks' + print('estimated time taken ' + str(block_count*10) + + ' minutes or ' + str(block_count/6.0) + ' hours') + maker_count = sum([tx['makercount'] for tx in tx_list]) + relorder_fee = 0.001 + print('uses ' + str(maker_count) + ' makers, at ' + str(relorder_fee*100) + '% per maker, estimated total cost ' + + str(round((1 - (1 - relorder_fee)**maker_count) * 100, 3)) + '%') - sqlorders = db.execute('SELECT * FROM orderbook;').fetchall() - orders = [(o['counterparty'], o['oid'], calc_zero_change_cj_amount(o['ordertype'], o['cjfee']), - o['minsize'], o['maxsize']) for o in sqlorders] - #filter cj_amounts that are not in range - orders = [o[:3] for o in orders if o[2] >= o[3] and o[2] <= o[4]] - orders = sorted(orders, key=lambda k: k[2]) - print 'sweep orders = ' + str(orders) - return orders[-1] #choose one with the highest cj_amount, most left over after paying everything else +def main(): + parser = OptionParser(usage='usage: %prog [options] [seed] [tumble-file / destaddr...]', + description='Sends bitcoins to many different addresses using coinjoin in' + ' an attempt to break the link between them. Sending to multiple ' + ' addresses is highly recommended for privacy. This tumbler can' + ' be configured to ask for more address mid-run, giving the user' + ' a chance to click `Generate New Deposit Address` on whatever service' + ' they are using.') + parser.add_option('-m', '--mixdepthsource', type='int', dest='mixdepthsrc', + help='mixing depth to spend from, default=0', default=0) + parser.add_option('-f', '--txfee', type='int', dest='txfee', + default=10000, help='miner fee contribution, in satoshis, default=10000') + parser.add_option('-a', '--addrask', type='int', dest='addrask', + default=2, help='How many more addresses to ask for in the terminal, default=2') + parser.add_option('-N', '--makercountrange', type='float', nargs=2, action='store', + dest='makercountrange', + help='Input the range of makers to use. e.g. 3-5 will random use between ' + '3 and 5 makers inclusive, default=3 4', default=(3, 1)) + parser.add_option('-M', '--mixdepthcount', type='int', dest='mixdepthcount', + help='how many mixing depths to mix through', default=3) + parser.add_option('-c', '--txcountparams', type='float', nargs=2, dest='txcountparams', default=(5, 1), + help='The number of transactions to take coins from one mixing depth to the next, it is' + ' randomly chosen following a normal distribution. This option controlled the parameters' + ' of that normal curve. (mean, standard deviation). default=(5, 1)') + parser.add_option('--amountpower', type='float', dest='amountpower', default=3.0, + help='the output amounts follow a power law distribution, this is the power, default=3.0') + parser.add_option('-l', '--timelambda', type='float', dest='timelambda', default=2, + help='the number of blocks to wait between transactions is randomly chosen ' + ' following a poisson distribution. This parameter is the lambda of that ' + ' distribution. default=2 blocks') -#thread which does the buy-side algorithm -# chooses which coinjoins to initiate and when -class AlgoThread(threading.Thread): - def __init__(self, taker, initial_unspents): - threading.Thread.__init__(self) - self.daemon = True - self.taker = taker - self.initial_unspents = initial_unspents - self.finished_cj = False + parser.add_option('-w', '--wait-time', action='store', type='float', dest='waittime', + help='wait time in seconds to allow orders to arrive, default=5', default=5) + (options, args) = parser.parse_args() - def finished_cj_callback(self): - self.finished_cj = True - print 'finished cj' + if len(args) < 2: + parser.error('Needs a seed and destination addresses') + sys.exit(0) + seed = args[0] + destaddrs = args[1:] - def run(self): - global cjtx - time.sleep(ORDER_ARRIVAL_WAIT_TIME) - #while True: - if 1: - #wait for orders to arrive - #TODO just make this do one tx and then stop - if len(self.initial_unspents) == 0: - print 'finished mixing, closing...' - self.taker.shutdown() - #break + if len(destaddrs) + options.addrask <= 1: + print '='*50 + print 'WARNING: You are only using one destination address' + print 'this is almost useless for privacy' + print '='*50 - #utxo, addrvalue = self.initial_unspents.popitem() - utxo, addrvalue = [(k, v) for k, v in self.initial_unspents.iteritems() if v['value'] == 200000000][0] - counterparty, oid, cj_amount = choose_sweep_order(addrvalue['value'], my_tx_fee) - self.finished_cj = False - cjtx = CoinJoinTX(self.taker, cj_amount, [counterparty], [int(oid)], - [utxo], self.taker.wallet.get_receive_addr(mixing_depth=1), None, - my_tx_fee, self.finished_cj_callback) - #algorithm for making - ''' - single_cj_amount = 112000000 - unspent = [] - for utxo, addrvalue in self.initial_unspents.iteritems(): - unspent.append({'value': addrvalue['value'], 'utxo': utxo}) - inputs = btc.select(unspent, single_cj_amount) - my_utxos = [i['utxo'] for i in inputs] - counterparty, oid = choose_order(single_cj_amount) - cjtx = CoinJoinTX(self.irc, int(single_cj_amount), [counterparty], [int(oid)], - my_utxos, wallet.get_receive_addr(mixing_depth=1), wallet.get_change_addr(mixing_depth=0)) - ''' - while not self.finished_cj: - time.sleep(5) - print 'woken algo thread' + print 'seed=' + seed + print 'destaddrs=' + str(destaddrs) + print str(options) + generate_tumbler_tx(destaddrs, options) -def main(): - print 'downloading wallet history' - wallet = Wallet(seed) - wallet.download_wallet_history() - wallet.find_unspent_addresses() + #a couple of overarching modes + #im-running-from-the-nsa, takes about 80 hours, costs a lot + #python tumbler.py -N 10 5 -c 10 5 -l 5 -M 10 seed 1 + # + #quick and cheap, takes about 90 minutes + #python tumbler.py -N 2 1 -c 3 0.001 -l 2 -M 2 seed 1 + # + #default, good enough for most, takes about 5 hours + #python tumbler.py seed 1 if __name__ == "__main__": main() From 75b2375d58ac14bb22516f00f71cded808152330 Mon Sep 17 00:00:00 2001 From: chris belcher Date: Mon, 16 Feb 2015 20:52:52 +0000 Subject: [PATCH 131/409] fixed bug from issue #46 --- irc.py | 9 +++------ sendpayment.py | 2 +- taker.py | 4 ++-- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/irc.py b/irc.py index 5f25f654..d8d77433 100644 --- a/irc.py +++ b/irc.py @@ -83,6 +83,7 @@ def send_tx(self, nick_list, txhex): txb64 = base64.b64encode(txhex.decode('hex')) for nick in nick_list: self.__privmsg(nick, 'tx', txb64) + time.sleep(1) #Maker callbacks def announce_orders(self, orderlist, nick=None): @@ -105,13 +106,13 @@ def send_pubkey(self, nick, pubkey): self.__privmsg(nick, 'pubkey', pubkey) def send_ioauth(self, nick, utxo_list, cj_pubkey, change_addr, sig): - authmsg = (','.join(utxo_list) + ' ' + + authmsg = (str(','.join(utxo_list)) + ' ' + cj_pubkey + ' ' + change_addr + ' ' + sig) self.__privmsg(nick, 'ioauth', authmsg) def send_sigs(self, nick, sig_list): #TODO make it send the sigs on one line if there's space - for s in sigs_list: + for s in sig_list: self.__privmsg(nick, 'sig', s) def __pubmsg(self, message): @@ -125,7 +126,6 @@ def __privmsg(self, nick, cmd, message): #encrypt before chunking if box: message = enc_wrapper.encrypt_encode(message, box) - print 'emsg=' + message if len(message) > 350: message_chunks = chunks(message, 350) @@ -136,7 +136,6 @@ def __privmsg(self, nick, cmd, message): trailer = ' ~' if m==message_chunks[-1] else ' ;' header = "PRIVMSG " + nick + " :" if m==message_chunks[0]: header += '!'+cmd + ' ' - print 'sendraw ' + header + m + trailer self.send_raw(header + m + trailer) def send_raw(self, line): @@ -294,8 +293,6 @@ def __handle_privmsg(self, source, target, message): else: self.built_privmsg[nick][1] += message[:-2] box = self.__encrypting(self.built_privmsg[nick][0], nick) - print 'cmd=' + self.built_privmsg[nick][0] + ' nick=' + nick + ' box=' + str(box) - print 'msg=' + message if message[-1]==';': self.waiting[nick]=True elif message[-1]=='~': diff --git a/sendpayment.py b/sendpayment.py index 28c4f2c7..d54c0e1d 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -36,7 +36,7 @@ def run(self): for utxo in utxo_list: total_value += self.taker.wallet.unspent[utxo]['value'] orders, cjamount = choose_sweep_order(self.taker.db, total_value, self.taker.txfee, self.taker.makercount) - self.taker.cjtx = takermodule.CoinJoinTX(self.taker, cjamount, orders, utxo_list, + self.taker.cjtx = takermodule.CoinJoinTX(self.taker.msgchan, self.taker, cjamount, orders, utxo_list, self.taker.destaddr, None, self.taker.txfee, self.finishcallback) else: orders, total_cj_fee = choose_order(self.taker.db, self.taker.amount, self.taker.makercount) diff --git a/taker.py b/taker.py index 41080bf1..c3fba33c 100644 --- a/taker.py +++ b/taker.py @@ -143,8 +143,8 @@ def add_signature(self, sigb64): debug('the entire tx is signed, ready to pushtx()') print btc.serialize(self.latest_tx) - #ret = btc.blockr_pushtx(btc.serialize(self.latest_tx), get_network()) - #debug('pushed tx ' + str(ret)) + ret = btc.blockr_pushtx(btc.serialize(self.latest_tx), get_network()) + debug('pushed tx ' + str(ret)) if self.finishcallback != None: self.finishcallback() From 6840812716e75efe023a1401beeca7b2ac5e3b04 Mon Sep 17 00:00:00 2001 From: chris belcher Date: Mon, 16 Feb 2015 21:10:58 +0000 Subject: [PATCH 132/409] fixed bug --- irc.py | 2 ++ message_channel.py | 22 ++++++++++++++++++++++ taker.py | 1 + 3 files changed, 25 insertions(+) diff --git a/irc.py b/irc.py index d8d77433..5232e700 100644 --- a/irc.py +++ b/irc.py @@ -245,6 +245,7 @@ def __on_pubmsg(self, nick, message): if self.on_orderbook_requested: self.on_orderbook_requested(nick) else: + #TODO this is for testing/debugging, should be removed, see taker.py if hasattr(self, 'debug_on_pubmsg_cmd'): self.debug_on_pubmsg_cmd(nick, chunks) @@ -372,6 +373,7 @@ def __handle_line(self, line): ''' def __init__(self, nick, server=HOST, port=PORT, channel=CHANNEL, username='username', realname='realname'): + MessageChannel.__init__(self) self.cjpeer = None #subclasses have to set this to self self.nick = nick self.serverport = (server, port) diff --git a/message_channel.py b/message_channel.py index db3dfdf2..56023020 100644 --- a/message_channel.py +++ b/message_channel.py @@ -7,6 +7,28 @@ class MessageChannel(object): Abstract class which implements a way for bots to communicate ''' + def __init__(self): + #all + self.on_welcome = None + self.on_set_topic = None + self.on_connect = None + self.on_disconnect = None + self.on_nick_leave = None + self.on_nick_change = None + #orderbook watch functions + self.on_order_seen = None + self.on_order_cancel = None + #taker functions + self.on_error = None + self.on_pubkey = None + self.on_ioauth = None + self.on_sig = None + #maker functions + self.on_orderbook_requested = None + self.on_order_fill = None + self.on_seen_auth = None + self.on_seen_tx = None + def run(self): pass def shutdown(self): pass def send_error(self, nick, errormsg): pass diff --git a/taker.py b/taker.py index c3fba33c..12385b1e 100644 --- a/taker.py +++ b/taker.py @@ -234,6 +234,7 @@ class TestTaker(Taker): def __init__(self, msgchan, wallet): Taker.__init__(self, msgchan) self.wallet = wallet + #TODO this is for testing/debugging, should be removed self.msgchan.debug_on_pubmsg_cmd = self.debug_on_pubmsg_cmd def finish_callback(self): From 0fbd737104f23bb9e05b0391b13347bba4cb0fc2 Mon Sep 17 00:00:00 2001 From: chris belcher Date: Mon, 16 Feb 2015 21:16:48 +0000 Subject: [PATCH 133/409] bugfix --- yield-generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yield-generator.py b/yield-generator.py index 1f95e2e5..0711cc1f 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -35,7 +35,7 @@ def __init__(self, msgchan, wallet): def on_connect(self): if len(nickserv_password) > 0: - self.send_raw('PRIVMSG NickServ :identify ' + nickserv_password) + self.msgchan.send_raw('PRIVMSG NickServ :identify ' + nickserv_password) def create_my_orders(self): mix_balance = self.wallet.get_balance_by_mixdepth() From cbf30577c0c2d066787764e7fdc80fd2329f7f1c Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Tue, 17 Feb 2015 14:13:57 +0200 Subject: [PATCH 134/409] sendpayment/yigen combo now working --- blockchaininterface.py | 55 +++++++++++++++++++----------------------- common.py | 24 ++++++++++++------ taker.py | 6 +++-- 3 files changed, 46 insertions(+), 39 deletions(-) diff --git a/blockchaininterface.py b/blockchaininterface.py index fb93b229..43f6cfb1 100644 --- a/blockchaininterface.py +++ b/blockchaininterface.py @@ -17,7 +17,7 @@ def get_utxos_from_addr(self, address): '''Given an address, return a list of utxos in format txid:vout''' pass - def get_balance_at_addr(self,address): + def get_balance_at_addr(self, address): '''Given an address, return a balance in satoshis''' pass @@ -30,7 +30,7 @@ def get_net_info(self): pass def get_addr_from_utxo(self, txhash, index): '''Given utxo in form txhash, index, return the address - owning the utxo and the amount in satoshis in form (addr,amt)''' + owning the utxo and the amount in satoshis in form (addr, amt)''' pass @@ -51,7 +51,7 @@ def __init__(self, client_location): except Exception as e: print e - def rpc(self,args,accept_failure=[]): + def rpc(self, args, accept_failure=[]): try: res = subprocess.check_output(self.command_params+args) except subprocess.CalledProcessError, e: @@ -61,23 +61,17 @@ def rpc(self,args,accept_failure=[]): return res def send_tx(self, tx_hex): - res = self.rpc(['sendrawtransaction',tx_hex]) + res = self.rpc(['sendrawtransaction', tx_hex]) + self.tick_forward_chain(1) #TODO parse return string - print res - return True + return {'data':res} - def get_utxos_from_addr(self,addresses): - for address in addresses: - res = json.loads(self.rpc(['listunspent','1','9999999','[\"'+address+'\"]'])) - #utxos = [] - #for r in res: - # utxos.append(r['txid']+':'+str(r['vout'])) - #return utxos + def get_utxos_from_addr(self, addresses): r = [] for address in addresses: - utxos = [x for x in res if x['address']==address] + res = json.loads(self.rpc(['listunspent','1','9999999','[\"'+address+'\"]'])) unspents=[] - for u in utxos: + for u in res: unspents.append({'tx':u['txid'],'n':u['vout'],'amount':str(u['amount']),'address':address,'confirmations':u['confirmations']}) r.append({'address':address,'unspent':unspents}) return {'data':r} @@ -105,19 +99,20 @@ def get_txs_from_addr(self, addresses): #print result return {'data':result} - def get_tx_info(self, txhash): - res = json.loads(self.rpc(['gettransaction',txhash,'true'])) + def get_tx_info(self, txhash, raw=False): + res = json.loads(self.rpc(['gettransaction', txhash,'true'])) + if raw: + return {'data':{'tx':{'hex':res['hex']}}} tx = btc.deserialize(res['hex']) #build vout list vouts = [] n=0 for o in tx['outs']: - vouts.append({'n':n,'amount':o['value'],'address':btc.script_to_address(o['script'])}) + vouts.append({'n':n,'amount':o['value'],'address':btc.script_to_address(o['script'],0x6f)}) n+=1 return {'data':{'vouts':vouts}} - - + def get_balance_at_addr(self, addresses): #NB This will NOT return coinbase coins (but wont matter in our use case). #In order to have the Bitcoin RPC read balances at addresses @@ -128,16 +123,16 @@ def get_balance_at_addr(self, addresses): #allow importaddress to fail in case the address is already in the wallet res = [] for address in addresses: - self.rpc(['importaddress',address,'watchonly'],[4]) + self.rpc(['importaddress', address,'watchonly'],[4]) res.append({'address':address,'balance':int(Decimal(1e8) * Decimal(self.rpc(['getreceivedbyaddress',address])))}) return {'data':res} def tick_forward_chain(self, n): '''Special method for regtest only; instruct to mine n blocks.''' - self.rpc(['setgenerate','true',str(n)]) + self.rpc(['setgenerate','true', str(n)]) - def grab_coins(self,receiving_addr,amt=50): + def grab_coins(self, receiving_addr, amt=50): ''' NOTE! amt is passed in Coins, not Satoshis! Special method for regtest only: @@ -151,10 +146,10 @@ def grab_coins(self,receiving_addr,amt=50): #mine enough to get to the reqd amt reqd = int(amt - self.current_balance) reqd_blocks = str(int(reqd/50) +1) - if self.rpc(['setgenerate','true',reqd_blocks]): + if self.rpc(['setgenerate','true', reqd_blocks]): raise Exception("Something went wrong") #now we do a custom create transaction and push to the receiver - txid = self.rpc(['sendtoaddress',receiving_addr,str(amt)]) + txid = self.rpc(['sendtoaddress', receiving_addr, str(amt)]) if not txid: raise Exception("Failed to broadcast transaction") #confirm @@ -163,10 +158,10 @@ def grab_coins(self,receiving_addr,amt=50): def get_addr_from_utxo(self, txhash, index): #get the transaction details - res = json.loads(self.rpc(['gettxout',txhash, str(index)])) + res = json.loads(self.rpc(['gettxout', txhash, str(index)])) amt = int(Decimal(1e8)*Decimal(res['value'])) address = res('addresses')[0] - return (address,amt) + return (address, amt) def main(): bitcointoolsdir = '/home/adam/DevRepos/bitcoin/src/' @@ -175,10 +170,10 @@ def main(): #myBCI.send_tx('stuff') print myBCI.get_utxos_from_addr(["n4EjHhGVS4Rod8ociyviR3FH442XYMWweD"]) print myBCI.get_balance_at_addr(["n4EjHhGVS4Rod8ociyviR3FH442XYMWweD"]) - txid = myBCI.grab_coins('mtc6UaPPp2x1Fabugi8JG4BNouFo9rADNb',23) + txid = myBCI.grab_coins('mygp9fsgEJ5U7jkPpDjX9nxRj8b5nC3Hnd',23) print txid - print myBCI.get_balance_at_addr(['mtc6UaPPp2x1Fabugi8JG4BNouFo9rADNb']) - print myBCI.get_utxos_from_addr(['mtc6UaPPp2x1Fabugi8JG4BNouFo9rADNb']) + print myBCI.get_balance_at_addr(['mygp9fsgEJ5U7jkPpDjX9nxRj8b5nC3Hnd']) + print myBCI.get_utxos_from_addr(['mygp9fsgEJ5U7jkPpDjX9nxRj8b5nC3Hnd']) if __name__ == '__main__': main() diff --git a/common.py b/common.py index 6cc1bfda..74a34419 100644 --- a/common.py +++ b/common.py @@ -7,7 +7,7 @@ import blockchaininterface HOST = 'irc.freenode.net' -CHANNEL = '#joinmarket-pit-test' +CHANNEL = '#joinmarket-pit-test2' PORT = 6667 #for the mainnet its #joinmarket-pit @@ -82,7 +82,7 @@ def get_blockchain_data(body, csv_params=[], raise Exception("Unrecognised blockchain source") bodies = {'addrtx':'address/txs/','txinfo':'tx/info/','addrunspent':'address/unspent/', - 'addrbalance':'address/balance/'} + 'addrbalance':'address/balance/','txraw':'tx/raw/','txpush':'tx/push/'} url = stem + bodies[body] + ','.join(csv_params) if query_params: url += '?'+','.join(query_params) @@ -115,17 +115,24 @@ def get_regtest_data(req): elif req[0]=='tx' and req[1]=='info': txhash = req[2] #TODO currently only allowing one tx return myBCI.get_tx_info(txhash) - elif req[0]=='addr' and req[1] == 'balance': + elif req[0]=='address' and req[1] == 'balance': addrs = req[2].split(',') - if 'unconfirmed' in addrs[-1]: - addrs = addrs[-1] + if '?' in addrs[-1]: + #TODO simulate dealing with unconfirmed + addrs[-1] = addrs[-1].split('?')[0] return myBCI.get_balance_at_addr(addrs) elif req[0]=='address' and req[1] == 'unspent': if '?' in req[2]: req[2] = req[2].split('?')[0] addrs = req[2].split(',') return myBCI.get_utxos_from_addr(addrs) + elif req[0]=='tx' and req[1] == 'raw': + txhex = req[2] + return myBCI.get_tx_info(txhex, raw=True) + elif req[0]=='tx' and req[1] == 'push': + txraw = req[2] + return myBCI.send_tx(txraw) else: - raise Exception ("Unrecognized call to regtest blockchain interface") + raise Exception ("Unrecognized call to regtest blockchain interface: " + '/'.join(req)) class Wallet(object): @@ -344,6 +351,8 @@ def run(self): return data = get_blockchain_data('addrbalance', csv_params=[self.address], query_params=['confirmations=0']) + if type(data) == list: + data = data[0] #needed because blockr's json structure is inconsistent if data['balance'] > 0: break self.unconfirmfun(data['balance']*1e8) @@ -381,7 +390,8 @@ def calc_cj_fee(ordertype, cjfee, cj_amount): def calc_total_input_value(utxos): input_sum = 0 for utxo in utxos: - tx = btc.blockr_fetchtx(utxo[:64], get_network()) + tx = get_blockchain_data('txraw',csv_params=[utxo[:64]])['tx']['hex'] + #tx = btc.blockr_fetchtx(utxo[:64], get_network()) input_sum += int(btc.deserialize(tx)['outs'][int(utxo[65:])]['value']) return input_sum diff --git a/taker.py b/taker.py index 2f2d4c48..7c7225c5 100644 --- a/taker.py +++ b/taker.py @@ -125,7 +125,8 @@ def add_signature(self, sigb64): for index, ins in enumerate(self.latest_tx['ins']): if ins['script'] != '': continue - ftx = btc.blockr_fetchtx(ins['outpoint']['hash'], get_network()) + ftx = get_blockchain_data('txraw',csv_params=[ins['outpoint']['hash']])['tx']['hex'] + #ftx = btc.blockr_fetchtx(ins['outpoint']['hash'], get_network()) src_val = btc.deserialize(ftx)['outs'][ ins['outpoint']['index'] ] sig_good = btc.verify_tx_input(tx, index, src_val['script'], *btc.deserialize_script(sig)) if sig_good: @@ -148,7 +149,8 @@ def add_signature(self, sigb64): debug('the entire tx is signed, ready to pushtx()') print btc.serialize(self.latest_tx) - ret = btc.blockr_pushtx(btc.serialize(self.latest_tx), get_network()) + ret = get_blockchain_data('txpush',csv_params=[btc.serialize(self.latest_tx)]) + #ret = btc.blockr_pushtx(btc.serialize(self.latest_tx), get_network()) debug('pushed tx ' + str(ret)) if self.finishcallback != None: self.finishcallback() From 20e0958806d99d8b040a728d24213f4b741ec476 Mon Sep 17 00:00:00 2001 From: chris belcher Date: Tue, 17 Feb 2015 18:33:51 +0000 Subject: [PATCH 135/409] added comment --- irc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/irc.py b/irc.py index 5232e700..b28e7332 100644 --- a/irc.py +++ b/irc.py @@ -83,7 +83,7 @@ def send_tx(self, nick_list, txhex): txb64 = base64.b64encode(txhex.decode('hex')) for nick in nick_list: self.__privmsg(nick, 'tx', txb64) - time.sleep(1) + time.sleep(1) #HACK! really there should be rate limiting, see issue#31 #Maker callbacks def announce_orders(self, orderlist, nick=None): From 0e2d7d6a6508b412dd6d249caca51ce6941830dc Mon Sep 17 00:00:00 2001 From: chris belcher Date: Wed, 18 Feb 2015 21:53:36 +0000 Subject: [PATCH 136/409] some fixes but not yet finished --- irc.py | 2 +- patientsendpayment.py | 38 +++++++++++++++++++------------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/irc.py b/irc.py index b28e7332..233fdf4e 100644 --- a/irc.py +++ b/irc.py @@ -100,7 +100,7 @@ def announce_orders(self, orderlist, nick=None): def cancel_orders(self, oid_list): clines = [COMMAND_PREFIX + 'cancel ' + str(oid) for oid in oid_list] - self.pubmsg(''.join(clines)) + self.__pubmsg(''.join(clines)) def send_pubkey(self, nick, pubkey): self.__privmsg(nick, 'pubkey', pubkey) diff --git a/patientsendpayment.py b/patientsendpayment.py index 5e8ff934..3928d189 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -2,8 +2,8 @@ from common import * import taker import maker +from irc import IRCMessageChannel import bitcoin as btc -import sendpayment from optparse import OptionParser from datetime import timedelta @@ -17,7 +17,7 @@ def __init__(self, tmaker): self.finished = False def finishcallback(self): - self.tmaker.shutdown() + self.tmaker.msgchan.shutdown() def run(self): time.sleep(self.tmaker.waittime) @@ -31,14 +31,14 @@ def run(self): total_amount = self.tmaker.amount + total_cj_fee + self.tmaker.txfee print 'total amount spent = ' + str(total_amount) - utxos = self.taker.wallet.select_utxos(self.tmaker.mixdepth, total_amount) - self.tmaker.cjtx = taker.CoinJoinTX(self.tmaker, self.tmaker.amount, + utxos = self.tmaker.wallet.select_utxos(self.tmaker.mixdepth, total_amount) + self.tmaker.cjtx = taker.CoinJoinTX(self.tmaker.msgchan, self.tmaker, self.tmaker.amount, orders, utxos, self.tmaker.destaddr, self.tmaker.wallet.get_change_addr(self.tmaker.mixdepth), self.tmaker.txfee, self.finishcallback) class PatientSendPayment(maker.Maker, taker.Taker): - def __init__(self, wallet, destaddr, amount, makercount, txfee, cjfee, + def __init__(self, msgchan, wallet, destaddr, amount, makercount, txfee, cjfee, waittime, mixdepth): self.destaddr = destaddr self.amount = amount @@ -47,12 +47,14 @@ def __init__(self, wallet, destaddr, amount, makercount, txfee, cjfee, self.cjfee = cjfee self.waittime = waittime self.mixdepth = mixdepth - maker.Maker.__init__(self, wallet) - taker.Taker.__init__(self) + maker.Maker.__init__(self, msgchan, wallet) + taker.Taker.__init__(self, msgchan) - def on_privmsg(self, nick, message): - maker.Maker.on_privmsg(self, nick, message) - taker.Taker.on_privmsg(self, nick, message) + def get_crypto_box_from_nick(self, nick): + if len(self.active_orders) == 0: + return taker.Taker.get_crypto_box_from_nick(self, nick) + else: + return maker.Maker.get_crypto_box_from_nick(self, nick) def on_welcome(self): maker.Maker.on_welcome(self) @@ -60,10 +62,6 @@ def on_welcome(self): self.takerthread = TakerThread(self) self.takerthread.start() - def on_pubmsg(self, nick, message): - maker.Maker.on_pubmsg(self, nick, message) - taker.Taker.on_pubmsg(self, nick, message) - def create_my_orders(self): #choose an absolute fee order to discourage people from # mixing smaller amounts @@ -150,16 +148,18 @@ def main(): from socket import gethostname nickname = 'ppayer-' + btc.sha256(gethostname())[:6] - print 'starting irc' - bot = PatientSendPayment(wallet, destaddr, amount, options.makercount, + irc = IRCMessageChannel(nickname) + bot = PatientSendPayment(irc, wallet, destaddr, amount, options.makercount, options.txfee, options.cjfee, waittime, options.mixdepth) try: - bot.run(HOST, PORT, nickname, CHANNEL) - finally: + irc.run() + except: debug('CRASHING, DUMPING EVERYTHING') debug('wallet seed = ' + seed) debug_dump_object(wallet, ['addr_cache']) - debug_dump_object(bot) + debug_dump_object(taker) + import traceback + traceback.print_exc() if __name__ == "__main__": main() From c03f0107c574f65fc5a2c5f9806b00f33231edbc Mon Sep 17 00:00:00 2001 From: chris belcher Date: Wed, 18 Feb 2015 21:54:03 +0000 Subject: [PATCH 137/409] made wallet-tool not print out used empty addresses by default --- wallet-tool.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/wallet-tool.py b/wallet-tool.py index f74e428f..ddc199d1 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -48,6 +48,28 @@ wallet.find_unspent_addresses() if method == 'display': + total_balance = 0 + for m in range(wallet.max_mix_depth): + print 'mixing depth %d m/0/%d/' % (m, m) + balance_depth = 0 + for forchange in [0, 1]: + print(' ' + ('receive' if forchange==0 else 'change') + + ' addresses m/0/%d/%d/' % (m, forchange)) + for k in range(wallet.index[m][forchange] + options.gaplimit): + addr = wallet.get_addr(m, forchange, k) + balance = 0.0 + for addrvalue in wallet.unspent.values(): + if addr == addrvalue['address']: + balance += addrvalue['value'] + balance_depth += balance + used = ('used' if k < wallet.index[m][forchange] else ' new') + if balance > 0 or used == ' new': + print ' m/0/%d/%d/%02d %s %s %.8fbtc' % (m, forchange, k, addr, used, balance/1e8) + print 'for mixdepth=%d balance=%.8fbtc' % (m, balance_depth/1e8) + total_balance += balance_depth + print 'total balance = %.8fbtc' % (total_balance/1e8) + +if method == 'displayall': total_balance = 0 for m in range(wallet.max_mix_depth): print 'mixing depth %d m/0/%d/' % (m, m) From af4a19a1a06206b061d10571d431b5686637035f Mon Sep 17 00:00:00 2001 From: chris belcher Date: Thu, 19 Feb 2015 18:08:41 +0000 Subject: [PATCH 138/409] further bugfixes to patientsendpayment, still not done --- patientsendpayment.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/patientsendpayment.py b/patientsendpayment.py index 3928d189..640829fa 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -20,6 +20,8 @@ def finishcallback(self): self.tmaker.msgchan.shutdown() def run(self): + #TODO this thread doesnt wake up for what could be hours + # need a loop that periodically checks self.finished time.sleep(self.tmaker.waittime) if self.finished: return @@ -51,6 +53,7 @@ def __init__(self, msgchan, wallet, destaddr, amount, makercount, txfee, cjfee, taker.Taker.__init__(self, msgchan) def get_crypto_box_from_nick(self, nick): + debug('getting cryptobox for patient, len(active_orders)=' + str(len(self.active_orders))) if len(self.active_orders) == 0: return taker.Taker.get_crypto_box_from_nick(self, nick) else: @@ -82,7 +85,8 @@ def on_tx_unconfirmed(self, cjorder, balance, removed_utxos): if self.amount == 0: self.takerthread.finished = True print 'finished sending, exiting..' - self.shutdown() + self.msgchan.shutdown() + return ([], []) utxo_list = self.wallet.get_utxo_list_by_mixdepth()[self.mixdepth] available_balance = 0 for utxo in utxo_list: From 7337807c8dcd16475ae786c44b350b0ce1a2ce07 Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 20 Feb 2015 00:11:23 +0000 Subject: [PATCH 139/409] fixed patientsendpayments bug, works fine now --- patientsendpayment.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/patientsendpayment.py b/patientsendpayment.py index 640829fa..a1ad5c15 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -22,6 +22,9 @@ def finishcallback(self): def run(self): #TODO this thread doesnt wake up for what could be hours # need a loop that periodically checks self.finished + #TODO another issue is, what if the bot has run out of utxos and + # needs to wait for some tx to confirm before it can trade + # presumably it needs to wait here until the tx confirms time.sleep(self.tmaker.waittime) if self.finished: return @@ -53,8 +56,7 @@ def __init__(self, msgchan, wallet, destaddr, amount, makercount, txfee, cjfee, taker.Taker.__init__(self, msgchan) def get_crypto_box_from_nick(self, nick): - debug('getting cryptobox for patient, len(active_orders)=' + str(len(self.active_orders))) - if len(self.active_orders) == 0: + if self.cjtx: return taker.Taker.get_crypto_box_from_nick(self, nick) else: return maker.Maker.get_crypto_box_from_nick(self, nick) @@ -90,7 +92,7 @@ def on_tx_unconfirmed(self, cjorder, balance, removed_utxos): utxo_list = self.wallet.get_utxo_list_by_mixdepth()[self.mixdepth] available_balance = 0 for utxo in utxo_list: - available_balance = self.wallet.unspent[utxo]['value'] + available_balance += self.wallet.unspent[utxo]['value'] if available_balance > self.amount: order = {'oid': 0, 'ordertype': 'absorder', 'minsize': 0, 'maxsize': self.amount, 'txfee': self.txfee, 'cjfee': self.cjfee} From 005c5a58ecbda6fd582f800fa83149bc6da762f3 Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 20 Feb 2015 00:37:19 +0000 Subject: [PATCH 140/409] modified parameters of CoinJoinTX, now takes wallet and db seperately --- patientsendpayment.py | 4 ++-- sendpayment.py | 6 +++--- taker.py | 27 ++++++++++++++------------- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/patientsendpayment.py b/patientsendpayment.py index a1ad5c15..88a3132e 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -37,8 +37,8 @@ def run(self): print 'total amount spent = ' + str(total_amount) utxos = self.tmaker.wallet.select_utxos(self.tmaker.mixdepth, total_amount) - self.tmaker.cjtx = taker.CoinJoinTX(self.tmaker.msgchan, self.tmaker, self.tmaker.amount, - orders, utxos, self.tmaker.destaddr, + self.tmaker.cjtx = taker.CoinJoinTX(self.tmaker.msgchan, self.tmaker.wallet, + self.tmaker.db, self.tmaker.amount, orders, utxos, self.tmaker.destaddr, self.tmaker.wallet.get_change_addr(self.tmaker.mixdepth), self.tmaker.txfee, self.finishcallback) diff --git a/sendpayment.py b/sendpayment.py index d54c0e1d..eb3a907c 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -36,7 +36,7 @@ def run(self): for utxo in utxo_list: total_value += self.taker.wallet.unspent[utxo]['value'] orders, cjamount = choose_sweep_order(self.taker.db, total_value, self.taker.txfee, self.taker.makercount) - self.taker.cjtx = takermodule.CoinJoinTX(self.taker.msgchan, self.taker, cjamount, orders, utxo_list, + self.taker.cjtx = takermodule.CoinJoinTX(self.taker.msgchan, self.taker.wallet, self.taker.db, cjamount, orders, utxo_list, self.taker.destaddr, None, self.taker.txfee, self.finishcallback) else: orders, total_cj_fee = choose_order(self.taker.db, self.taker.amount, self.taker.makercount) @@ -45,8 +45,8 @@ def run(self): print 'total amount spent = ' + str(total_amount) utxos = self.taker.wallet.select_utxos(self.taker.mixdepth, total_amount) - self.taker.cjtx = takermodule.CoinJoinTX(self.taker.msgchan, self.taker, self.taker.amount, - orders, utxos, self.taker.destaddr, + self.taker.cjtx = takermodule.CoinJoinTX(self.taker.msgchan, self.taker.wallet, + self.taker.db, self.taker.amount, orders, utxos, self.taker.destaddr, self.taker.wallet.get_change_addr(self.taker.mixdepth), self.taker.txfee, self.finishcallback) diff --git a/taker.py b/taker.py index 12385b1e..efa2e05d 100644 --- a/taker.py +++ b/taker.py @@ -8,7 +8,7 @@ class CoinJoinTX(object): #soon the taker argument will be removed and just be replaced by wallet or some other interface - def __init__(self, msgchan, taker, cj_amount, orders, my_utxos, my_cj_addr, + def __init__(self, msgchan, wallet, db, cj_amount, orders, my_utxos, my_cj_addr, my_change_addr, my_txfee, finishcallback=None): ''' if my_change is None then there wont be a change address @@ -17,7 +17,8 @@ def __init__(self, msgchan, taker, cj_amount, orders, my_utxos, my_cj_addr, ''' debug('starting cj to ' + my_cj_addr + ' with change at ' + str(my_change_addr)) self.msgchan = msgchan - self.taker = taker + self.wallet = wallet + self.db = db self.cj_amount = cj_amount self.active_orders = dict(orders) self.nonrespondants = list(orders.keys()) @@ -33,8 +34,8 @@ def __init__(self, msgchan, taker, cj_amount, orders, my_utxos, my_cj_addr, self.kp = enc_wrapper.init_keypair() self.crypto_boxes = {} #find the btc pubkey of the first utxo being used - self.signing_btc_add = taker.wallet.unspent[self.my_utxos[0]]['address'] - self.signing_btc_pub = btc.privtopub(taker.wallet.get_key_from_addr(self.signing_btc_add)) + self.signing_btc_add = wallet.unspent[self.my_utxos[0]]['address'] + self.signing_btc_pub = btc.privtopub(wallet.get_key_from_addr(self.signing_btc_add)) self.msgchan.fill_orders(orders, cj_amount, self.kp.hex_pk()) def start_encryption(self, nick, maker_pk): @@ -43,8 +44,8 @@ def start_encryption(self, nick, maker_pk): self.crypto_boxes[nick] = [maker_pk, enc_wrapper.as_init_encryption(\ self.kp, enc_wrapper.init_pubkey(maker_pk))] #send authorisation request - my_btc_priv = self.taker.wallet.get_key_from_addr(\ - self.taker.wallet.unspent[self.my_utxos[0]]['address']) + my_btc_priv = self.wallet.get_key_from_addr(\ + self.wallet.unspent[self.my_utxos[0]]['address']) my_btc_pub = btc.privtopub(my_btc_priv) my_btc_sig = btc.ecdsa_sign(self.kp.hex_pk(), my_btc_priv) self.msgchan.send_auth(nick, my_btc_pub, my_btc_sig) @@ -65,7 +66,7 @@ def recv_txio(self, nick, utxo_list, cj_pub, change_addr): return self.utxos[nick] = utxo_list self.nonrespondants.remove(nick) - order = self.taker.db.execute('SELECT ordertype, txfee, cjfee FROM ' + order = self.db.execute('SELECT ordertype, txfee, cjfee FROM ' 'orderbook WHERE oid=? AND counterparty=?', (self.active_orders[nick], nick)).fetchone() total_input = calc_total_input_value(self.utxos[nick]) @@ -82,7 +83,7 @@ def recv_txio(self, nick, utxo_list, cj_pub, change_addr): my_total_in = 0 for u in self.my_utxos: - usvals = self.taker.wallet.unspent[u] + usvals = self.wallet.unspent[u] my_total_in += int(usvals['value']) my_change_value = my_total_in - self.cj_amount - self.cjfee_total - self.my_txfee @@ -107,10 +108,10 @@ def recv_txio(self, nick, utxo_list, cj_pub, change_addr): utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) if utxo not in self.my_utxos: continue - if utxo not in self.taker.wallet.unspent: + if utxo not in self.wallet.unspent: continue - addr = self.taker.wallet.unspent[utxo]['address'] - tx = btc.sign(tx, index, self.taker.wallet.get_key_from_addr(addr)) + addr = self.wallet.unspent[utxo]['address'] + tx = btc.sign(tx, index, self.wallet.get_key_from_addr(addr)) self.latest_tx = btc.deserialize(tx) def add_signature(self, sigb64): @@ -272,7 +273,7 @@ def debug_on_pubmsg_cmd(self, nick, chunks): amount = chunks[3] my_utxo = chunks[4] print 'making cjtx' - self.cjtx = CoinJoinTX(self.msgchan, self, int(amount), {counterparty: oid}, + self.cjtx = CoinJoinTX(self.msgchan, self.wallet, self.db, int(amount), {counterparty: oid}, [my_utxo], self.wallet.get_receive_addr(mixing_depth=1), self.wallet.get_change_addr(mixing_depth=0), my_tx_fee, self.finish_callback) elif chunks[0] == '%2fill': @@ -284,7 +285,7 @@ def debug_on_pubmsg_cmd(self, nick, chunks): cp2 = chunks[5] oid2 = int(chunks[6]) print 'creating cjtx' - self.cjtx = CoinJoinTX(self.msgchan, self, amount, {cp1: oid1, cp2: oid2}, + self.cjtx = CoinJoinTX(self.msgchan, self.wallet, self.db, amount, {cp1: oid1, cp2: oid2}, [my_utxo], self.wallet.get_receive_addr(mixing_depth=1), self.wallet.get_change_addr(mixing_depth=0), my_tx_fee, self.finish_callback) From 42530c2f37bdb3a98b425535e6abbaaf03f866ce Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 20 Feb 2015 00:46:58 +0000 Subject: [PATCH 141/409] renamed __encrypting() as in issue #43 --- irc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/irc.py b/irc.py index 233fdf4e..c150928e 100644 --- a/irc.py +++ b/irc.py @@ -122,7 +122,7 @@ def __pubmsg(self, message): def __privmsg(self, nick, cmd, message): debug('>>privmsg ' + 'nick=' + nick + ' cmd=' + cmd + ' msg=' + message) #should we encrypt? - box = self.__encrypting(cmd, nick, sending=True) + box = self.__get_encryption_box(cmd, nick) #encrypt before chunking if box: message = enc_wrapper.encrypt_encode(message, box) @@ -249,7 +249,7 @@ def __on_pubmsg(self, nick, message): if hasattr(self, 'debug_on_pubmsg_cmd'): self.debug_on_pubmsg_cmd(nick, chunks) - def __encrypting(self, cmd, nick, sending=False): + def __get_encryption_box(self, cmd, nick): '''Establish whether the message is to be encrypted/decrypted based on the command string. If so, retrieve the appropriate crypto_box object @@ -293,7 +293,7 @@ def __handle_privmsg(self, source, target, message): self.built_privmsg[nick] = [cmd_string, message[:-2]] else: self.built_privmsg[nick][1] += message[:-2] - box = self.__encrypting(self.built_privmsg[nick][0], nick) + box = self.__get_encryption_box(self.built_privmsg[nick][0], nick) if message[-1]==';': self.waiting[nick]=True elif message[-1]=='~': From 3c0d0672e009c80a8e8d8110e41988490372b10f Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Fri, 20 Feb 2015 21:25:39 +0200 Subject: [PATCH 142/409] single sendpayment unit test working --- blockchaininterface.py | 20 +++---- common.py | 21 ++++--- regtesttest.py | 128 +++++++++++++++++++++++++++++++++++++++++ sendpayment.py | 2 + testing/regtesttest.py | 34 ----------- yield-generator.py | 2 +- 6 files changed, 151 insertions(+), 56 deletions(-) create mode 100644 regtesttest.py delete mode 100644 testing/regtesttest.py diff --git a/blockchaininterface.py b/blockchaininterface.py index 43f6cfb1..3a5b104a 100644 --- a/blockchaininterface.py +++ b/blockchaininterface.py @@ -24,7 +24,6 @@ def get_balance_at_addr(self, address): @abc.abstractmethod def send_tx(self, tx_hex): '''Given raw txhex, push to network and return result in form: TODO''' - print 'got herer' pass def get_net_info(self): pass @@ -40,7 +39,7 @@ def get_addr_from_utxo(self, txhash, index): #with > 100 blocks. class RegTestImp(BlockChainInterface): def __init__(self, client_location): - self.command_params = [client_location,'-regtest'] + self.command_params = [client_location+'bitcoin-cli','-regtest'] #quick check that it's up else quit res = self.rpc(['getbalance']) try: @@ -80,10 +79,11 @@ def get_txs_from_addr(self, addresses): #use listtransactions and then filter #e.g.: -regtest listtransactions 'watchonly' 1000 0 true #to get the last 1000 transactions TODO 1000 is arbitrary - - res = json.loads(self.rpc(['listtransactions','watchonly','1000','0','true'])) - #print "Got this res: " - #print res + acct_addrlist = self.rpc(['getaddressesbyaccount', 'watchonly']) + for address in addresses: + if address not in acct_addrlist: + self.rpc(['importaddress', address,'watchonly'],[4]) + res = json.loads(self.rpc(['listtransactions','watchonly','2','0','true'])) result=[] for address in addresses: @@ -95,8 +95,6 @@ def get_txs_from_addr(self, addresses): nbtxs += 1 txs.append({'confirmations':a['confirmations'],'tx':a['txid'],'amount':a['amount']}) result.append({'nb_txs':nbtxs,'address':address,'txs':txs}) - #print "Returning this data: " - #print result return {'data':result} def get_tx_info(self, txhash, raw=False): @@ -141,7 +139,7 @@ def grab_coins(self, receiving_addr, amt=50): Return the txid. ''' if amt > 500: - raise Exception("Either you forgot to pass the amount in Bitcoins, or you\'re too greedy") + raise Exception("too greedy") if amt > self.current_balance: #mine enough to get to the reqd amt reqd = int(amt - self.current_balance) @@ -164,9 +162,7 @@ def get_addr_from_utxo(self, txhash, index): return (address, amt) def main(): - bitcointoolsdir = '/home/adam/DevRepos/bitcoin/src/' - btc_client = bitcointoolsdir + 'bitcoin-cli' - myBCI = RegTestImp(btc_client) + myBCI = RegTestImp(btc_cli_loc) #myBCI.send_tx('stuff') print myBCI.get_utxos_from_addr(["n4EjHhGVS4Rod8ociyviR3FH442XYMWweD"]) print myBCI.get_balance_at_addr(["n4EjHhGVS4Rod8ociyviR3FH442XYMWweD"]) diff --git a/common.py b/common.py index 74a34419..6f146634 100644 --- a/common.py +++ b/common.py @@ -16,12 +16,16 @@ command_prefix = '!' MAX_PRIVMSG_LEN = 400 blockchain_source = 'regtest' +btc_cli_loc = '' ordername_list = ["absorder", "relorder"] encrypted_commands = ["auth", "ioauth", "tx", "sig"] plaintext_commands = ["fill", "error", "pubkey", "orderbook", "relorder", "absorder"] -def debug(msg): +def debug(msg, fname = None): print datetime.datetime.now().strftime("[%Y/%m/%d %H:%M:%S] ") + msg + if fname: #TODO: this is an awfully ugly way to write a log... + with open(fname,'ab') as f: + f.write(outmsg) def chunks(d, n): return [d[x: x+n] for x in xrange(0, len(d), n)] @@ -43,12 +47,12 @@ def get_signed_tx(wallet, ins, outs): tx = btc.sign(tx, index, wallet.get_key_from_addr(addr)) return tx -def debug_dump_object(obj, skip_fields=[]): - print 'Class debug dump, name:' + obj.__class__.__name__ +def debug_dump_object(obj, skip_fields=[], fname = None): + debug('Class debug dump, name:' + obj.__class__.__name__, fname) for k, v in obj.__dict__.iteritems(): if k in skip_fields: continue - print 'key=' + k + debug('key=' + k, fname) if isinstance(v, str): print 'string: len:' + str(len(v)) print v @@ -99,14 +103,15 @@ def get_blockr_data(req): return btc.make_request(req) def get_regtest_data(req): - bitcointoolsdir = '/home/adam/DevRepos/bitcoin/src/' - btc_client = bitcointoolsdir + 'bitcoin-cli' - myBCI = blockchaininterface.RegTestImp(btc_client) + myBCI = blockchaininterface.RegTestImp(btc_cli_loc) if not req.startswith('regtest'): raise Exception("Invalid request to regtest") req = ''.join(req.split(':')[1:]).split('/') if req[0]=='address' and req[1]=='txs': addrs = req[2].split(',') + if '?' in addrs[-1]: + #TODO simulate dealing with unconfirmed + addrs[-1] = addrs[-1].split('?')[0] #NB: we don't allow unconfirmeds in regtest #for now; TODO if 'unconfirmed' in addrs[-1]: @@ -302,8 +307,6 @@ def find_unspent_addresses(self): # but dont know which privkey to sign with data = get_blockchain_data('addrunspent', csv_params=req, query_params=['unconfirmed=1']) - print 'got this addressunspent data: ' - print data if 'unspent' in data: data = [data] for dat in data: diff --git a/regtesttest.py b/regtesttest.py new file mode 100644 index 00000000..2b431118 --- /dev/null +++ b/regtesttest.py @@ -0,0 +1,128 @@ +import sys +import os +import subprocess +import unittest +from common import * +from blockchaininterface import * +import bitcoin as btc +import binascii + +'''Expectations +1. Any bot should run indefinitely irrespective of the input +messages it receives, except bots which perform a finite action + +2. A bot must never spend an unacceptably high transaction fee. + +3. A bot must explicitly reject interactions with another bot not +respecting the JoinMarket protocol for its version. + +4. Bots must never send bitcoin data in the clear over the wire. +''' + +'''helper functions put here to avoid polluting the main codebase.''' + +import platform +OS = platform.system() +PINL = '\r\n' if OS == 'Windows' else '\n' + +def local_command(command,bg=False,redirect=''): + if redirect=='NULL': + if OS=='Windows': + command.append(' > NUL 2>&1') + elif OS=='Linux': + command.extend(['>', '/dev/null', '2>&1']) + else: + print "OS not recognised, quitting." + elif redirect: + command.extend(['>',redirect]) + if OS == 'Windows': + if bg: + #20 Sep 2013: + #a hack is needed here. + #Additional note: finally fixed this incredibly pernicious bug! + #for details, see my post at: http://www.reddit.com/r/Python/ + #comments/1mpxus/subprocess_modules_and_double_quotes/ccc4sqr + return subprocess.Popen(command,stdout=subprocess.PIPE,\ + stderr=subprocess.PIPE,stdin=subprocess.PIPE) + else: + return subprocess.check_output(command) + elif OS == 'Linux': + if bg: + return subprocess.Popen(command,stdout=subprocess.PIPE,\ + stderr=subprocess.PIPE,stdin=subprocess.PIPE) + else: + #in case of foreground execution, we can use the output; if not + #it doesn't matter + return subprocess.check_output(command) + else: + print "OS not recognised, quitting." + +class Join2PTests(unittest.TestCase): + def setUp(self): + for cf in ['yield.out','send.out']: + if (os.path.isfile(cf)): + os.remove(cf) + #create 2 new random wallets. + #put 100 coins into the first receive address + #to allow that bot to start. + seed1, seed2 = [binascii.hexlify(x) for x in [os.urandom(15), os.urandom(15)]] + self.wallets = {} + wallet1 = Wallet(seed1) + wallet2 = Wallet(seed2) + self.wallets[1] = {'seed':seed1,'wallet':wallet1} + self.wallets[2] = {'seed':seed2,'wallet':wallet2} + bci = RegTestImp(btc_cli_loc) + #get first address in each wallet + addr1 = wallet1.get_receive_addr(0) + print "got this address for wallet1: "+addr1 + addr2 = wallet2.get_receive_addr(0) + print "got this address for wallet2: "+addr2 + bci.grab_coins(addr1,10) + bci.grab_coins(addr2,10) + + def run_single_send(self): + #start yield generator with wallet1 + print "This is the seed: " + print self.wallets[1]['seed'] + yigen_proc = local_command(['python','yield-generator.py',str(self.wallets[1]['seed'])],bg=True) + + #A significant delay is needed to wait for the yield generator to sync its wallet + time.sleep(30) + + #run a single sendpayment call with wallet2 + amt = 100000000 #in satoshis + dest_address = btc.privkey_to_address(os.urandom(32),get_addr_vbyte()) + try: + sp_proc = local_command(['python','sendpayment.py','-N','1',self.wallets[2]['seed'],\ + str(amt),dest_address]) + except subprocess.CalledProcessError, e: + if yigen_proc: + yigen_proc.kill() + print e.returncode + print e.message + raise + + if yigen_proc: + yigen_proc.kill() + + for cf in ['yield.out','send.out']: + if os.path.isfile(cf): + with open(cf, 'rb') as f: + if 'CRASHING' in f.read(): return False + + myBCI = blockchaininterface.RegTestImp(btc_cli_loc) + received = myBCI.get_balance_at_addr([dest_address])['data'][0]['balance'] + if received != amt: + return False + return True + + def testSimpleSend(self): + self.failUnless(self.run_single_send()) + +def main(): + unittest.main() + +if __name__ == '__main__': + main() + + diff --git a/sendpayment.py b/sendpayment.py index 95e87219..513a730d 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -97,6 +97,8 @@ def main(): options.waittime, options.mixdepth) try: taker.run(HOST, PORT, nickname, CHANNEL) + except: + debug('We got an exception', fname='send.out') finally: debug('CRASHING, DUMPING EVERYTHING') debug('wallet seed = ' + seed) diff --git a/testing/regtesttest.py b/testing/regtesttest.py deleted file mode 100644 index a9e5d9d5..00000000 --- a/testing/regtesttest.py +++ /dev/null @@ -1,34 +0,0 @@ -import sys -import os -import subprocess -import unittest - -'''Expectations -1. Any bot should run indefinitely irrespective of the input -messages it receives, except: -a. Bots which perform a finite action -b. When there is a network failure, the bot should quit gracefully. - -2. A bot must never spend an unacceptably high transaction fee. - -3. A bot must explicitly reject interactions with another bot not -respecting the JoinMarket protocol for its version. - - -''' -bitcointoolsdir = '/home/adam/bitcoin/bitcoin-0.9.1-linux/bin/64/' -btc_client = bitcointoolsdir + 'bitcoin-cli' -btc_client_flags = '-regtest' - -class FooTests(unittest.TestCase): - def testFoo(self): - self.failUnless(False) - #subprocess.Popen([btc_client,btc_client_flags,'listunspent']) - -def main(): - unittest.main() - -if __name__ == '__main__': - main() - - diff --git a/yield-generator.py b/yield-generator.py index 1453a32b..4c0c4b03 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -88,7 +88,7 @@ def main(): try: maker.run(HOST, PORT, nickname, CHANNEL) finally: - debug('CRASHING, DUMPING EVERYTHING') + debug('CRASHING, DUMPING EVERYTHING',fname='yield.out') debug('wallet seed = ' + seed) debug_dump_object(wallet, ['addr_cache']) debug_dump_object(maker) From b395a9dc752c864402f15794342883569369e9e6 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Sat, 21 Feb 2015 14:43:47 +0200 Subject: [PATCH 143/409] N party join tests and other cleanup --- blockchaininterface.py | 9 ++-- common.py | 7 +-- regtesttest.py | 118 +++++++++++++++++++++++++++++++---------- sendpayment.py | 2 +- taker.py | 4 +- yield-generator.py | 7 +-- 6 files changed, 105 insertions(+), 42 deletions(-) diff --git a/blockchaininterface.py b/blockchaininterface.py index 3a5b104a..ffa00d36 100644 --- a/blockchaininterface.py +++ b/blockchaininterface.py @@ -38,8 +38,8 @@ def get_addr_from_utxo(self, txhash, index): #to be instantiated after network is up #with > 100 blocks. class RegTestImp(BlockChainInterface): - def __init__(self, client_location): - self.command_params = [client_location+'bitcoin-cli','-regtest'] + def __init__(self): + self.command_params = ['bitcoin-cli','-regtest'] #quick check that it's up else quit res = self.rpc(['getbalance']) try: @@ -122,7 +122,8 @@ def get_balance_at_addr(self, addresses): res = [] for address in addresses: self.rpc(['importaddress', address,'watchonly'],[4]) - res.append({'address':address,'balance':int(Decimal(1e8) * Decimal(self.rpc(['getreceivedbyaddress',address])))}) + res.append({'address':address,'balance':\ + int(Decimal(1e8) * Decimal(self.rpc(['getreceivedbyaddress', address])))}) return {'data':res} def tick_forward_chain(self, n): @@ -162,7 +163,7 @@ def get_addr_from_utxo(self, txhash, index): return (address, amt) def main(): - myBCI = RegTestImp(btc_cli_loc) + myBCI = RegTestImp() #myBCI.send_tx('stuff') print myBCI.get_utxos_from_addr(["n4EjHhGVS4Rod8ociyviR3FH442XYMWweD"]) print myBCI.get_balance_at_addr(["n4EjHhGVS4Rod8ociyviR3FH442XYMWweD"]) diff --git a/common.py b/common.py index 6f146634..c7c1b496 100644 --- a/common.py +++ b/common.py @@ -22,7 +22,8 @@ plaintext_commands = ["fill", "error", "pubkey", "orderbook", "relorder", "absorder"] def debug(msg, fname = None): - print datetime.datetime.now().strftime("[%Y/%m/%d %H:%M:%S] ") + msg + outmsg = datetime.datetime.now().strftime("[%Y/%m/%d %H:%M:%S] ") + msg + print outmsg if fname: #TODO: this is an awfully ugly way to write a log... with open(fname,'ab') as f: f.write(outmsg) @@ -103,7 +104,7 @@ def get_blockr_data(req): return btc.make_request(req) def get_regtest_data(req): - myBCI = blockchaininterface.RegTestImp(btc_cli_loc) + myBCI = blockchaininterface.RegTestImp() if not req.startswith('regtest'): raise Exception("Invalid request to regtest") req = ''.join(req.split(':')[1:]).split('/') @@ -393,7 +394,7 @@ def calc_cj_fee(ordertype, cjfee, cj_amount): def calc_total_input_value(utxos): input_sum = 0 for utxo in utxos: - tx = get_blockchain_data('txraw',csv_params=[utxo[:64]])['tx']['hex'] + tx = get_blockchain_data('txraw', csv_params=[utxo[:64]])['tx']['hex'] #tx = btc.blockr_fetchtx(utxo[:64], get_network()) input_sum += int(btc.deserialize(tx)['outs'][int(utxo[65:])]['value']) return input_sum diff --git a/regtesttest.py b/regtesttest.py index 2b431118..35c2384f 100644 --- a/regtesttest.py +++ b/regtesttest.py @@ -25,7 +25,7 @@ OS = platform.system() PINL = '\r\n' if OS == 'Windows' else '\n' -def local_command(command,bg=False,redirect=''): +def local_command(command, bg=False, redirect=''): if redirect=='NULL': if OS=='Windows': command.append(' > NUL 2>&1') @@ -34,34 +34,27 @@ def local_command(command,bg=False,redirect=''): else: print "OS not recognised, quitting." elif redirect: - command.extend(['>',redirect]) + command.extend(['>', redirect]) if OS == 'Windows': if bg: - #20 Sep 2013: - #a hack is needed here. - #Additional note: finally fixed this incredibly pernicious bug! - #for details, see my post at: http://www.reddit.com/r/Python/ - #comments/1mpxus/subprocess_modules_and_double_quotes/ccc4sqr - return subprocess.Popen(command,stdout=subprocess.PIPE,\ - stderr=subprocess.PIPE,stdin=subprocess.PIPE) + return subprocess.Popen(command, stdout=subprocess.PIPE,\ + stderr=subprocess.PIPE, stdin=subprocess.PIPE) else: return subprocess.check_output(command) elif OS == 'Linux': if bg: - return subprocess.Popen(command,stdout=subprocess.PIPE,\ - stderr=subprocess.PIPE,stdin=subprocess.PIPE) + return subprocess.Popen(command, stdout=subprocess.PIPE,\ + stderr=subprocess.PIPE, stdin=subprocess.PIPE) else: #in case of foreground execution, we can use the output; if not #it doesn't matter return subprocess.check_output(command) else: print "OS not recognised, quitting." + class Join2PTests(unittest.TestCase): def setUp(self): - for cf in ['yield.out','send.out']: - if (os.path.isfile(cf)): - os.remove(cf) #create 2 new random wallets. #put 100 coins into the first receive address #to allow that bot to start. @@ -71,30 +64,30 @@ def setUp(self): wallet2 = Wallet(seed2) self.wallets[1] = {'seed':seed1,'wallet':wallet1} self.wallets[2] = {'seed':seed2,'wallet':wallet2} - bci = RegTestImp(btc_cli_loc) + bci = RegTestImp() #get first address in each wallet addr1 = wallet1.get_receive_addr(0) - print "got this address for wallet1: "+addr1 + debug("address for wallet1: "+addr1) addr2 = wallet2.get_receive_addr(0) - print "got this address for wallet2: "+addr2 + debug("address for wallet2: "+addr2) bci.grab_coins(addr1,10) bci.grab_coins(addr2,10) - def run_single_send(self): + def run_simple_send(self, n): #start yield generator with wallet1 - print "This is the seed: " - print self.wallets[1]['seed'] - yigen_proc = local_command(['python','yield-generator.py',str(self.wallets[1]['seed'])],bg=True) + yigen_proc = local_command(['python','yield-generator.py', str(self.wallets[1]['seed'])],\ + bg=True) #A significant delay is needed to wait for the yield generator to sync its wallet time.sleep(30) #run a single sendpayment call with wallet2 amt = 100000000 #in satoshis - dest_address = btc.privkey_to_address(os.urandom(32),get_addr_vbyte()) + dest_address = btc.privkey_to_address(os.urandom(32), get_addr_vbyte()) try: - sp_proc = local_command(['python','sendpayment.py','-N','1',self.wallets[2]['seed'],\ - str(amt),dest_address]) + for i in range(n): + sp_proc = local_command(['python','sendpayment.py','-N','1', self.wallets[2]['seed'],\ + str(amt), dest_address]) except subprocess.CalledProcessError, e: if yigen_proc: yigen_proc.kill() @@ -105,20 +98,87 @@ def run_single_send(self): if yigen_proc: yigen_proc.kill() - for cf in ['yield.out','send.out']: + for cf in [self.wallets[1]['seed']+'_yieldgen.out', self.wallets[2]['seed']+'_send.out']: if os.path.isfile(cf): with open(cf, 'rb') as f: if 'CRASHING' in f.read(): return False - myBCI = blockchaininterface.RegTestImp(btc_cli_loc) + myBCI = blockchaininterface.RegTestImp() received = myBCI.get_balance_at_addr([dest_address])['data'][0]['balance'] if received != amt: return False return True - def testSimpleSend(self): - self.failUnless(self.run_single_send()) - + def test_simple_send(self): + self.failUnless(self.run_simple_send(2)) + + +class JoinNPTests(unittest.TestCase): + + def setUp(self): + self.n = 2 + #create n+1 new random wallets. + #put 10 coins into the first receive address + #to allow that bot to start. + seeds = map(None, *([iter(os.urandom((self.n+1)*15))]*15)) + + seeds = [binascii.hexlify(''.join(x)) for x in seeds] + self.wallets = {} + for i, seed in enumerate(seeds): + self.wallets[i] = {'seed':seed, 'wallet':Wallet(seed)} + + bci = RegTestImp() + #get first address in each wallet + for i in self.wallets.keys(): + bci.grab_coins(self.wallets[i]['wallet'].get_receive_addr(0), amt=10) + + #the sender is wallet (n+1), i.e. index wallets[n] + + + def test_n_partySend(self): + self.failUnless(self.run_nparty_join()) + + def run_nparty_join(self): + yigen_procs = [] + for i in range(self.n): + ygp = local_command(['python','yield-generator.py',\ + str(self.wallets[i]['seed'])], bg=True) + time.sleep(2) #give it a chance + yigen_procs.append(ygp) + + #A significant delay is needed to wait for the yield generators to sync + time.sleep(60) + + #run a single sendpayment call + amt = 100000000 #in satoshis + dest_address = btc.privkey_to_address(os.urandom(32), get_addr_vbyte()) + try: + sp_proc = local_command(['python','sendpayment.py','-N', str(self.n),\ + self.wallets[self.n]['seed'], str(amt), dest_address]) + except subprocess.CalledProcessError, e: + for ygp in yigen_procs: + ygp.kill() + print e.returncode + print e.message + raise + + if any(yigen_procs): + for ygp in yigen_procs: + ygp.kill() + + crash_files = [self.wallets[i]['seed']+'_yieldgen.out' for i in range(self.n)] + crash_files.append(self.wallets[self.n]['seed']+'_send.out') + for cf in crash_files: + if os.path.isfile(cf): return False + #with open(cf, 'rb') as f: + # if 'CRASHING' in f.read(): return False + + myBCI = blockchaininterface.RegTestImp() + received = myBCI.get_balance_at_addr([dest_address])['data'][0]['balance'] + if received != amt: + return False + return True + def main(): unittest.main() diff --git a/sendpayment.py b/sendpayment.py index 513a730d..1feb15d8 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -98,7 +98,7 @@ def main(): try: taker.run(HOST, PORT, nickname, CHANNEL) except: - debug('We got an exception', fname='send.out') + debug('We got an exception', fname=seed+'_send.out') finally: debug('CRASHING, DUMPING EVERYTHING') debug('wallet seed = ' + seed) diff --git a/taker.py b/taker.py index 7c7225c5..bbe864bb 100644 --- a/taker.py +++ b/taker.py @@ -125,7 +125,7 @@ def add_signature(self, sigb64): for index, ins in enumerate(self.latest_tx['ins']): if ins['script'] != '': continue - ftx = get_blockchain_data('txraw',csv_params=[ins['outpoint']['hash']])['tx']['hex'] + ftx = get_blockchain_data('txraw', csv_params=[ins['outpoint']['hash']])['tx']['hex'] #ftx = btc.blockr_fetchtx(ins['outpoint']['hash'], get_network()) src_val = btc.deserialize(ftx)['outs'][ ins['outpoint']['index'] ] sig_good = btc.verify_tx_input(tx, index, src_val['script'], *btc.deserialize_script(sig)) @@ -149,7 +149,7 @@ def add_signature(self, sigb64): debug('the entire tx is signed, ready to pushtx()') print btc.serialize(self.latest_tx) - ret = get_blockchain_data('txpush',csv_params=[btc.serialize(self.latest_tx)]) + ret = get_blockchain_data('txpush', csv_params=[btc.serialize(self.latest_tx)]) #ret = btc.blockr_pushtx(btc.serialize(self.latest_tx), get_network()) debug('pushed tx ' + str(ret)) if self.finishcallback != None: diff --git a/yield-generator.py b/yield-generator.py index 4c0c4b03..44a8748b 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -3,7 +3,7 @@ from maker import * import bitcoin as btc import time - +import os, binascii import pprint from socket import gethostname @@ -11,7 +11,8 @@ txfee = 1000 cjfee = '0.01' # 1% fee mix_levels = 5 -nickname = 'yigen-' + btc.sha256(gethostname())[:6] +#nickname = 'yigen-' + btc.sha256(gethostname())[:6] +nickname = 'yigen-'+binascii.hexlify(os.urandom(4)) nickserv_password = '' minsize = int(2 * txfee / float(cjfee)) #minimum size is such that you always net profit at least the miners fee @@ -88,7 +89,7 @@ def main(): try: maker.run(HOST, PORT, nickname, CHANNEL) finally: - debug('CRASHING, DUMPING EVERYTHING',fname='yield.out') + debug('CRASHING, DUMPING EVERYTHING', fname=seed+'_yieldgen.out') debug('wallet seed = ' + seed) debug_dump_object(wallet, ['addr_cache']) debug_dump_object(maker) From b038ed5ffa6d9a91a46c5917687ca9c0bf85627f Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Sat, 21 Feb 2015 17:08:19 +0200 Subject: [PATCH 144/409] channel name --- common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common.py b/common.py index 46baa1e8..11aff00a 100644 --- a/common.py +++ b/common.py @@ -7,7 +7,7 @@ import blockchaininterface HOST = 'irc.freenode.net' -CHANNEL = '#joinmarket-pit-test2' +CHANNEL = '#joinmarket-pit-test' PORT = 6667 #for the mainnet its #joinmarket-pit From ea829adcceaaf165e186aa3734e54b600c8b32af Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Sat, 21 Feb 2015 19:34:12 +0200 Subject: [PATCH 145/409] reset blockchain source to blockr by default --- common.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/common.py b/common.py index 11aff00a..fed89a72 100644 --- a/common.py +++ b/common.py @@ -14,7 +14,7 @@ COMMAND_PREFIX = '!' MAX_PRIVMSG_LEN = 400 -blockchain_source = 'regtest' +blockchain_source = 'blockr' btc_cli_loc = '' ordername_list = ["absorder", "relorder"] encrypted_commands = ["auth", "ioauth", "tx", "sig"] @@ -100,6 +100,8 @@ def get_blockchain_data(body, csv_params=[], return res[output_key] def get_blockr_data(req): + if 'tx/push' in req: #manually parse this; special case because POST not GET? + return btc.blockr_pushtx(req.split('/')[-1], get_network()) return btc.make_request(req) def get_regtest_data(req): From f67c90ee24c229624577002e233ef4677cd77420 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Sun, 22 Feb 2015 22:02:15 +0200 Subject: [PATCH 146/409] changes to allow debug logging to file --- common.py | 42 ++++++++++++++++++++++-------------------- irc.py | 4 ++-- regtesttest.py | 44 +++++++++++++++++++------------------------- sendpayment.py | 13 +++++++------ yield-generator.py | 12 ++++++------ 5 files changed, 56 insertions(+), 59 deletions(-) diff --git a/common.py b/common.py index fed89a72..3871704d 100644 --- a/common.py +++ b/common.py @@ -11,21 +11,23 @@ PORT = 6667 #for the mainnet its #joinmarket-pit - +nickname = '' COMMAND_PREFIX = '!' MAX_PRIVMSG_LEN = 400 blockchain_source = 'blockr' -btc_cli_loc = '' ordername_list = ["absorder", "relorder"] encrypted_commands = ["auth", "ioauth", "tx", "sig"] plaintext_commands = ["fill", "error", "pubkey", "orderbook", "relorder", "absorder"] +debug_file_handle = None -def debug(msg, fname = None): +def debug(msg): + global debug_file_handle + if nickname and not debug_file_handle: + debug_file_handle = open(nickname+'.log','ab') outmsg = datetime.datetime.now().strftime("[%Y/%m/%d %H:%M:%S] ") + msg print outmsg - if fname: #TODO: this is an awfully ugly way to write a log... - with open(fname,'ab') as f: - f.write(outmsg) + if nickname: #debugs before creating bot nick won't be handled like this + debug_file_handle.write(outmsg + '\n') def chunks(d, n): return [d[x: x+n] for x in xrange(0, len(d), n)] @@ -47,19 +49,19 @@ def get_signed_tx(wallet, ins, outs): tx = btc.sign(tx, index, wallet.get_key_from_addr(addr)) return tx -def debug_dump_object(obj, skip_fields=[], fname = None): - debug('Class debug dump, name:' + obj.__class__.__name__, fname) +def debug_dump_object(obj, skip_fields=[]): + debug('Class debug dump, name:' + obj.__class__.__name__) for k, v in obj.__dict__.iteritems(): if k in skip_fields: continue - debug('key=' + k, fname) + debug('key=' + k) if isinstance(v, str): - print 'string: len:' + str(len(v)) - print v + debug('string: len:' + str(len(v))) + debug(v) elif isinstance(v, dict) or isinstance(v, list): - pprint.pprint(v) + debug(pprint.pformat(v)) else: - print v + debug(v) def get_addr_from_utxo(txhash, index): '''return the bitcoin address of the outpoint at @@ -239,7 +241,7 @@ def select_utxos(self, mixdepth, amount): for utxo in utxo_list] inputs = btc.select(unspent, amount) debug('for mixdepth=' + str(mixdepth) + ' amount=' + str(amount) + ' selected:') - pprint.pprint(inputs) + debug(pprint.pformat(inputs)) return [i['utxo'] for i in inputs] def sync_wallet(self, gaplimit=6): @@ -293,7 +295,7 @@ def find_unspent_addresses(self): for n in range(self.index[m][forchange]): addrs[self.get_addr(m, forchange, n)] = m if len(addrs) == 0: - print 'no tx used' + debug('no tx used') return i = 0 @@ -319,10 +321,10 @@ def find_unspent_addresses(self): def print_debug_wallet_info(self): debug('printing debug wallet information') - print 'utxos' - pprint.pprint(self.unspent) - print 'wallet.index' - pprint.pprint(self.index) + debug('utxos') + debug(pprint.pformat(self.unspent)) + debug('wallet.index') + debug(pprint.pformat(self.index)) #awful way of doing this, but works for now @@ -492,7 +494,7 @@ def is_amount_in_range(ordercombo, cjamount): ordercombos = sorted(ordercombos, key=lambda k: k[1]) dbgprint = [([(o['counterparty'], o['oid']) for o in oc[0]], oc[1]) for oc in ordercombos] debug('considered order combinations') - pprint.pprint(dbgprint) + debug(pprint.pformat(dbgprint)) ordercombo = ordercombos[-1] #choose the cheapest, i.e. highest cj_amount orders = dict([(o['counterparty'], o['oid']) for o in ordercombo[0]]) cjamount = ordercombo[1] diff --git a/irc.py b/irc.py index c150928e..6807261e 100644 --- a/irc.py +++ b/irc.py @@ -410,13 +410,13 @@ def run(self): break self.__handle_line(line) except IOError as e: - print repr(e) + debug(repr(e)) finally: self.fd.close() self.sock.close() if self.on_disconnect: self.on_disconnect() - print 'disconnected irc' + debug('disconnected irc') time.sleep(10) self.connect_attempts += 1 debug('ending irc') diff --git a/regtesttest.py b/regtesttest.py index 35c2384f..3e767954 100644 --- a/regtesttest.py +++ b/regtesttest.py @@ -35,22 +35,15 @@ def local_command(command, bg=False, redirect=''): print "OS not recognised, quitting." elif redirect: command.extend(['>', redirect]) - if OS == 'Windows': - if bg: - return subprocess.Popen(command, stdout=subprocess.PIPE,\ - stderr=subprocess.PIPE, stdin=subprocess.PIPE) - else: - return subprocess.check_output(command) - elif OS == 'Linux': - if bg: - return subprocess.Popen(command, stdout=subprocess.PIPE,\ - stderr=subprocess.PIPE, stdin=subprocess.PIPE) - else: - #in case of foreground execution, we can use the output; if not - #it doesn't matter - return subprocess.check_output(command) + + if bg: + return subprocess.Popen(command, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, stdin=subprocess.PIPE) else: - print "OS not recognised, quitting." + #in case of foreground execution, we can use the output; if not + #it doesn't matter + return subprocess.check_output(command) + class Join2PTests(unittest.TestCase): @@ -75,8 +68,8 @@ def setUp(self): def run_simple_send(self, n): #start yield generator with wallet1 - yigen_proc = local_command(['python','yield-generator.py', str(self.wallets[1]['seed'])],\ - bg=True) + yigen_proc = local_command(['python','yield-generator.py', + str(self.wallets[1]['seed'])], bg=True) #A significant delay is needed to wait for the yield generator to sync its wallet time.sleep(30) @@ -90,18 +83,18 @@ def run_simple_send(self, n): str(amt), dest_address]) except subprocess.CalledProcessError, e: if yigen_proc: - yigen_proc.kill() + yigen_proc.terminate() print e.returncode print e.message raise - + if yigen_proc: - yigen_proc.kill() + yigen_proc.terminate() - for cf in [self.wallets[1]['seed']+'_yieldgen.out', self.wallets[2]['seed']+'_send.out']: - if os.path.isfile(cf): - with open(cf, 'rb') as f: - if 'CRASHING' in f.read(): return False + #for cf in [self.wallets[1]['seed']+'_yieldgen.out', self.wallets[2]['seed']+'_send.out']: + # if os.path.isfile(cf): + # with open(cf, 'rb') as f: + # if 'CRASHING' in f.read(): return False myBCI = blockchaininterface.RegTestImp() received = myBCI.get_balance_at_addr([dest_address])['data'][0]['balance'] @@ -178,7 +171,8 @@ def run_nparty_join(self): if received != amt: return False return True - + + def main(): unittest.main() diff --git a/sendpayment.py b/sendpayment.py index 6c2aa1d4..7dcc50ed 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -1,6 +1,7 @@ #! /usr/bin/env python from common import * +import common import taker as takermodule from irc import IRCMessageChannel import bitcoin as btc @@ -87,25 +88,25 @@ def main(): amount = int(args[1]) destaddr = args[2] - from socket import gethostname - nickname = 'payer-' + btc.sha256(gethostname())[:6] + import binascii, os + common.nickname = 'payer-' +binascii.hexlify(os.urandom(4)) wallet = Wallet(seed, options.mixdepth + 1) wallet.sync_wallet() - irc = IRCMessageChannel(nickname) + irc = IRCMessageChannel(common.nickname) taker = SendPayment(irc, wallet, destaddr, amount, options.makercount, options.txfee, options.waittime, options.mixdepth) try: - print 'starting irc' + debug('starting irc') irc.run() except: - debug('CRASHING, DUMPING EVERYTHING', fname=seed+'_send.out') + debug('CRASHING, DUMPING EVERYTHING') debug('wallet seed = ' + seed) debug_dump_object(wallet, ['addr_cache']) debug_dump_object(taker) import traceback - traceback.print_exc() + debug(traceback.print_exc()) if __name__ == "__main__": main() diff --git a/yield-generator.py b/yield-generator.py index db8e2a42..66c5a5fe 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -6,14 +6,13 @@ import time import os, binascii import pprint +import common from socket import gethostname txfee = 1000 cjfee = '0.01' # 1% fee mix_levels = 5 -#nickname = 'yigen-' + btc.sha256(gethostname())[:6] -nickname = 'yigen-'+binascii.hexlify(os.urandom(4)) nickserv_password = '' minsize = int(2 * txfee / float(cjfee)) #minimum size is such that you always net profit at least the miners fee @@ -87,18 +86,19 @@ def main(): wallet = Wallet(seed, max_mix_depth = mix_levels) wallet.sync_wallet() - irc = IRCMessageChannel(nickname) + common.nickname = 'yigen-'+binascii.hexlify(os.urandom(4)) + irc = IRCMessageChannel(common.nickname) maker = YieldGenerator(irc, wallet) try: - print 'connecting to irc' + debug('connecting to irc') irc.run() except: - debug('CRASHING, DUMPING EVERYTHING', fname=seed+'_yieldgen.out') + debug('CRASHING, DUMPING EVERYTHING') debug('wallet seed = ' + seed) debug_dump_object(wallet, ['addr_cache']) debug_dump_object(maker) import traceback - traceback.print_exc() + debug(traceback.print_exc()) if __name__ == "__main__": main() From cc59f06c1cfcd739d015f142e7e789c559ccd470 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Sun, 22 Feb 2015 22:37:09 +0200 Subject: [PATCH 147/409] fixed exception passing to log --- common.py | 2 +- sendpayment.py | 2 +- yield-generator.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common.py b/common.py index 3871704d..c01de30b 100644 --- a/common.py +++ b/common.py @@ -61,7 +61,7 @@ def debug_dump_object(obj, skip_fields=[]): elif isinstance(v, dict) or isinstance(v, list): debug(pprint.pformat(v)) else: - debug(v) + debug(str(v)) def get_addr_from_utxo(txhash, index): '''return the bitcoin address of the outpoint at diff --git a/sendpayment.py b/sendpayment.py index 7dcc50ed..872bbe10 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -106,7 +106,7 @@ def main(): debug_dump_object(wallet, ['addr_cache']) debug_dump_object(taker) import traceback - debug(traceback.print_exc()) + debug(traceback.format_exc()) if __name__ == "__main__": main() diff --git a/yield-generator.py b/yield-generator.py index 66c5a5fe..3b44b58a 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -98,7 +98,7 @@ def main(): debug_dump_object(wallet, ['addr_cache']) debug_dump_object(maker) import traceback - debug(traceback.print_exc()) + debug(traceback.format_exc()) if __name__ == "__main__": main() From d2b2c723b953b83e008f6b1a9e6170b2f3f76a61 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Mon, 23 Feb 2015 22:19:58 +0200 Subject: [PATCH 148/409] changes to other bot setup --- gui-taker.py | 7 ++++--- patientsendpayment.py | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/gui-taker.py b/gui-taker.py index 41cecd64..6c2b14fe 100644 --- a/gui-taker.py +++ b/gui-taker.py @@ -159,11 +159,12 @@ def on_welcome(self): HTTPDThread(self).start() def main(): - from socket import gethostname import bitcoin as btc - nickname = 'guitaker-' + btc.sha256(gethostname())[:6] + import common + import binascii, os + common.nickname = 'guitaker-' +binascii.hexlify(os.urandom(4)) - irc = IRCMessageChannel(nickname) + irc = IRCMessageChannel(common.nickname) taker = GUITaker(irc) print 'starting irc' irc.run() diff --git a/patientsendpayment.py b/patientsendpayment.py index 88a3132e..b70717c6 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -151,10 +151,10 @@ def main(): print 'not enough money at mixdepth=%d, exiting' % (options.mixdepth) return - from socket import gethostname - nickname = 'ppayer-' + btc.sha256(gethostname())[:6] + import common, binascii, os + common.nickname = 'ppayer-' +binascii.hexlify(os.urandom(4)) - irc = IRCMessageChannel(nickname) + irc = IRCMessageChannel(common.nickname) bot = PatientSendPayment(irc, wallet, destaddr, amount, options.makercount, options.txfee, options.cjfee, waittime, options.mixdepth) try: From 3b36f4c0374e833bb965e2e0a008e52f4df49096 Mon Sep 17 00:00:00 2001 From: chris belcher Date: Wed, 25 Feb 2015 18:58:27 +0000 Subject: [PATCH 149/409] fixed tumbler power law code, and other stuff --- tumbler.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tumbler.py b/tumbler.py index 9dd67310..0c913d44 100644 --- a/tumbler.py +++ b/tumbler.py @@ -4,7 +4,7 @@ import numpy as np from pprint import pprint -def int_lower_bounded(thelist, lowerbound): +def lower_bounded_int(thelist, lowerbound): return [int(l) if int(l) >= lowerbound else lowerbound for l in thelist] def generate_tumbler_tx(destaddrs, options): @@ -18,18 +18,17 @@ def generate_tumbler_tx(destaddrs, options): # follows a normal distribution txcounts = np.random.normal(options.txcountparams[0], options.txcountparams[1], options.mixdepthcount) - txcounts = int_lower_bounded(txcounts, 1) - + txcounts = lower_bounded_int(txcounts, 1) tx_list = [] for m, txcount in enumerate(txcounts): #assume that the sizes of outputs will follow a power law - amount_ratios = np.random.power(options.amountpower, txcount) + amount_ratios = 1.0 - np.random.power(options.amountpower, txcount) amount_ratios /= sum(amount_ratios) #transaction times are uncorrelated and therefore follow poisson blockheight_waits = np.random.poisson(options.timelambda, txcount) #number of makers to use follows a normal distribution makercounts = np.random.normal(options.makercountrange[0], options.makercountrange[1], txcount) - makercounts = int_lower_bounded(makercounts, 2) + makercounts = lower_bounded_int(makercounts, 2) for amount_ratio, blockheight_wait, makercount in zip(amount_ratios, blockheight_waits, makercounts): tx = {'amount_ratio': amount_ratio, 'blockheight_wait': blockheight_wait, 'srcmixdepth': m + options.mixdepthsrc, 'makercount': makercount} @@ -57,7 +56,8 @@ def main(): parser.add_option('-f', '--txfee', type='int', dest='txfee', default=10000, help='miner fee contribution, in satoshis, default=10000') parser.add_option('-a', '--addrask', type='int', dest='addrask', - default=2, help='How many more addresses to ask for in the terminal, default=2') + default=2, help='How many more addresses to ask for in the terminal. Should ' + 'be similar to --txcountparams. default=2') parser.add_option('-N', '--makercountrange', type='float', nargs=2, action='store', dest='makercountrange', help='Input the range of makers to use. e.g. 3-5 will random use between ' @@ -66,10 +66,10 @@ def main(): help='how many mixing depths to mix through', default=3) parser.add_option('-c', '--txcountparams', type='float', nargs=2, dest='txcountparams', default=(5, 1), help='The number of transactions to take coins from one mixing depth to the next, it is' - ' randomly chosen following a normal distribution. This option controlled the parameters' - ' of that normal curve. (mean, standard deviation). default=(5, 1)') - parser.add_option('--amountpower', type='float', dest='amountpower', default=3.0, - help='the output amounts follow a power law distribution, this is the power, default=3.0') + ' randomly chosen following a normal distribution. Should be similar to --addrask. ' + 'This option controlled the parameters of that normal curve. (mean, standard deviation). default=(3, 1)') + parser.add_option('--amountpower', type='float', dest='amountpower', default=100.0, + help='the output amounts follow a power law distribution, this is the power, default=100.0') parser.add_option('-l', '--timelambda', type='float', dest='timelambda', default=2, help='the number of blocks to wait between transactions is randomly chosen ' ' following a poisson distribution. This parameter is the lambda of that ' @@ -98,10 +98,10 @@ def main(): #a couple of overarching modes #im-running-from-the-nsa, takes about 80 hours, costs a lot - #python tumbler.py -N 10 5 -c 10 5 -l 5 -M 10 seed 1 + #python tumbler.py -a 10 -N 10 5 -c 10 5 -l 5 -M 10 seed 1xxx # #quick and cheap, takes about 90 minutes - #python tumbler.py -N 2 1 -c 3 0.001 -l 2 -M 2 seed 1 + #python tumbler.py -N 2 1 -c 3 0.001 -l 2 -M 2 seed 1xxx 1yyy # #default, good enough for most, takes about 5 hours #python tumbler.py seed 1 From ad7bfa328398e0f3c6f7e479ac6ebfef7e8361e9 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Fri, 27 Feb 2015 00:06:51 +0200 Subject: [PATCH 150/409] basic config in joinmarket.cfg file --- common.py | 29 +++++++++++++++++++++++------ irc.py | 6 +++--- joinmarket.cfg | 11 +++++++++++ patientsendpayment.py | 4 +++- sendpayment.py | 2 +- tumbler.py | 5 ++++- wallet-tool.py | 4 +++- yield-generator.py | 4 ++-- 8 files changed, 50 insertions(+), 15 deletions(-) create mode 100644 joinmarket.cfg diff --git a/common.py b/common.py index 5fa820e3..38903e78 100644 --- a/common.py +++ b/common.py @@ -4,12 +4,8 @@ from math import factorial import sys, datetime, json, time, pprint import threading - -HOST = 'irc.freenode.net' -CHANNEL = '#joinmarket-pit-test' -PORT = 6667 - -#for the mainnet its #joinmarket-pit +from ConfigParser import SafeConfigParser +import os COMMAND_PREFIX = '!' MAX_PRIVMSG_LEN = 400 @@ -18,6 +14,27 @@ encrypted_commands = ["auth", "ioauth", "tx", "sig"] plaintext_commands = ["fill", "error", "pubkey", "orderbook", "relorder", "absorder"] +config = SafeConfigParser() +config_location = os.path.join(os.path.dirname(os.path.realpath(__file__)),'joinmarket.cfg') +required_options = {'BLOCKCHAIN':['blockchain_source','port','rpcport','network'], + 'MESSAGING':['host','channel','port']} + +def load_program_config(): + loadedFiles = config.read([config_location]) + #detailed sanity checking : + #did the file exist? + if len(loadedFiles) != 1: + raise Exception("Could not find config file: "+config_location) + #check for sections + for s in required_options: + if s not in config.sections(): + raise Exception("Config file does not contain the required section: "+s) + #then check for specific options + for k,v in required_options.iteritems(): + for o in v: + if o not in config.options(k): + raise Exception("Config file does not contain the required option: "+o) + def debug(msg): print datetime.datetime.now().strftime("[%Y/%m/%d %H:%M:%S] ") + msg diff --git a/irc.py b/irc.py index c150928e..7bf12f4b 100644 --- a/irc.py +++ b/irc.py @@ -372,12 +372,12 @@ def __handle_line(self, line): self.motd_fd.close() ''' - def __init__(self, nick, server=HOST, port=PORT, channel=CHANNEL, username='username', realname='realname'): + def __init__(self, nick, username='username', realname='realname'): MessageChannel.__init__(self) self.cjpeer = None #subclasses have to set this to self self.nick = nick - self.serverport = (server, port) - self.channel = channel + self.serverport = (config.get("MESSAGING","host"), int(config.get("MESSAGING","port"))) + self.channel = '#'+config.get("MESSAGING","channel") self.userrealname = (username, realname) def run(self): diff --git a/joinmarket.cfg b/joinmarket.cfg new file mode 100644 index 00000000..7b4b9e23 --- /dev/null +++ b/joinmarket.cfg @@ -0,0 +1,11 @@ +[BLOCKCHAIN] +blockchain_source = blockr +port = 8331 +rpcport = 18331 +network = testnet + +[MESSAGING] +host = irc.freenode.net +channel = joinmarket-pit-test +port = 6667 +#More stuff to come \ No newline at end of file diff --git a/patientsendpayment.py b/patientsendpayment.py index 88a3132e..a254acd5 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -134,7 +134,9 @@ def main(): seed = args[0] amount = int(args[1]) destaddr = args[2] - + + load_program_config() + waittime = timedelta(hours=options.waittime).total_seconds() print 'Running patient sender of a payment' print 'txfee=%d cjfee=%d waittime=%s makercount=%d' % (options.txfee, options.cjfee, diff --git a/sendpayment.py b/sendpayment.py index eb3a907c..6504e7f1 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -86,7 +86,7 @@ def main(): seed = args[0] amount = int(args[1]) destaddr = args[2] - + load_program_config() from socket import gethostname nickname = 'payer-' + btc.sha256(gethostname())[:6] diff --git a/tumbler.py b/tumbler.py index 0c913d44..2c086d53 100644 --- a/tumbler.py +++ b/tumbler.py @@ -3,6 +3,7 @@ import datetime import numpy as np from pprint import pprint +import common def lower_bounded_int(thelist, lowerbound): return [int(l) if int(l) >= lowerbound else lowerbound for l in thelist] @@ -84,7 +85,9 @@ def main(): sys.exit(0) seed = args[0] destaddrs = args[1:] - + + common.load_program_config() + if len(destaddrs) + options.addrask <= 1: print '='*50 print 'WARNING: You are only using one destination address' diff --git a/wallet-tool.py b/wallet-tool.py index ddc199d1..803c2050 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -1,6 +1,6 @@ import bitcoin as btc -from common import Wallet, get_signed_tx +from common import Wallet, get_signed_tx, load_program_config import sys from optparse import OptionParser @@ -38,6 +38,8 @@ method = ('display' if len(args) == 1 else args[1].lower()) +load_program_config() + #seed = '256 bits of randomness' print_privkey = options.showprivkey diff --git a/yield-generator.py b/yield-generator.py index 0711cc1f..43210842 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -4,7 +4,7 @@ from irc import IRCMessageChannel import bitcoin as btc import time - +from common import * import pprint from socket import gethostname @@ -83,7 +83,7 @@ def on_tx_confirmed(self, cjorder, confirmations, txid, balance, added_utxos): def main(): import sys seed = sys.argv[1] #btc.sha256('dont use brainwallets except for holding testnet coins') - + load_program_config() wallet = Wallet(seed, max_mix_depth = mix_levels) wallet.sync_wallet() irc = IRCMessageChannel(nickname) From f55279a52ba44d5eccb8d706900cbd4ae2d29dfe Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Thu, 26 Feb 2015 23:32:40 +0200 Subject: [PATCH 151/409] waypoint blockchain interface cleaned up, calls polymorphic bug fixes --- blockchaininterface.py | 172 ++++++++++++++++++++++++++++++----------- common.py | 132 ++++++++++++++----------------- irc.py | 6 +- joinmarket.cfg | 11 +++ sendpayment.py | 4 +- taker.py | 4 +- wallet-tool.py | 4 +- yield-generator.py | 1 + 8 files changed, 207 insertions(+), 127 deletions(-) create mode 100644 joinmarket.cfg diff --git a/blockchaininterface.py b/blockchaininterface.py index ffa00d36..5e182ad6 100644 --- a/blockchaininterface.py +++ b/blockchaininterface.py @@ -11,47 +11,106 @@ class BlockChainInterface(object): __metaclass__ = abc.ABCMeta def __init__(self): #TODO: pass in network type (main/test) + self.fptrs = {'addrtx': self.get_txs_from_addr, + 'txinfo': self.get_tx_info, + 'addrunspent': self.get_utxos_from_addr, + 'addrbalance': self.get_balance_at_addr, + 'txraw': self.get_tx_info, + 'txpush': self.send_tx} + + def parse_request(self, body, csv_params, query_params=None): + return self.fptrs[body](csv_params, query_params) + + @abc.abstractmethod + def get_txs_from_addr(self, addresses, query_params): + '''Given a list of addresses, list all transactions''' pass + @abc.abstractmethod - def get_utxos_from_addr(self, address): + def get_tx_info(self, txhash, query_params): + '''Given a txhash and query params indicating raw, + return the tx hex. If indicating non-raw, return a list of vouts. + May need some more structure in query_params to handle unconfirmed. TODO''' + pass + + @abc.abstractmethod + def get_utxos_from_addr(self, addresses, query_params): '''Given an address, return a list of utxos in format txid:vout''' pass - def get_balance_at_addr(self, address): + + @abc.abstractmethod + def get_balance_at_addr(self, addresses, query_params): '''Given an address, return a balance in satoshis''' pass @abc.abstractmethod - def send_tx(self, tx_hex): + def send_tx(self, tx_hexs, query_params): '''Given raw txhex, push to network and return result in form: TODO''' - pass + pass + + @abc.abstractmethod def get_net_info(self): pass + ''' + @abc.abstractmethod def get_addr_from_utxo(self, txhash, index): - '''Given utxo in form txhash, index, return the address - owning the utxo and the amount in satoshis in form (addr, amt)''' + Given utxo in form txhash, index, return the address + owning the utxo and the amount in satoshis in form (addr, amt) pass + ''' +class BlockrImp(BlockChainInterface): + def __init__(self, testnet = True): + super(BlockrImp, self).__init__() + self.bodies = {'addrtx':'address/txs/','txinfo':'tx/info/','addrunspent':'address/unspent/', + 'addrbalance':'address/balance/','txraw':'tx/raw/','txpush':'tx/push/'} + self.testnet = 'testnet' if testnet else 'btc' #see bci.py in bitcoin module + self.query_stem = 'http://tbtc.blockr.io/api/v1/' if testnet else 'http://tbtc.blockr.io/api/v1/' + + def parse_request(self, body, csv_params, query_params=None): + if body=='pushtx': + return super(BlockrImp, self).parse_request(body, csv_params, query_params) + else: + req = self.query_stem + self.bodies[body] + '/' + ','.join(csv_params) + '?' + ','.join(query_params) + return btc.make_request(req) + + def send_tx(self, tx_hexs, query_params): + #TODO: handle multiple txs? + return btc.blockr_pushtx(tx_hexs[0], self.testnet) -#class for regtest chain access -#running on local daemon. Only -#to be instantiated after network is up -#with > 100 blocks. -class RegTestImp(BlockChainInterface): - def __init__(self): - self.command_params = ['bitcoin-cli','-regtest'] + def get_net_info(self): + print 'not yet done' + + def get_txs_from_addr(self, addresses, query_params): + pass + + def get_tx_info(self, txhash, query_params): + pass + + def get_utxos_from_addr(self, addresses, query_params): + pass + + def get_balance_at_addr(self, addresses, query_params): + pass + +class TestNetImp(BlockChainInterface): + def __init__(self, rpcport = 18332, port = 8332): + super(TestNetImp, self).__init__() + self.command_params = ['bitcoin-cli', '-port='+str(port), '-rpcport='+str(rpcport),'-testnet'] #quick check that it's up else quit - res = self.rpc(['getbalance']) try: - self.current_balance = int(Decimal(res)) - print "Instantiated interface to regtest, wallet balance is: "+str(self.current_balance) +" bitcoins." - if not self.current_balance > 0: - raise Exception("Regtest network not properly initialised.") + res = self.rpc(['getbalance']) except Exception as e: print e - + + def get_net_info(self): + print 'not yet done' + def rpc(self, args, accept_failure=[]): try: + #print 'making an rpc call with these parameters: ' + #print self.command_params+args res = subprocess.check_output(self.command_params+args) except subprocess.CalledProcessError, e: if e.returncode in accept_failure: @@ -59,13 +118,14 @@ def rpc(self, args, accept_failure=[]): raise return res - def send_tx(self, tx_hex): - res = self.rpc(['sendrawtransaction', tx_hex]) - self.tick_forward_chain(1) - #TODO parse return string + def send_tx(self, tx_hexs, query_params): + '''csv params contains only tx hex''' + for txhex in tx_hexs: + res = self.rpc(['sendrawtransaction', txhex]) + #TODO only handles a single push; handle multiple return {'data':res} - def get_utxos_from_addr(self, addresses): + def get_utxos_from_addr(self, addresses, query_params): r = [] for address in addresses: res = json.loads(self.rpc(['listunspent','1','9999999','[\"'+address+'\"]'])) @@ -75,7 +135,7 @@ def get_utxos_from_addr(self, addresses): r.append({'address':address,'unspent':unspents}) return {'data':r} - def get_txs_from_addr(self, addresses): + def get_txs_from_addr(self, addresses, query_params): #use listtransactions and then filter #e.g.: -regtest listtransactions 'watchonly' 1000 0 true #to get the last 1000 transactions TODO 1000 is arbitrary @@ -83,7 +143,7 @@ def get_txs_from_addr(self, addresses): for address in addresses: if address not in acct_addrlist: self.rpc(['importaddress', address,'watchonly'],[4]) - res = json.loads(self.rpc(['listtransactions','watchonly','2','0','true'])) + res = json.loads(self.rpc(['listtransactions','watchonly','2000','0','true'])) result=[] for address in addresses: @@ -97,9 +157,11 @@ def get_txs_from_addr(self, addresses): result.append({'nb_txs':nbtxs,'address':address,'txs':txs}) return {'data':result} - def get_tx_info(self, txhash, raw=False): - res = json.loads(self.rpc(['gettransaction', txhash,'true'])) - if raw: + def get_tx_info(self, txhashes, query_params): + '''Returns a list of vouts if raw is False, else returns tx hex''' + #TODO: handle more than one tx hash + res = json.loads(self.rpc(['getrawtransaction', txhashes[0], '1'])) + if not query_params[0]: return {'data':{'tx':{'hex':res['hex']}}} tx = btc.deserialize(res['hex']) #build vout list @@ -111,13 +173,13 @@ def get_tx_info(self, txhash, raw=False): return {'data':{'vouts':vouts}} - def get_balance_at_addr(self, addresses): + def get_balance_at_addr(self, addresses, query_params): #NB This will NOT return coinbase coins (but wont matter in our use case). #In order to have the Bitcoin RPC read balances at addresses #it doesn't own, we must import the addresses as watch-only #Note that this is a 0.10 feature; won't work with older bitcoin clients. #TODO : there can be a performance issue with rescanning here. - + #TODO: This code is WRONG, reports *received* coins in total, not current balance. #allow importaddress to fail in case the address is already in the wallet res = [] for address in addresses: @@ -126,11 +188,42 @@ def get_balance_at_addr(self, addresses): int(Decimal(1e8) * Decimal(self.rpc(['getreceivedbyaddress', address])))}) return {'data':res} - def tick_forward_chain(self, n): - '''Special method for regtest only; - instruct to mine n blocks.''' - self.rpc(['setgenerate','true', str(n)]) + #Not used; I think, not needed + '''def get_addr_from_utxo(self, txhash, index): + #get the transaction details + res = json.loads(self.rpc(['gettxout', txhash, str(index)])) + amt = int(Decimal(1e8)*Decimal(res['value'])) + address = res('addresses')[0] + return (address, amt) + ''' + +#class for regtest chain access +#running on local daemon. Only +#to be instantiated after network is up +#with > 100 blocks. +class RegTestImp(TestNetImp): + def __init__(self, port=8331, rpcport=18331): + self.command_params = ['bitcoin-cli', '-port='+str(port), '-rpcport='+str(rpcport),'-regtest'] + #quick check that it's up else quit + try: + res = self.rpc(['getbalance']) + self.current_balance = int(Decimal(res)) + print "Instantiated interface to regtest, wallet balance is: "+str(self.current_balance) +" bitcoins." + if not self.current_balance > 0: + raise Exception("Regtest network not properly initialised.") + except Exception as e: + print e + + def send_tx(self, tx_hex): + super(RegTestImp, self).send_tx(tx_hex) + self.tick_forward_chain(1) + + def tick_forward_chain(self, n): + '''Special method for regtest only; + instruct to mine n blocks.''' + self.rpc(['setgenerate','true', str(n)]) + def grab_coins(self, receiving_addr, amt=50): ''' NOTE! amt is passed in Coins, not Satoshis! @@ -153,14 +246,7 @@ def grab_coins(self, receiving_addr, amt=50): raise Exception("Failed to broadcast transaction") #confirm self.tick_forward_chain(1) - return txid - - def get_addr_from_utxo(self, txhash, index): - #get the transaction details - res = json.loads(self.rpc(['gettxout', txhash, str(index)])) - amt = int(Decimal(1e8)*Decimal(res['value'])) - address = res('addresses')[0] - return (address, amt) + return txid def main(): myBCI = RegTestImp() diff --git a/common.py b/common.py index c01de30b..3813140d 100644 --- a/common.py +++ b/common.py @@ -5,21 +5,54 @@ import sys, datetime, json, time, pprint import threading import blockchaininterface - -HOST = 'irc.freenode.net' -CHANNEL = '#joinmarket-pit-test' -PORT = 6667 +from ConfigParser import SafeConfigParser +import os #for the mainnet its #joinmarket-pit nickname = '' COMMAND_PREFIX = '!' MAX_PRIVMSG_LEN = 400 -blockchain_source = 'blockr' +bc_interface = None ordername_list = ["absorder", "relorder"] encrypted_commands = ["auth", "ioauth", "tx", "sig"] plaintext_commands = ["fill", "error", "pubkey", "orderbook", "relorder", "absorder"] debug_file_handle = None +config = SafeConfigParser() +config_location = os.path.join(os.path.dirname(os.path.realpath(__file__)),'joinmarket.cfg') +required_options = {'BLOCKCHAIN':['blockchain_source','port','rpcport','network'], + 'MESSAGING':['host','channel','port']} + +def load_program_config(): + loadedFiles = config.read([config_location]) + #detailed sanity checking : + #did the file exist? + if len(loadedFiles) != 1: + raise Exception("Could not find config file: "+config_location) + #check for sections + for s in required_options: + if s not in config.sections(): + raise Exception("Config file does not contain the required section: "+s) + #then check for specific options + for k,v in required_options.iteritems(): + for o in v: + if o not in config.options(k): + raise Exception("Config file does not contain the required option: "+o) + + #configure the interface to the blockchain on startup + global bc_interface + source = config.get("BLOCKCHAIN","blockchain_source") + port = config.get("BLOCKCHAIN","port") + rpcport = config.get("BLOCKCHAIN","rpcport") + if source == 'testnet': + bc_interface = blockchaininterface.TestNetImp(rpcport=rpcport,port=port) + elif source == 'regtest': + bc_interface = blockchaininterface.RegTestImp(rpcport=rpcport,port=port) + elif source == 'blockr': + bc_interface = blockchaininterface.BlockrImp() + else: + raise ValueError("Invalid blockchain source") + def debug(msg): global debug_file_handle if nickname and not debug_file_handle: @@ -33,7 +66,11 @@ def chunks(d, n): return [d[x: x+n] for x in xrange(0, len(d), n)] def get_network(): - return 'testnet' + '''Returns network name as required by pybitcointools''' + if config.get("BLOCKCHAIN","network") == 'testnet': + return 'testnet' + else: + raise Exception("Only testnet is currently implemented") #TODO change this name into get_addr_ver() or something def get_addr_vbyte(): @@ -67,7 +104,7 @@ def get_addr_from_utxo(txhash, index): '''return the bitcoin address of the outpoint at the specified index for the transaction with specified hash. Return None if no such index existed for that transaction.''' - data = get_blockchain_data('txinfo', csv_params=[txhash]) + data = get_blockchain_data('txinfo', csv_params=[txhash], query_params=[True]) for a in data['vouts']: if a['n']==index: return a['address'] @@ -75,74 +112,11 @@ def get_addr_from_utxo(txhash, index): def get_blockchain_data(body, csv_params=[], query_params=[], network='test', output_key='data'): - '''A first step towards encapsulating blockchain queries.''' - if blockchain_source=='regtest': - stem = 'regtest:' - elif blockchain_source=='blockr': - stem = 'http://btc.blockr.io/api/v1/' - if network=='test': - stem = stem[:7]+'t'+stem[7:] - elif network != 'main': - raise Exception("unrecognised bitcoin network type") - else: - raise Exception("Unrecognised blockchain source") - - bodies = {'addrtx':'address/txs/','txinfo':'tx/info/','addrunspent':'address/unspent/', - 'addrbalance':'address/balance/','txraw':'tx/raw/','txpush':'tx/push/'} - url = stem + bodies[body] + ','.join(csv_params) - if query_params: - url += '?'+','.join(query_params) - if blockchain_source=='blockr': - res = json.loads(get_blockr_data(url)) - elif blockchain_source=='regtest': - res = get_regtest_data(url) - else: - raise Exception("Unrecognised blockchain source") - + #TODO: do we still need the 'network' parameter? Almost certainly not. + res = bc_interface.parse_request(body, csv_params, query_params) + if type(bc_interface) == blockchaininterface.BlockrImp: + res = json.loads(res) return res[output_key] - -def get_blockr_data(req): - if 'tx/push' in req: #manually parse this; special case because POST not GET? - return btc.blockr_pushtx(req.split('/')[-1], get_network()) - return btc.make_request(req) - -def get_regtest_data(req): - myBCI = blockchaininterface.RegTestImp() - if not req.startswith('regtest'): - raise Exception("Invalid request to regtest") - req = ''.join(req.split(':')[1:]).split('/') - if req[0]=='address' and req[1]=='txs': - addrs = req[2].split(',') - if '?' in addrs[-1]: - #TODO simulate dealing with unconfirmed - addrs[-1] = addrs[-1].split('?')[0] - #NB: we don't allow unconfirmeds in regtest - #for now; TODO - if 'unconfirmed' in addrs[-1]: - addrs = addrs[:-1] - return myBCI.get_txs_from_addr(addrs) - elif req[0]=='tx' and req[1]=='info': - txhash = req[2] #TODO currently only allowing one tx - return myBCI.get_tx_info(txhash) - elif req[0]=='address' and req[1] == 'balance': - addrs = req[2].split(',') - if '?' in addrs[-1]: - #TODO simulate dealing with unconfirmed - addrs[-1] = addrs[-1].split('?')[0] - return myBCI.get_balance_at_addr(addrs) - elif req[0]=='address' and req[1] == 'unspent': - if '?' in req[2]: req[2] = req[2].split('?')[0] - addrs = req[2].split(',') - return myBCI.get_utxos_from_addr(addrs) - elif req[0]=='tx' and req[1] == 'raw': - txhex = req[2] - return myBCI.get_tx_info(txhex, raw=True) - elif req[0]=='tx' and req[1] == 'push': - txraw = req[2] - return myBCI.send_tx(txraw) - else: - raise Exception ("Unrecognized call to regtest blockchain interface: " + '/'.join(req)) - class Wallet(object): def __init__(self, seed, max_mix_depth=2): @@ -286,7 +260,8 @@ def find_unspent_addresses(self): ''' addr_req_count = 20 - + #print 'working with index: ' + #print self.index #TODO handle the case where there are so many addresses it cant # fit into one api call (>50 or so) addrs = {} @@ -308,8 +283,13 @@ def find_unspent_addresses(self): #TODO send a pull request to pybitcointools # unspent() doesnt tell you which address, you get a bunch of utxos # but dont know which privkey to sign with + #print 'testing with addresses: ' + #print req data = get_blockchain_data('addrunspent', csv_params=req, query_params=['unconfirmed=1']) + #print 'got data: ' + #print data + if 'unspent' in data: data = [data] for dat in data: @@ -396,7 +376,7 @@ def calc_cj_fee(ordertype, cjfee, cj_amount): def calc_total_input_value(utxos): input_sum = 0 for utxo in utxos: - tx = get_blockchain_data('txraw', csv_params=[utxo[:64]])['tx']['hex'] + tx = get_blockchain_data('txraw', csv_params=[utxo[:64]],query_params=[False])['tx']['hex'] #tx = btc.blockr_fetchtx(utxo[:64], get_network()) input_sum += int(btc.deserialize(tx)['outs'][int(utxo[65:])]['value']) return input_sum diff --git a/irc.py b/irc.py index 6807261e..19b95d66 100644 --- a/irc.py +++ b/irc.py @@ -372,12 +372,12 @@ def __handle_line(self, line): self.motd_fd.close() ''' - def __init__(self, nick, server=HOST, port=PORT, channel=CHANNEL, username='username', realname='realname'): + def __init__(self, nick, username='username', realname='realname'): MessageChannel.__init__(self) self.cjpeer = None #subclasses have to set this to self self.nick = nick - self.serverport = (server, port) - self.channel = channel + self.serverport = (config.get("MESSAGING","host"), int(config.get("MESSAGING","port"))) + self.channel = '#'+ config.get("MESSAGING","channel") self.userrealname = (username, realname) def run(self): diff --git a/joinmarket.cfg b/joinmarket.cfg new file mode 100644 index 00000000..0c884b80 --- /dev/null +++ b/joinmarket.cfg @@ -0,0 +1,11 @@ +[BLOCKCHAIN] +blockchain_source = testnet +port = 8331 +rpcport = 18331 +network = testnet + +[MESSAGING] +host = irc.freenode.net +channel = joinmarket-pit-test +port = 6667 +#More stuff to come diff --git a/sendpayment.py b/sendpayment.py index 872bbe10..fc18d55e 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -87,7 +87,9 @@ def main(): seed = args[0] amount = int(args[1]) destaddr = args[2] - + + common.load_program_config() + import binascii, os common.nickname = 'payer-' +binascii.hexlify(os.urandom(4)) diff --git a/taker.py b/taker.py index 9ae679db..f14cf1dd 100644 --- a/taker.py +++ b/taker.py @@ -121,8 +121,7 @@ def add_signature(self, sigb64): for index, ins in enumerate(self.latest_tx['ins']): if ins['script'] != '': continue - ftx = get_blockchain_data('txraw', csv_params=[ins['outpoint']['hash']])['tx']['hex'] - #ftx = btc.blockr_fetchtx(ins['outpoint']['hash'], get_network()) + ftx = get_blockchain_data('txraw', csv_params=[ins['outpoint']['hash']], query_params=[False])['tx']['hex'] src_val = btc.deserialize(ftx)['outs'][ ins['outpoint']['index'] ] sig_good = btc.verify_tx_input(tx, index, src_val['script'], *btc.deserialize_script(sig)) if sig_good: @@ -146,7 +145,6 @@ def add_signature(self, sigb64): print btc.serialize(self.latest_tx) ret = get_blockchain_data('txpush', csv_params=[btc.serialize(self.latest_tx)]) - #ret = btc.blockr_pushtx(btc.serialize(self.latest_tx), get_network()) debug('pushed tx ' + str(ret)) if self.finishcallback != None: self.finishcallback() diff --git a/wallet-tool.py b/wallet-tool.py index ddc199d1..ef4dca5a 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -1,6 +1,6 @@ import bitcoin as btc -from common import Wallet, get_signed_tx +from common import Wallet, get_signed_tx, load_program_config import sys from optparse import OptionParser @@ -36,6 +36,8 @@ sys.exit(0) seed = args[0] +load_program_config() + method = ('display' if len(args) == 1 else args[1].lower()) #seed = '256 bits of randomness' diff --git a/yield-generator.py b/yield-generator.py index 3b44b58a..c1fa07a0 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -81,6 +81,7 @@ def on_tx_confirmed(self, cjorder, confirmations, txid, balance, added_utxos): return self.on_tx_unconfirmed(None, None, None) def main(): + common.load_program_config() import sys seed = sys.argv[1] #btc.sha256('dont use brainwallets except for holding testnet coins') From d832f08a55dfd6f71eeb702213ce851b3081cf38 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Sat, 28 Feb 2015 22:33:20 +0200 Subject: [PATCH 152/409] regtest fixup --- blockchaininterface.py | 5 +++-- common.py | 1 + regtesttest.py | 36 ++++++++++++++++++------------------ 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/blockchaininterface.py b/blockchaininterface.py index 5e182ad6..2a9a5426 100644 --- a/blockchaininterface.py +++ b/blockchaininterface.py @@ -203,6 +203,7 @@ def get_balance_at_addr(self, addresses, query_params): #with > 100 blocks. class RegTestImp(TestNetImp): def __init__(self, port=8331, rpcport=18331): + super(TestNetImp,self).__init__() #note: call to *grandparent* init for fptrs self.command_params = ['bitcoin-cli', '-port='+str(port), '-rpcport='+str(rpcport),'-regtest'] #quick check that it's up else quit try: @@ -215,8 +216,8 @@ def __init__(self, port=8331, rpcport=18331): print e - def send_tx(self, tx_hex): - super(RegTestImp, self).send_tx(tx_hex) + def send_tx(self, tx_hex, query_params): + super(RegTestImp, self).send_tx(tx_hex, query_params) self.tick_forward_chain(1) def tick_forward_chain(self, n): diff --git a/common.py b/common.py index 3813140d..1a4253c2 100644 --- a/common.py +++ b/common.py @@ -47,6 +47,7 @@ def load_program_config(): if source == 'testnet': bc_interface = blockchaininterface.TestNetImp(rpcport=rpcport,port=port) elif source == 'regtest': + print 'setting it' bc_interface = blockchaininterface.RegTestImp(rpcport=rpcport,port=port) elif source == 'blockr': bc_interface = blockchaininterface.BlockrImp() diff --git a/regtesttest.py b/regtesttest.py index 3e767954..1de906d8 100644 --- a/regtesttest.py +++ b/regtesttest.py @@ -1,8 +1,8 @@ import sys -import os +import os, time import subprocess import unittest -from common import * +import common from blockchaininterface import * import bitcoin as btc import binascii @@ -53,18 +53,17 @@ def setUp(self): #to allow that bot to start. seed1, seed2 = [binascii.hexlify(x) for x in [os.urandom(15), os.urandom(15)]] self.wallets = {} - wallet1 = Wallet(seed1) - wallet2 = Wallet(seed2) + wallet1 = common.Wallet(seed1) + wallet2 = common.Wallet(seed2) self.wallets[1] = {'seed':seed1,'wallet':wallet1} self.wallets[2] = {'seed':seed2,'wallet':wallet2} - bci = RegTestImp() #get first address in each wallet addr1 = wallet1.get_receive_addr(0) - debug("address for wallet1: "+addr1) + common.debug("address for wallet1: "+addr1) addr2 = wallet2.get_receive_addr(0) - debug("address for wallet2: "+addr2) - bci.grab_coins(addr1,10) - bci.grab_coins(addr2,10) + common.debug("address for wallet2: "+addr2) + common.bc_interface.grab_coins(addr1,10) + common.bc_interface.grab_coins(addr2,10) def run_simple_send(self, n): #start yield generator with wallet1 @@ -76,7 +75,7 @@ def run_simple_send(self, n): #run a single sendpayment call with wallet2 amt = 100000000 #in satoshis - dest_address = btc.privkey_to_address(os.urandom(32), get_addr_vbyte()) + dest_address = btc.privkey_to_address(os.urandom(32), common.get_addr_vbyte()) try: for i in range(n): sp_proc = local_command(['python','sendpayment.py','-N','1', self.wallets[2]['seed'],\ @@ -96,8 +95,7 @@ def run_simple_send(self, n): # with open(cf, 'rb') as f: # if 'CRASHING' in f.read(): return False - myBCI = blockchaininterface.RegTestImp() - received = myBCI.get_balance_at_addr([dest_address])['data'][0]['balance'] + received = common.bc_interface.get_balance_at_addr([dest_address], None)['data'][0]['balance'] if received != amt: return False return True @@ -118,12 +116,11 @@ def setUp(self): seeds = [binascii.hexlify(''.join(x)) for x in seeds] self.wallets = {} for i, seed in enumerate(seeds): - self.wallets[i] = {'seed':seed, 'wallet':Wallet(seed)} + self.wallets[i] = {'seed':seed, 'wallet':common.Wallet(seed)} - bci = RegTestImp() #get first address in each wallet for i in self.wallets.keys(): - bci.grab_coins(self.wallets[i]['wallet'].get_receive_addr(0), amt=10) + common.bc_interface.grab_coins(self.wallets[i]['wallet'].get_receive_addr(0), amt=10) #the sender is wallet (n+1), i.e. index wallets[n] @@ -144,7 +141,7 @@ def run_nparty_join(self): #run a single sendpayment call amt = 100000000 #in satoshis - dest_address = btc.privkey_to_address(os.urandom(32), get_addr_vbyte()) + dest_address = btc.privkey_to_address(os.urandom(32), common.get_addr_vbyte()) try: sp_proc = local_command(['python','sendpayment.py','-N', str(self.n),\ self.wallets[self.n]['seed'], str(amt), dest_address]) @@ -166,14 +163,17 @@ def run_nparty_join(self): #with open(cf, 'rb') as f: # if 'CRASHING' in f.read(): return False - myBCI = blockchaininterface.RegTestImp() - received = myBCI.get_balance_at_addr([dest_address])['data'][0]['balance'] + received = common.bc_interface.get_balance_at_addr([dest_address], None)['data'][0]['balance'] if received != amt: return False return True def main(): + common.load_program_config() + if not common.bc_interface: + print 'not there' + exit() unittest.main() if __name__ == '__main__': From 38b20106785de0efa00d6c19a91f67d03dec52ef Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Tue, 3 Mar 2015 17:12:52 +0200 Subject: [PATCH 153/409] bugfixes --- blockchaininterface.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/blockchaininterface.py b/blockchaininterface.py index 2a9a5426..b2ea50b3 100644 --- a/blockchaininterface.py +++ b/blockchaininterface.py @@ -69,9 +69,11 @@ def __init__(self, testnet = True): self.query_stem = 'http://tbtc.blockr.io/api/v1/' if testnet else 'http://tbtc.blockr.io/api/v1/' def parse_request(self, body, csv_params, query_params=None): - if body=='pushtx': + if body=='txpush': return super(BlockrImp, self).parse_request(body, csv_params, query_params) else: + if body == 'txinfo' or body == 'txraw': + query_params = query_params[1:] req = self.query_stem + self.bodies[body] + '/' + ','.join(csv_params) + '?' + ','.join(query_params) return btc.make_request(req) @@ -158,7 +160,7 @@ def get_txs_from_addr(self, addresses, query_params): return {'data':result} def get_tx_info(self, txhashes, query_params): - '''Returns a list of vouts if raw is False, else returns tx hex''' + '''Returns a list of vouts if first entry in query params is False, else returns tx hex''' #TODO: handle more than one tx hash res = json.loads(self.rpc(['getrawtransaction', txhashes[0], '1'])) if not query_params[0]: From e0000ca099ad897b1e9a59d5ba14a180c4efe4f0 Mon Sep 17 00:00:00 2001 From: Belcher Date: Mon, 9 Mar 2015 21:25:02 +0000 Subject: [PATCH 154/409] added bitcoin-cli-cmd to config file, renamed some classes, moved function --- blockchaininterface.py | 61 +++++++++++++++++++++++++++--------------- common.py | 17 +++--------- joinmarket.cfg | 6 ++--- 3 files changed, 45 insertions(+), 39 deletions(-) diff --git a/blockchaininterface.py b/blockchaininterface.py index b2ea50b3..baa58ea0 100644 --- a/blockchaininterface.py +++ b/blockchaininterface.py @@ -6,6 +6,22 @@ from decimal import Decimal import bitcoin as btc +import common + +def get_blockchain_interface_instance(config): + source = config.get("BLOCKCHAIN", "blockchain_source") + bitcoin_cli_cmd = config.get("BLOCKCHAIN", "bitcoin_cli_cmd").split(' ') + testnet = common.get_network()=='testnet' + if source == 'json-rpc': + bc_interface = BitcoinCoreInterface(bitcoin_cli_cmd, testnet) + elif source == 'regtest': + bc_interface = RegtestBitcoinCoreInterface(bitcoin_cli_cmd) + elif source == 'blockr': + bc_interface = BlockrInterface(testnet) + else: + raise ValueError("Invalid blockchain source") + return bc_interface + class BlockChainInterface(object): __metaclass__ = abc.ABCMeta @@ -60,9 +76,9 @@ def get_addr_from_utxo(self, txhash, index): pass ''' -class BlockrImp(BlockChainInterface): - def __init__(self, testnet = True): - super(BlockrImp, self).__init__() +class BlockrInterface(BlockChainInterface): + def __init__(self, testnet = False): + super(BlockrInterface, self).__init__() self.bodies = {'addrtx':'address/txs/','txinfo':'tx/info/','addrunspent':'address/unspent/', 'addrbalance':'address/balance/','txraw':'tx/raw/','txpush':'tx/push/'} self.testnet = 'testnet' if testnet else 'btc' #see bci.py in bitcoin module @@ -70,7 +86,7 @@ def __init__(self, testnet = True): def parse_request(self, body, csv_params, query_params=None): if body=='txpush': - return super(BlockrImp, self).parse_request(body, csv_params, query_params) + return super(BlockrInterface, self).parse_request(body, csv_params, query_params) else: if body == 'txinfo' or body == 'txraw': query_params = query_params[1:] @@ -96,10 +112,13 @@ def get_utxos_from_addr(self, addresses, query_params): def get_balance_at_addr(self, addresses, query_params): pass -class TestNetImp(BlockChainInterface): - def __init__(self, rpcport = 18332, port = 8332): - super(TestNetImp, self).__init__() - self.command_params = ['bitcoin-cli', '-port='+str(port), '-rpcport='+str(rpcport),'-testnet'] +class BitcoinCoreInterface(BlockChainInterface): + def __init__(self, bitcoin_cli_cmd, testnet=False): + super(BitcoinCoreInterface, self).__init__() + #self.command_params = ['bitcoin-cli', '-port='+str(port), '-rpcport='+str(rpcport),'-testnet'] + self.command_params = bitcoin_cli_cmd + if testnet: + self.command_params += ['-testnet'] #quick check that it's up else quit try: res = self.rpc(['getbalance']) @@ -112,7 +131,7 @@ def get_net_info(self): def rpc(self, args, accept_failure=[]): try: #print 'making an rpc call with these parameters: ' - #print self.command_params+args + common.debug(str(self.command_params+args)) res = subprocess.check_output(self.command_params+args) except subprocess.CalledProcessError, e: if e.returncode in accept_failure: @@ -144,7 +163,7 @@ def get_txs_from_addr(self, addresses, query_params): acct_addrlist = self.rpc(['getaddressesbyaccount', 'watchonly']) for address in addresses: if address not in acct_addrlist: - self.rpc(['importaddress', address,'watchonly'],[4]) + self.rpc(['importaddress', address, 'watchonly'],[4]) res = json.loads(self.rpc(['listtransactions','watchonly','2000','0','true'])) result=[] @@ -203,23 +222,19 @@ def get_balance_at_addr(self, addresses, query_params): #running on local daemon. Only #to be instantiated after network is up #with > 100 blocks. -class RegTestImp(TestNetImp): - def __init__(self, port=8331, rpcport=18331): - super(TestNetImp,self).__init__() #note: call to *grandparent* init for fptrs - self.command_params = ['bitcoin-cli', '-port='+str(port), '-rpcport='+str(rpcport),'-regtest'] +class RegtestBitcoinCoreInterface(BitcoinCoreInterface): + def __init__(self, bitcoin_cli_cmd): + super(BitcoinCoreInterface, self).__init__() + #self.command_params = ['bitcoin-cli', '-port='+str(port), '-rpcport='+str(rpcport),'-testnet'] + self.command_params = bitcoin_cli_cmd + ['-regtest'] #quick check that it's up else quit try: res = self.rpc(['getbalance']) - self.current_balance = int(Decimal(res)) - print "Instantiated interface to regtest, wallet balance is: "+str(self.current_balance) +" bitcoins." - if not self.current_balance > 0: - raise Exception("Regtest network not properly initialised.") except Exception as e: - print e - + print e def send_tx(self, tx_hex, query_params): - super(RegTestImp, self).send_tx(tx_hex, query_params) + super(RegtestBitcoinCoreInterface, self).send_tx(tx_hex, query_params) self.tick_forward_chain(1) def tick_forward_chain(self, n): @@ -237,12 +252,14 @@ def grab_coins(self, receiving_addr, amt=50): ''' if amt > 500: raise Exception("too greedy") + ''' if amt > self.current_balance: #mine enough to get to the reqd amt reqd = int(amt - self.current_balance) reqd_blocks = str(int(reqd/50) +1) if self.rpc(['setgenerate','true', reqd_blocks]): raise Exception("Something went wrong") + ''' #now we do a custom create transaction and push to the receiver txid = self.rpc(['sendtoaddress', receiving_addr, str(amt)]) if not txid: @@ -252,7 +269,7 @@ def grab_coins(self, receiving_addr, amt=50): return txid def main(): - myBCI = RegTestImp() + myBCI = RegtestBitcoinCoreInterface() #myBCI.send_tx('stuff') print myBCI.get_utxos_from_addr(["n4EjHhGVS4Rod8ociyviR3FH442XYMWweD"]) print myBCI.get_balance_at_addr(["n4EjHhGVS4Rod8ociyviR3FH442XYMWweD"]) diff --git a/common.py b/common.py index 26aaf322..3017a9ed 100644 --- a/common.py +++ b/common.py @@ -18,7 +18,7 @@ config = SafeConfigParser() config_location = os.path.join(os.path.dirname(os.path.realpath(__file__)),'joinmarket.cfg') -required_options = {'BLOCKCHAIN':['blockchain_source','port','rpcport','network'], +required_options = {'BLOCKCHAIN':['blockchain_source', 'network', 'bitcoin_cli_cmd'], 'MESSAGING':['host','channel','port']} def load_program_config(): @@ -39,18 +39,7 @@ def load_program_config(): #configure the interface to the blockchain on startup global bc_interface - source = config.get("BLOCKCHAIN","blockchain_source") - port = config.get("BLOCKCHAIN","port") - rpcport = config.get("BLOCKCHAIN","rpcport") - if source == 'testnet': - bc_interface = blockchaininterface.TestNetImp(rpcport=rpcport,port=port) - elif source == 'regtest': - print 'setting it' - bc_interface = blockchaininterface.RegTestImp(rpcport=rpcport,port=port) - elif source == 'blockr': - bc_interface = blockchaininterface.BlockrImp() - else: - raise ValueError("Invalid blockchain source") + bc_interface = blockchaininterface.get_blockchain_interface_instance(config) def debug(msg): global debug_file_handle @@ -113,7 +102,7 @@ def get_blockchain_data(body, csv_params=[], query_params=[], network='test', output_key='data'): #TODO: do we still need the 'network' parameter? Almost certainly not. res = bc_interface.parse_request(body, csv_params, query_params) - if type(bc_interface) == blockchaininterface.BlockrImp: + if type(bc_interface) == blockchaininterface.BlockrInterface: res = json.loads(res) return res[output_key] diff --git a/joinmarket.cfg b/joinmarket.cfg index 7b4b9e23..c0faa4d2 100644 --- a/joinmarket.cfg +++ b/joinmarket.cfg @@ -1,11 +1,11 @@ [BLOCKCHAIN] blockchain_source = blockr -port = 8331 -rpcport = 18331 +#options: blockr, json-rpc, regtest network = testnet +bitcoin_cli_cmd = bitcoin-cli [MESSAGING] host = irc.freenode.net channel = joinmarket-pit-test port = 6667 -#More stuff to come \ No newline at end of file +#More stuff to come From 076531798cc29979ea3dd0d565158c3b1c9b053c Mon Sep 17 00:00:00 2001 From: Belcher Date: Tue, 10 Mar 2015 21:08:58 +0000 Subject: [PATCH 155/409] added load_program_config() to guitaker --- gui-taker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gui-taker.py b/gui-taker.py index 6c2b14fe..da13cf23 100644 --- a/gui-taker.py +++ b/gui-taker.py @@ -162,7 +162,8 @@ def main(): import bitcoin as btc import common import binascii, os - common.nickname = 'guitaker-' +binascii.hexlify(os.urandom(4)) + common.nickname = 'guitaker-' +binascii.hexlify(os.urandom(4)) + common.load_program_config() irc = IRCMessageChannel(common.nickname) taker = GUITaker(irc) From eb9efb3fdcd3b5344bd3107ecd62efff48377cf4 Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 13 Mar 2015 02:12:20 +0000 Subject: [PATCH 156/409] started recoding BlockchainInterface --- blockchaininterface.py | 280 ++++++++++++++++++++++++++++------------- common.py | 162 +----------------------- irc.py | 12 +- maker.py | 15 +-- sendpayment.py | 3 +- taker.py | 9 +- wallet-tool.py | 6 +- yield-generator.py | 8 +- 8 files changed, 226 insertions(+), 269 deletions(-) diff --git a/blockchaininterface.py b/blockchaininterface.py index baa58ea0..c0b1d74b 100644 --- a/blockchaininterface.py +++ b/blockchaininterface.py @@ -1,8 +1,7 @@ #from joinmarket import * import subprocess import unittest -import json -import abc +import json, threading, abc, pprint, time, random from decimal import Decimal import bitcoin as btc @@ -23,96 +22,205 @@ def get_blockchain_interface_instance(config): return bc_interface -class BlockChainInterface(object): - __metaclass__ = abc.ABCMeta - def __init__(self): - #TODO: pass in network type (main/test) - self.fptrs = {'addrtx': self.get_txs_from_addr, - 'txinfo': self.get_tx_info, - 'addrunspent': self.get_utxos_from_addr, - 'addrbalance': self.get_balance_at_addr, - 'txraw': self.get_tx_info, - 'txpush': self.send_tx} - - def parse_request(self, body, csv_params, query_params=None): - return self.fptrs[body](csv_params, query_params) - - @abc.abstractmethod - def get_txs_from_addr(self, addresses, query_params): - '''Given a list of addresses, list all transactions''' - pass - - @abc.abstractmethod - def get_tx_info(self, txhash, query_params): - '''Given a txhash and query params indicating raw, - return the tx hex. If indicating non-raw, return a list of vouts. - May need some more structure in query_params to handle unconfirmed. TODO''' - pass - - @abc.abstractmethod - def get_utxos_from_addr(self, addresses, query_params): - '''Given an address, return a list of utxos - in format txid:vout''' - pass - - @abc.abstractmethod - def get_balance_at_addr(self, addresses, query_params): - '''Given an address, return a balance in satoshis''' - pass +#download_wallet_history() find_unspent_addresses() #finding where to put index and my utxos +#add address notify() +#fetchtx() needs to accept a list of addresses too +#pushtx() +class BlockchainInterface(object): + __metaclass__ = abc.ABCMeta + def __init__(self): + pass - @abc.abstractmethod - def send_tx(self, tx_hexs, query_params): - '''Given raw txhex, push to network and return result in form: TODO''' - pass - - @abc.abstractmethod - def get_net_info(self): - pass - ''' - @abc.abstractmethod - def get_addr_from_utxo(self, txhash, index): - Given utxo in form txhash, index, return the address - owning the utxo and the amount in satoshis in form (addr, amt) - pass - ''' + @abc.abstractmethod + def sync_wallet(self, wallet, gaplimit=6): + '''Finds used addresses and utxos, puts in wallet.index and wallet.unspent''' + pass -class BlockrInterface(BlockChainInterface): - def __init__(self, testnet = False): - super(BlockrInterface, self).__init__() - self.bodies = {'addrtx':'address/txs/','txinfo':'tx/info/','addrunspent':'address/unspent/', - 'addrbalance':'address/balance/','txraw':'tx/raw/','txpush':'tx/push/'} - self.testnet = 'testnet' if testnet else 'btc' #see bci.py in bitcoin module - self.query_stem = 'http://tbtc.blockr.io/api/v1/' if testnet else 'http://tbtc.blockr.io/api/v1/' - - def parse_request(self, body, csv_params, query_params=None): - if body=='txpush': - return super(BlockrInterface, self).parse_request(body, csv_params, query_params) - else: - if body == 'txinfo' or body == 'txraw': - query_params = query_params[1:] - req = self.query_stem + self.bodies[body] + '/' + ','.join(csv_params) + '?' + ','.join(query_params) - return btc.make_request(req) - - def send_tx(self, tx_hexs, query_params): - #TODO: handle multiple txs? - return btc.blockr_pushtx(tx_hexs[0], self.testnet) + @abc.abstractmethod + def add_tx_notify(self, tx, unconfirmfun, confirmfun): + '''Invokes unconfirmfun and confirmfun when tx is seen on the network''' + pass - def get_net_info(self): - print 'not yet done' + @abc.abstractmethod + def fetchtx(self, txid): + '''Returns a txhash of a given txid, or list of txids''' + pass - def get_txs_from_addr(self, addresses, query_params): - pass - - def get_tx_info(self, txhash, query_params): - pass - - def get_utxos_from_addr(self, addresses, query_params): - pass + @abc.abstractmethod + def pushtx(self, txhash): + '''pushes tx to the network, returns txhash''' + pass + +class BlockrInterface(BlockchainInterface): + def __init__(self, testnet = False): + super(BlockrInterface, self).__init__() + self.network = 'testnet' if testnet else 'btc' #see bci.py in bitcoin module + self.blockr_domain = 'tbtc' if testnet else 'btc' - def get_balance_at_addr(self, addresses, query_params): - pass + def sync_wallet(self, wallet, gaplimit=6): + common.debug('downloading wallet history') + #sets Wallet internal indexes to be at the next unused address + addr_req_count = 20 + for mix_depth in range(wallet.max_mix_depth): + for forchange in [0, 1]: + unused_addr_count = 0 + last_used_addr = '' + while unused_addr_count < gaplimit: + addrs = [wallet.get_new_addr(mix_depth, forchange) for i in range(addr_req_count)] + + #TODO send a pull request to pybitcointools + # because this surely should be possible with a function from it + blockr_url = 'http://' + self.blockr_domain + '.blockr.io/api/v1/address/txs/' + res = btc.make_request(blockr_url+','.join(addrs)) + data = json.loads(res)['data'] + for dat in data: + if dat['nb_txs'] != 0: + last_used_addr = dat['address'] + else: + unused_addr_count += 1 + if unused_addr_count >= gaplimit: + break + if last_used_addr == '': + wallet.index[mix_depth][forchange] = 0 + else: + wallet.index[mix_depth][forchange] = wallet.addr_cache[last_used_addr][2] + 1 + + #finds utxos in the wallet + + addrs = {} + for m in range(wallet.max_mix_depth): + for forchange in [0, 1]: + for n in range(wallet.index[m][forchange]): + addrs[wallet.get_addr(m, forchange, n)] = m + if len(addrs) == 0: + common.debug('no tx used') + return + + #TODO handle the case where there are so many addresses it cant + # fit into one api call (>50 or so) + i = 0 + addrkeys = addrs.keys() + while i < len(addrkeys): + inc = min(len(addrkeys) - i, addr_req_count) + req = addrkeys[i:i + inc] + i += inc + + #TODO send a pull request to pybitcointools + # unspent() doesnt tell you which address, you get a bunch of utxos + # but dont know which privkey to sign with + + blockr_url = 'http://' + self.blockr_domain + '.blockr.io/api/v1/address/unspent/' + res = btc.make_request(blockr_url+','.join(req)) + data = json.loads(res)['data'] + if 'unspent' in data: + data = [data] + for dat in data: + for u in dat['unspent']: + wallet.unspent[u['tx']+':'+str(u['n'])] = {'address': + dat['address'], 'value': int(u['amount'].replace('.', ''))} + + def add_tx_notify(self, txd, unconfirmfun, confirmfun): + unconfirm_timeout = 5*60 #seconds + unconfirm_poll_period = 5 + confirm_timeout = 120*60 + confirm_poll_period = 5*60 + class NotifyThread(threading.Thread): + def __init__(self, blockr_domain, txd, unconfirmfun, confirmfun): + threading.Thread.__init__(self) + self.daemon = True + self.blockr_domain = blockr_domain + self.unconfirmfun = unconfirmfun + self.confirmfun = confirmfun + self.tx_output_set = set([(sv['script'], sv['value']) for sv in txd['outs']]) + self.output_addresses = [btc.script_to_address(scrval[0], + common.get_addr_vbyte()) for scrval in self.tx_output_set] + common.debug('txoutset=' + pprint.pformat(self.tx_output_set)) + common.debug('outaddrs=' + ','.join(self.output_addresses)) + + def run(self): + st = int(time.time()) + unconfirmed_txid = None + unconfirmed_txhex = None + while not unconfirmed_txid: + time.sleep(unconfirm_poll_period) + if int(time.time()) - st > unconfirm_timeout: + debug('checking for unconfirmed tx timed out') + return + blockr_url = 'http://' + self.blockr_domain + '.blockr.io/api/v1/address/unspent/' + random.shuffle(self.output_addresses) #seriously weird bug with blockr.io + data = json.loads(btc.make_request(blockr_url + ','.join(self.output_addresses) + '?unconfirmed=1'))['data'] + shared_txid = None + for unspent_list in data: + txs = set([str(txdata['tx']) for txdata in unspent_list['unspent']]) + if not shared_txid: + shared_txid = txs + else: + shared_txid = shared_txid.intersection(txs) + common.debug('sharedtxid = ' + str(shared_txid)) + if len(shared_txid) == 0: + continue + blockr_url = 'http://' + self.blockr_domain + '.blockr.io/api/v1/tx/raw/' + data = json.loads(btc.make_request(blockr_url + ','.join(shared_txid)))['data'] + if not isinstance(data, list): + data = [data] + for txinfo in data: + outs = set([(sv['script'], sv['value']) for sv in btc.deserialize(txinfo['tx']['hex'])['outs']]) + print 'outs = ' + str(outs) + if outs == self.tx_output_set: + unconfirmed_txid = txinfo['tx']['txid'] + unconfirmed_txhex = txinfo['tx']['hex'] + break + + self.unconfirmfun(btc.deserialize(unconfirmed_txhex), unconfirmed_txid) + + st = int(time.time()) + confirmed_txid = None + confirmed_txhex = None + while not confirmed_txid: + time.sleep(confirm_poll_period) + if int(time.time()) - st > confirm_timeout: + debug('checking for confirmed tx timed out') + return + blockr_url = 'http://' + self.blockr_domain + '.blockr.io/api/v1/address/txs/' + data = json.loads(btc.make_request(blockr_url + ','.join(self.output_addresses)))['data'] + shared_txid = None + for addrtxs in data: + txs = set([str(txdata['tx']) for txdata in addrtxs['txs']]) + if not shared_txid: + shared_txid = txs + else: + shared_txid = shared_txid.intersection(txs) + common.debug('sharedtxid = ' + str(shared_txid)) + if len(shared_txid) == 0: + continue + blockr_url = 'http://' + self.blockr_domain + '.blockr.io/api/v1/tx/raw/' + data = json.loads(btc.make_request(blockr_url + ','.join(shared_txid)))['data'] + if not isinstance(data, list): + data = [data] + for txinfo in data: + outs = set([(sv['script'], sv['value']) for sv in btc.deserialize(txinfo['tx']['hex'])['outs']]) + print 'outs = ' + str(outs) + if outs == self.tx_output_set: + confirmed_txid = txinfo['tx']['txid'] + confirmed_txhex = txinfo['tx']['hex'] + break + self.confirmfun(btc.deserialize(confirmed_txhex), confirmed_txid, 1) + + NotifyThread(self.blockr_domain, txd, unconfirmfun, confirmfun).start() + + def fetchtx(self, txid): + return btc.blockr_fetchtx(txid, self.network) + + def pushtx(self, txhex): + data = json.loads(btc.blockr_pushtx(txhex, self.network)) + if data['status'] != 'success': + #error message generally useless so there might not be a point returning + debug(data) + return None + return data['data'] + -class BitcoinCoreInterface(BlockChainInterface): +class BitcoinCoreInterface(BlockchainInterface): def __init__(self, bitcoin_cli_cmd, testnet=False): super(BitcoinCoreInterface, self).__init__() #self.command_params = ['bitcoin-cli', '-port='+str(port), '-rpcport='+str(rpcport),'-testnet'] diff --git a/common.py b/common.py index 3017a9ed..74b00b71 100644 --- a/common.py +++ b/common.py @@ -92,19 +92,7 @@ def get_addr_from_utxo(txhash, index): '''return the bitcoin address of the outpoint at the specified index for the transaction with specified hash. Return None if no such index existed for that transaction.''' - data = get_blockchain_data('txinfo', csv_params=[txhash], query_params=[True]) - for a in data['vouts']: - if a['n']==index: - return a['address'] - return None - -def get_blockchain_data(body, csv_params=[], - query_params=[], network='test', output_key='data'): - #TODO: do we still need the 'network' parameter? Almost certainly not. - res = bc_interface.parse_request(body, csv_params, query_params) - if type(bc_interface) == blockchaininterface.BlockrInterface: - res = json.loads(res) - return res[output_key] + return btc.script_to_address(btc.deserialize(bc_interface.fetchtx(txhash))['outs'][index]['script'], get_addr_vbyte()) class Wallet(object): def __init__(self, seed, max_mix_depth=2): @@ -206,151 +194,6 @@ def select_utxos(self, mixdepth, amount): debug(pprint.pformat(inputs)) return [i['utxo'] for i in inputs] - def sync_wallet(self, gaplimit=6): - debug('synchronizing wallet') - self.download_wallet_history(gaplimit) - self.find_unspent_addresses() - self.print_debug_wallet_info() - - def download_wallet_history(self, gaplimit=6): - ''' - sets Wallet internal indexes to be at the next unused address - ''' - addr_req_count = 20 - - for mix_depth in range(self.max_mix_depth): - for forchange in [0, 1]: - unused_addr_count = 0 - last_used_addr = '' - while unused_addr_count < gaplimit: - addrs = [self.get_new_addr(mix_depth, forchange) for i in range(addr_req_count)] - - #TODO send a pull request to pybitcointools - # because this surely should be possible with a function from it - data = get_blockchain_data('addrtx', csv_params=addrs) - for dat in data: - if dat['nb_txs'] != 0: - last_used_addr = dat['address'] - else: - unused_addr_count += 1 - if unused_addr_count >= gaplimit: - break - if last_used_addr == '': - self.index[mix_depth][forchange] = 0 - else: - self.index[mix_depth][forchange] = self.addr_cache[last_used_addr][2] + 1 - - def find_unspent_addresses(self): - ''' - finds utxos in the wallet - assumes you've already called download_wallet_history() so - you know which addresses have been used - ''' - - addr_req_count = 20 - #print 'working with index: ' - #print self.index - #TODO handle the case where there are so many addresses it cant - # fit into one api call (>50 or so) - addrs = {} - for m in range(self.max_mix_depth): - for forchange in [0, 1]: - for n in range(self.index[m][forchange]): - addrs[self.get_addr(m, forchange, n)] = m - if len(addrs) == 0: - debug('no tx used') - return - - i = 0 - addrkeys = addrs.keys() - while i < len(addrkeys): - inc = min(len(addrkeys) - i, addr_req_count) - req = addrkeys[i:i + inc] - i += inc - - #TODO send a pull request to pybitcointools - # unspent() doesnt tell you which address, you get a bunch of utxos - # but dont know which privkey to sign with - #print 'testing with addresses: ' - #print req - data = get_blockchain_data('addrunspent', csv_params=req, - query_params=['unconfirmed=1']) - #print 'got data: ' - #print data - - if 'unspent' in data: - data = [data] - for dat in data: - for u in dat['unspent']: - if u['confirmations'] != 0: - u['amount'] = int(Decimal(1e8) * Decimal(u['amount'])) - self.unspent[u['tx']+':'+str(u['n'])] = {'address': - dat['address'], 'value': u['amount']} - - def print_debug_wallet_info(self): - debug('printing debug wallet information') - debug('utxos') - debug(pprint.pformat(self.unspent)) - debug('wallet.index') - debug(pprint.pformat(self.index)) - - -#awful way of doing this, but works for now -# and -walletnotify for people who do -#timeouts in minutes -def add_addr_notify(address, unconfirmfun, confirmfun, unconfirmtimeout=5, - unconfirmtimeoutfun=None, confirmtimeout=120, confirmtimeoutfun=None): - - class NotifyThread(threading.Thread): - def __init__(self, address, unconfirmfun, confirmfun, unconfirmtimeout, - unconfirmtimeoutfun, confirmtimeout, confirmtimeoutfun): - threading.Thread.__init__(self) - self.daemon = True - self.address = address - self.unconfirmfun = unconfirmfun - self.confirmfun = confirmfun - self.unconfirmtimeout = unconfirmtimeout*60 - self.unconfirmtimeoutfun = unconfirmtimeoutfun - self.confirmtimeout = confirmtimeout*60 - self.confirmtimeoutfun = confirmtimeoutfun - - def run(self): - st = int(time.time()) - while True: - time.sleep(5) - if int(time.time()) - st > self.unconfirmtimeout: - if unconfirmtimeoutfun != None: - unconfirmtimeoutfun() - debug('checking for unconfirmed tx timed out') - return - data = get_blockchain_data('addrbalance', csv_params=[self.address], - query_params=['confirmations=0']) - if type(data) == list: - data = data[0] #needed because blockr's json structure is inconsistent - if data['balance'] > 0: - break - self.unconfirmfun(data['balance']*1e8) - st = int(time.time()) - while True: - time.sleep(5 * 60) - if int(time.time()) - st > self.confirmtimeout: - if confirmtimeoutfun != None: - confirmtimeoutfun() - debug('checking for confirmed tx timed out') - return - data = get_blockchain_data('addrtx', csv_params=[self.address], - query_params=['confirmations=0']) - if data['nb_txs'] == 0: - continue - if data['txs'][0]['confirmations'] >= 1: #confirmation threshold - break - self.confirmfun(data['txs'][0]['confirmations'], - data['txs'][0]['tx'], data['txs'][0]['amount']*1e8) - - NotifyThread(address, unconfirmfun, confirmfun, unconfirmtimeout, - unconfirmtimeoutfun, confirmtimeout, confirmtimeoutfun).start() - - def calc_cj_fee(ordertype, cjfee, cj_amount): real_cjfee = None if ordertype == 'absorder': @@ -361,11 +204,12 @@ def calc_cj_fee(ordertype, cjfee, cj_amount): raise RuntimeError('unknown order type: ' + str(ordertype)) return real_cjfee +#TODO this function is used once, it has no point existing def calc_total_input_value(utxos): input_sum = 0 for utxo in utxos: - tx = get_blockchain_data('txraw', csv_params=[utxo[:64]],query_params=[False])['tx']['hex'] #tx = btc.blockr_fetchtx(utxo[:64], get_network()) + tx = bc_interface.fetchtx(utxo[:64]) input_sum += int(btc.deserialize(tx)['outs'][int(utxo[65:])]['value']) return input_sum diff --git a/irc.py b/irc.py index 19b95d66..e8bc240e 100644 --- a/irc.py +++ b/irc.py @@ -201,26 +201,26 @@ def __on_privmsg(self, nick, message): oid = int(chunks[1]) amount = int(chunks[2]) taker_pk = chunks[3] - if self.on_order_fill: - self.on_order_fill(nick, oid, amount, taker_pk) except (ValueError, IndexError) as e: self.send_error(nick, str(e)) + if self.on_order_fill: + self.on_order_fill(nick, oid, amount, taker_pk) elif chunks[0] == 'auth': try: i_utxo_pubkey = chunks[1] btc_sig = chunks[2] - if self.on_seen_auth: - self.on_seen_auth(nick, i_utxo_pubkey, btc_sig) except (ValueError, IndexError) as e: self.send_error(nick, str(e)) + if self.on_seen_auth: + self.on_seen_auth(nick, i_utxo_pubkey, btc_sig) elif chunks[0] == 'tx': b64tx = chunks[1] try: txhex = base64.b64decode(b64tx).encode('hex') - if self.on_seen_tx: - self.on_seen_tx(nick, txhex) except TypeError as e: self.send_error(nick, 'bad base64 tx. ' + repr(e)) + if self.on_seen_tx: + self.on_seen_tx(nick, txhex) except CJPeerError: #TODO proper error handling continue diff --git a/maker.py b/maker.py index 153f6034..fc6b2019 100644 --- a/maker.py +++ b/maker.py @@ -1,6 +1,7 @@ #! /usr/bin/env python from common import * +import common from taker import CoinJoinerPeer import bitcoin as btc import base64, pprint, threading @@ -73,22 +74,22 @@ def recv_tx(self, nick, txhex): sigs.append(base64.b64encode(btc.deserialize(txs)['ins'][index]['script'].decode('hex'))) #len(sigs) > 0 guarenteed since i did verify_unsigned_tx() - add_addr_notify(self.change_addr, self.unconfirm_callback, self.confirm_callback) + common.bc_interface.add_tx_notify(self.tx, self.unconfirm_callback, self.confirm_callback) debug('sending sigs ' + str(sigs)) self.maker.msgchan.send_sigs(nick, sigs) self.maker.active_orders[nick] = None - def unconfirm_callback(self, balance): + def unconfirm_callback(self, txd, txid): self.maker.wallet_unspent_lock.acquire() try: removed_utxos = self.maker.wallet.remove_old_utxos(self.tx) finally: self.maker.wallet_unspent_lock.release() debug('saw tx on network, removed_utxos=\n' + pprint.pformat(removed_utxos)) - to_cancel, to_announce = self.maker.on_tx_unconfirmed(self, balance, removed_utxos) + to_cancel, to_announce = self.maker.on_tx_unconfirmed(self, txid, removed_utxos) self.maker.modify_orders(to_cancel, to_announce) - def confirm_callback(self, confirmations, txid, balance): + def confirm_callback(self, txd, txid, confirmations): self.maker.wallet_unspent_lock.acquire() try: added_utxos = self.maker.wallet.add_new_utxos(self.tx, txid) @@ -96,7 +97,7 @@ def confirm_callback(self, confirmations, txid, balance): self.maker.wallet_unspent_lock.release() debug('tx in a block, added_utxos=\n' + pprint.pformat(added_utxos)) to_cancel, to_announce = self.maker.on_tx_confirmed(self, - confirmations, txid, balance, added_utxos) + confirmations, txid, added_utxos) self.maker.modify_orders(to_cancel, to_announce) def verify_unsigned_tx(self, txd): @@ -277,14 +278,14 @@ def get_next_oid(self): #gets called when the tx is seen on the network #must return which orders to cancel or recreate - def on_tx_unconfirmed(self, cjorder, balance, removed_utxos): + def on_tx_unconfirmed(self, cjorder, txid, removed_utxos): return ([cjorder.oid], []) #gets called when the tx is included in a block #must return which orders to cancel or recreate # and i have to think about how that will work for both # the blockchain explorer api method and the bitcoid walletnotify - def on_tx_confirmed(self, cjorder, confirmations, txid, balance, added_utxos): + def on_tx_confirmed(self, cjorder, confirmations, txid, added_utxos): to_announce = [] for i, out in enumerate(cjorder.tx['outs']): addr = btc.script_to_address(out['script'], get_addr_vbyte()) diff --git a/sendpayment.py b/sendpayment.py index fc18d55e..7ab463f4 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -94,7 +94,8 @@ def main(): common.nickname = 'payer-' +binascii.hexlify(os.urandom(4)) wallet = Wallet(seed, options.mixdepth + 1) - wallet.sync_wallet() + common.bc_interface.sync_wallet(wallet) + wallet.print_debug_wallet_info() irc = IRCMessageChannel(common.nickname) taker = SendPayment(irc, wallet, destaddr, amount, options.makercount, options.txfee, diff --git a/taker.py b/taker.py index f14cf1dd..5a4997d7 100644 --- a/taker.py +++ b/taker.py @@ -1,6 +1,7 @@ #! /usr/bin/env python from common import * +import common import enc_wrapper import bitcoin as btc @@ -121,7 +122,7 @@ def add_signature(self, sigb64): for index, ins in enumerate(self.latest_tx['ins']): if ins['script'] != '': continue - ftx = get_blockchain_data('txraw', csv_params=[ins['outpoint']['hash']], query_params=[False])['tx']['hex'] + ftx = common.bc_interface.fetchtx(ins['outpoint']['hash']) src_val = btc.deserialize(ftx)['outs'][ ins['outpoint']['index'] ] sig_good = btc.verify_tx_input(tx, index, src_val['script'], *btc.deserialize_script(sig)) if sig_good: @@ -143,9 +144,9 @@ def add_signature(self, sigb64): return debug('the entire tx is signed, ready to pushtx()') - print btc.serialize(self.latest_tx) - ret = get_blockchain_data('txpush', csv_params=[btc.serialize(self.latest_tx)]) - debug('pushed tx ' + str(ret)) + debug('\n' + btc.serialize(self.latest_tx)) + txid = common.bc_interface.pushtx(btc.serialize(self.latest_tx)) + debug('pushed tx ' + str(txid)) if self.finishcallback != None: self.finishcallback() diff --git a/wallet-tool.py b/wallet-tool.py index 8d5c947b..bbd39658 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -1,6 +1,7 @@ import bitcoin as btc from common import Wallet, get_signed_tx, load_program_config +import common import sys from optparse import OptionParser @@ -46,10 +47,9 @@ print_privkey = options.showprivkey + wallet = Wallet(seed, options.maxmixdepth) -print 'downloading wallet history' -wallet.download_wallet_history(options.gaplimit) -wallet.find_unspent_addresses() +common.bc_interface.sync_wallet(wallet, options.gaplimit) if method == 'display': total_balance = 0 diff --git a/yield-generator.py b/yield-generator.py index d0308909..1002bb45 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -65,7 +65,7 @@ def oid_to_order(self, oid, amount): change_addr = self.wallet.get_change_addr(mixdepth) return utxos, cj_addr, change_addr - def on_tx_unconfirmed(self, cjorder, balance, removed_utxos): + def on_tx_unconfirmed(self, cjorder, txid, removed_utxos): #if the balance of the highest-balance mixing depth change then reannounce it oldorder = self.orderlist[0] if len(self.orderlist) > 0 else None neworders = self.create_my_orders() @@ -77,7 +77,7 @@ def on_tx_unconfirmed(self, cjorder, balance, removed_utxos): #announce new order, replacing the old order return ([], [neworders[0]]) - def on_tx_confirmed(self, cjorder, confirmations, txid, balance, added_utxos): + def on_tx_confirmed(self, cjorder, confirmations, txid, added_utxos): return self.on_tx_unconfirmed(None, None, None) def main(): @@ -86,7 +86,9 @@ def main(): seed = sys.argv[1] #btc.sha256('dont use brainwallets except for holding testnet coins') load_program_config() wallet = Wallet(seed, max_mix_depth = mix_levels) - wallet.sync_wallet() + common.bc_interface.sync_wallet(wallet) + wallet.print_debug_wallet_info() + common.nickname = 'yigen-'+binascii.hexlify(os.urandom(4)) irc = IRCMessageChannel(common.nickname) maker = YieldGenerator(irc, wallet) From ed05258f5d934b44db56bae74a05c732db9823ed Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 13 Mar 2015 02:23:10 +0000 Subject: [PATCH 157/409] moved some protocol stuff away from README.txt --- README.txt | 43 ++++++------------------------------------- 1 file changed, 6 insertions(+), 37 deletions(-) diff --git a/README.txt b/README.txt index c0049457..8007c130 100644 --- a/README.txt +++ b/README.txt @@ -15,6 +15,9 @@ HOWTO try For real bitcoins you would probably generate it from 128 bits of entropy and encode in a 12-word mnemonic. For testnet just use anything. +$ python gui-taker.py + Starts a local http server which you can connect to and will display the orderbook as well as some graphs + $ python wallet-tool.py [seed] To print out a bunch of addresses, send some testnet coins to an address @@ -26,9 +29,9 @@ $ python patientsendpayments.py -N 1 -w 2 [wallet seed] [amount in satoshi] [des Announces orders and waits to coinjoin for a maximum of 2 hours. Once that time it up cancels the orders and pays to do a 2-party coinjoin. -$ python gui-taker.py - Starts a local http server which you can connect to and will display the orderbook as well as some graphs - +$ python yield-generator.py [seed] + Becomes an investor bot, being online indefinitely and doing coinjoin for the purpose of profit. + Edit the file to change the IRC nick, offered fee, nickserv password and so on Watch the output of your bot(s), soon enough the taker will say it has completed a transaction, maker will wait for the transaction to be seen and confirmed @@ -36,37 +39,3 @@ If there are no orders, you could run two bots from the same machine. Be sure to two seperate wallet seeds though. -some other notes below.. - -#COINJOIN PROTOCOL -when a maker joins the channel it says out all its orders - an order contains an order id, max size, min size, fee, whether the fee is absolute or - as a proportion of the coinjoin amount -when a taker joins the channel, it asks for orders to be pmed to him -taker initiates coinjoin -tells maker(s) by pm which order it wants to fill, sends the order id and the coinjoin amount -maker(s) pm back the utxos they will input, and exactly two addresses, the coinjoin output and the change address -taker collects all the utxos and outputs and makes a transaction - pms them to the maker(s) who check everything is ok - that the miner fee is right, that the cj fee is right - and pm back signatures - iv checked, it easily possible to put the signatures back into a tx -taker then signs his own and pushtx() - -IRC commands used when starting a coinjoin, everything in pm - !fill [order id] [coinjoin amount] [input_pubkey] - !io [comma seperated list of utxos] [coinjoin address] [change address] [coinjoin pubkey] [bitcoin signature] [encryption pubkey] - - !auth [encryption pubkey] [btc_sig] -After this, messages sent between taker and maker will be encrypted. - -when taker collects inputs and outputs of all the makers it's contacted, it creates a tx out of them - !txpart [base64 encoded tx part] -... - !tx [base64 encoded tx part] -maker concatenates all the !txpart and !tx commands and obtains unsigned tx -it signs its own utxos and extracts just the script from it which contains signature and pubkey - !sig [base64 encoded script] -taker collects all scripts and places them into the tx -taker pushes tx when all the scripts have arrived - From 3f1c6e84c7274f0a67b808defc1a9d9b3f18d51b Mon Sep 17 00:00:00 2001 From: Belcher Date: Sat, 14 Mar 2015 21:20:15 +0000 Subject: [PATCH 158/409] sync_wallet() with bitcoind --- blockchaininterface.py | 147 +++++++++++++++++++++++++++++------------ common.py | 7 ++ joinmarket.cfg | 2 +- 3 files changed, 111 insertions(+), 45 deletions(-) diff --git a/blockchaininterface.py b/blockchaininterface.py index c0b1d74b..141c525c 100644 --- a/blockchaininterface.py +++ b/blockchaininterface.py @@ -1,7 +1,7 @@ #from joinmarket import * import subprocess import unittest -import json, threading, abc, pprint, time, random +import json, threading, abc, pprint, time, random, sys from decimal import Decimal import bitcoin as btc @@ -47,7 +47,7 @@ def fetchtx(self, txid): pass @abc.abstractmethod - def pushtx(self, txhash): + def pushtx(self, txhex): '''pushes tx to the network, returns txhash''' pass @@ -221,39 +221,107 @@ def pushtx(self, txhex): class BitcoinCoreInterface(BlockchainInterface): - def __init__(self, bitcoin_cli_cmd, testnet=False): - super(BitcoinCoreInterface, self).__init__() - #self.command_params = ['bitcoin-cli', '-port='+str(port), '-rpcport='+str(rpcport),'-testnet'] - self.command_params = bitcoin_cli_cmd - if testnet: - self.command_params += ['-testnet'] - #quick check that it's up else quit - try: - res = self.rpc(['getbalance']) - except Exception as e: - print e - - def get_net_info(self): - print 'not yet done' - - def rpc(self, args, accept_failure=[]): - try: - #print 'making an rpc call with these parameters: ' - common.debug(str(self.command_params+args)) - res = subprocess.check_output(self.command_params+args) - except subprocess.CalledProcessError, e: - if e.returncode in accept_failure: - return '' - raise - return res - - def send_tx(self, tx_hexs, query_params): - '''csv params contains only tx hex''' - for txhex in tx_hexs: - res = self.rpc(['sendrawtransaction', txhex]) - #TODO only handles a single push; handle multiple - return {'data':res} + def __init__(self, bitcoin_cli_cmd, testnet = False): + super(BitcoinCoreInterface, self).__init__() + #self.command_params = ['bitcoin-cli', '-port='+str(port), '-rpcport='+str(rpcport),'-testnet'] + self.command_params = bitcoin_cli_cmd + if testnet: + self.command_params += ['-testnet'] + #quick check that it's up else quit + try: + res = self.rpc(['getbalance']) + except Exception as e: + print e + + def rpc(self, args, accept_failure=[]): + try: + if args[0] != 'importaddress': + common.debug('rpc: ' + str(self.command_params+args)) + res = subprocess.check_output(self.command_params+args) + except subprocess.CalledProcessError, e: + if e.returncode in accept_failure: + return '' + raise + return res + + def add_watchonly_addresses(self, addr_list, wallet_name): + debug('importing ' + str(len(addr_list)) + ' into account ' + wallet_name) + for addr in addr_list: + self.rpc(['importaddress', addr, wallet_name, 'false']) + print 'now restart bitcoind with -rescan' + sys.exit(0) + + def sync_wallet(self, wallet, gaplimit=6): + wallet_name = 'joinmarket-wallet-' + btc.dbl_sha256(wallet.keys[0][0])[:6] + addr_req_count = 50 + wallet_addr_list = [] + for mix_depth in range(wallet.max_mix_depth): + for forchange in [0, 1]: + wallet_addr_list += [wallet.get_new_addr(mix_depth, forchange) for i in range(addr_req_count)] + wallet.index[mix_depth][forchange] = 0 + imported_addr_list = json.loads(self.rpc(['getaddressesbyaccount', wallet_name])) + if not set(wallet_addr_list).issubset(set(imported_addr_list)): + self.add_watchonly_addresses(wallet_addr_list, wallet_name) + return + + #TODO get all the transactions above 200, by looping until len(result) < 200 + ret = self.rpc(['listtransactions', wallet_name, '200', '0', 'true']) + txs = json.loads(ret) + used_addr_list = [tx['address'] for tx in txs] + too_few_addr_mix_change = [] + for mix_depth in range(wallet.max_mix_depth): + for forchange in [0, 1]: + unused_addr_count = 0 + last_used_addr = '' + breakloop = False + while not breakloop: + if unused_addr_count >= gaplimit: + break + mix_change_addrs = [wallet.get_new_addr(mix_depth, forchange) for i in range(addr_req_count)] + for mc_addr in mix_change_addrs: + if mc_addr not in imported_addr_list: + print 'too few addresses for ' + str(mix_depth) + ', ' + str(forchange) + too_few_addr_mix_change.append((mix_depth, forchange)) + breakloop = True + break + if mc_addr in used_addr_list: + last_used_addr = mc_addr + else: + unused_addr_count += 1 + if unused_addr_count >= gaplimit: + breakloop = True + break + if last_used_addr == '': + wallet.index[mix_depth][forchange] = 0 + else: + wallet.index[mix_depth][forchange] = wallet.addr_cache[last_used_addr][2] + 1 + + wallet_addr_list = [] + if len(too_few_addr_mix_change) > 0: + debug('too few addresses in ' + str(too_few_addr_mix_change)) + for mix_depth, forchange in too_few_addr_mix_change: + wallet_addr_list += [wallet.get_new_addr(mix_depth, forchange) for i in range(addr_req_count)] + self.add_watchonly_addresses(wallet_addr_list, wallet_name) + return + + unspent_list = json.loads(self.rpc(['listunspent'])) + for u in unspent_list: + if u['account'] != wallet_name: + continue + wallet.unspent[u['txid'] + ':' + str(u['vout'])] = {'address': u['address'], + 'value': u['amount']*1e8} + + def add_tx_notify(self, tx, unconfirmfun, confirmfun): + pass + + def fetchtx(self, txid): + pass + + def pushtx(self, txhex): + txid = self.rpc(['sendrawtransaction', txhex]) + return txid +''' def get_utxos_from_addr(self, addresses, query_params): r = [] for address in addresses: @@ -287,7 +355,6 @@ def get_txs_from_addr(self, addresses, query_params): return {'data':result} def get_tx_info(self, txhashes, query_params): - '''Returns a list of vouts if first entry in query params is False, else returns tx hex''' #TODO: handle more than one tx hash res = json.loads(self.rpc(['getrawtransaction', txhashes[0], '1'])) if not query_params[0]: @@ -316,16 +383,8 @@ def get_balance_at_addr(self, addresses, query_params): res.append({'address':address,'balance':\ int(Decimal(1e8) * Decimal(self.rpc(['getreceivedbyaddress', address])))}) return {'data':res} +''' - #Not used; I think, not needed - '''def get_addr_from_utxo(self, txhash, index): - #get the transaction details - res = json.loads(self.rpc(['gettxout', txhash, str(index)])) - amt = int(Decimal(1e8)*Decimal(res['value'])) - address = res('addresses')[0] - return (address, amt) - ''' - #class for regtest chain access #running on local daemon. Only #to be instantiated after network is up diff --git a/common.py b/common.py index 74b00b71..e5f2a13d 100644 --- a/common.py +++ b/common.py @@ -194,6 +194,13 @@ def select_utxos(self, mixdepth, amount): debug(pprint.pformat(inputs)) return [i['utxo'] for i in inputs] + def print_debug_wallet_info(self): + debug('printing debug wallet information') + debug('utxos') + debug(pprint.pformat(self.unspent)) + debug('wallet.index') + debug(pprint.pformat(self.index)) + def calc_cj_fee(ordertype, cjfee, cj_amount): real_cjfee = None if ordertype == 'absorder': diff --git a/joinmarket.cfg b/joinmarket.cfg index c0faa4d2..3d1683ec 100644 --- a/joinmarket.cfg +++ b/joinmarket.cfg @@ -1,5 +1,5 @@ [BLOCKCHAIN] -blockchain_source = blockr +blockchain_source = json-rpc #options: blockr, json-rpc, regtest network = testnet bitcoin_cli_cmd = bitcoin-cli From d8329a1268d0b1f2648b0a89cfce817c4413fd85 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 15 Mar 2015 02:42:06 +0000 Subject: [PATCH 159/409] wrote -walletnotify code for add_tx_notify() method --- blockchaininterface.py | 271 +++++++++++++++++++---------------------- joinmarket.cfg | 2 +- wallet-tool.py | 1 - yield-generator.py | 1 - 4 files changed, 129 insertions(+), 146 deletions(-) diff --git a/blockchaininterface.py b/blockchaininterface.py index 141c525c..0b3966dc 100644 --- a/blockchaininterface.py +++ b/blockchaininterface.py @@ -2,6 +2,7 @@ import subprocess import unittest import json, threading, abc, pprint, time, random, sys +import BaseHTTPServer, SimpleHTTPServer from decimal import Decimal import bitcoin as btc @@ -37,7 +38,7 @@ def sync_wallet(self, wallet, gaplimit=6): pass @abc.abstractmethod - def add_tx_notify(self, tx, unconfirmfun, confirmfun): + def add_tx_notify(self, txd, unconfirmfun, confirmfun): '''Invokes unconfirmfun and confirmfun when tx is seen on the network''' pass @@ -144,7 +145,7 @@ def run(self): while not unconfirmed_txid: time.sleep(unconfirm_poll_period) if int(time.time()) - st > unconfirm_timeout: - debug('checking for unconfirmed tx timed out') + common.debug('checking for unconfirmed tx timed out') return blockr_url = 'http://' + self.blockr_domain + '.blockr.io/api/v1/address/unspent/' random.shuffle(self.output_addresses) #seriously weird bug with blockr.io @@ -179,7 +180,7 @@ def run(self): while not confirmed_txid: time.sleep(confirm_poll_period) if int(time.time()) - st > confirm_timeout: - debug('checking for confirmed tx timed out') + common.debug('checking for confirmed tx timed out') return blockr_url = 'http://' + self.blockr_domain + '.blockr.io/api/v1/address/txs/' data = json.loads(btc.make_request(blockr_url + ','.join(self.output_addresses)))['data'] @@ -215,37 +216,84 @@ def pushtx(self, txhex): data = json.loads(btc.blockr_pushtx(txhex, self.network)) if data['status'] != 'success': #error message generally useless so there might not be a point returning - debug(data) + common.debug(data) return None return data['data'] +class NotifyRequestHeader(SimpleHTTPServer.SimpleHTTPRequestHandler): + def __init__(self, request, client_address, base_server): + self.btcinterface = base_server.btcinterface + self.base_server = base_server + SimpleHTTPServer.SimpleHTTPRequestHandler.__init__(self, request, client_address, base_server) + + def do_HEAD(self): + print 'httpd received HEAD ' + self.path + ' request' + pages = ('/walletnotify?', '/alertnotify?') + if not self.path.startswith(pages): + return + if self.path.startswith('/walletnotify?'): + txid = self.path[len(pages[0]):] + txd = btc.deserialize(self.btcinterface.fetchtx(txid)) + tx_output_set = set([(sv['script'], sv['value']) for sv in txd['outs']]) + print 'outs = ' + str(tx_output_set) + + unconfirmfun, confirmfun = None, None + for tx_out, ucfun, cfun in self.btcinterface.txnotify_fun: + if tx_out == tx_output_set: + unconfirmfun = ucfun + confirmfun = cfun + break + if not unconfirmfun: + common.debug('txid=' + txid + ' not being listened for') + return + txdata = json.loads(self.btcinterface.rpc(['gettxout', txid, '0', 'true'])) + if txdata['confirmations'] == 0: + unconfirmfun(txd, txid) + else: + confirmfun(txd, txid, txdata['confirmations']) + self.btcinterface.txnotify_fun.remove((tx_out, unconfirmfun, confirmfun)) + + elif self.path.startswith('/alertnotify?'): + message = self.path[len(pages[1]):] + print 'got an alert, shit, shutting down. message=' + message + sys.exit(0) + self.send_response(200) + #self.send_header('Connection', 'close') + self.end_headers() + +class BitcoinCoreNotifyThread(threading.Thread): + def __init__(self, btcinterface): + threading.Thread.__init__(self) + self.daemon = True + self.btcinterface = btcinterface + + def run(self): + common.debug('started bitcoin core notify listening thread') + hostport = ('localhost', 62602) + httpd = BaseHTTPServer.HTTPServer(hostport, NotifyRequestHeader) + httpd.btcinterface = self.btcinterface + httpd.serve_forever() class BitcoinCoreInterface(BlockchainInterface): def __init__(self, bitcoin_cli_cmd, testnet = False): super(BitcoinCoreInterface, self).__init__() - #self.command_params = ['bitcoin-cli', '-port='+str(port), '-rpcport='+str(rpcport),'-testnet'] self.command_params = bitcoin_cli_cmd if testnet: self.command_params += ['-testnet'] - #quick check that it's up else quit - try: - res = self.rpc(['getbalance']) - except Exception as e: - print e + self.notifythread = None + self.txnotify_fun = [] - def rpc(self, args, accept_failure=[]): + def rpc(self, args): try: if args[0] != 'importaddress': - common.debug('rpc: ' + str(self.command_params+args)) - res = subprocess.check_output(self.command_params+args) + common.debug('rpc: ' + str(self.command_params + args)) + res = subprocess.check_output(self.command_params + args) + return res except subprocess.CalledProcessError, e: - if e.returncode in accept_failure: - return '' - raise - return res + raise #something here def add_watchonly_addresses(self, addr_list, wallet_name): - debug('importing ' + str(len(addr_list)) + ' into account ' + wallet_name) + common.debug('importing ' + str(len(addr_list)) + ' into account ' + wallet_name) for addr in addr_list: self.rpc(['importaddress', addr, wallet_name, 'false']) print 'now restart bitcoind with -rescan' @@ -264,9 +312,11 @@ def sync_wallet(self, wallet, gaplimit=6): self.add_watchonly_addresses(wallet_addr_list, wallet_name) return - #TODO get all the transactions above 200, by looping until len(result) < 200 - ret = self.rpc(['listtransactions', wallet_name, '200', '0', 'true']) + #TODO get all the transactions above 1000, by looping until len(result) < 1000 + ret = self.rpc(['listtransactions', wallet_name, '1000', '0', 'true']) txs = json.loads(ret) + if len(txs) == 1000: + raise Exception('time to stop putting off this bug and actually fix it, see the TODO') used_addr_list = [tx['address'] for tx in txs] too_few_addr_mix_change = [] for mix_depth in range(wallet.max_mix_depth): @@ -280,7 +330,6 @@ def sync_wallet(self, wallet, gaplimit=6): mix_change_addrs = [wallet.get_new_addr(mix_depth, forchange) for i in range(addr_req_count)] for mc_addr in mix_change_addrs: if mc_addr not in imported_addr_list: - print 'too few addresses for ' + str(mix_depth) + ', ' + str(forchange) too_few_addr_mix_change.append((mix_depth, forchange)) breakloop = True break @@ -299,7 +348,7 @@ def sync_wallet(self, wallet, gaplimit=6): wallet_addr_list = [] if len(too_few_addr_mix_change) > 0: - debug('too few addresses in ' + str(too_few_addr_mix_change)) + common.debug('too few addresses in ' + str(too_few_addr_mix_change)) for mix_depth, forchange in too_few_addr_mix_change: wallet_addr_list += [wallet.get_new_addr(mix_depth, forchange) for i in range(addr_req_count)] self.add_watchonly_addresses(wallet_addr_list, wallet_name) @@ -310,140 +359,76 @@ def sync_wallet(self, wallet, gaplimit=6): if u['account'] != wallet_name: continue wallet.unspent[u['txid'] + ':' + str(u['vout'])] = {'address': u['address'], - 'value': u['amount']*1e8} + 'value': int(u['amount']*1e8)} - def add_tx_notify(self, tx, unconfirmfun, confirmfun): - pass + def add_tx_notify(self, txd, unconfirmfun, confirmfun): + if not self.notifythread: + self.notifythread = BitcoinCoreNotifyThread(self) + self.notifythread.start() + tx_output_set = set([(sv['script'], sv['value']) for sv in txd['outs']]) + self.txnotify_fun.append((tx_output_set, unconfirmfun, confirmfun)) def fetchtx(self, txid): - pass + return self.rpc(['getrawtransaction', txid]).strip() def pushtx(self, txhex): - txid = self.rpc(['sendrawtransaction', txhex]) - return txid -''' - def get_utxos_from_addr(self, addresses, query_params): - r = [] - for address in addresses: - res = json.loads(self.rpc(['listunspent','1','9999999','[\"'+address+'\"]'])) - unspents=[] - for u in res: - unspents.append({'tx':u['txid'],'n':u['vout'],'amount':str(u['amount']),'address':address,'confirmations':u['confirmations']}) - r.append({'address':address,'unspent':unspents}) - return {'data':r} - - def get_txs_from_addr(self, addresses, query_params): - #use listtransactions and then filter - #e.g.: -regtest listtransactions 'watchonly' 1000 0 true - #to get the last 1000 transactions TODO 1000 is arbitrary - acct_addrlist = self.rpc(['getaddressesbyaccount', 'watchonly']) - for address in addresses: - if address not in acct_addrlist: - self.rpc(['importaddress', address, 'watchonly'],[4]) - res = json.loads(self.rpc(['listtransactions','watchonly','2000','0','true'])) - - result=[] - for address in addresses: - nbtxs = 0 - txs=[] - for a in res: - if a['address'] != address: - continue - nbtxs += 1 - txs.append({'confirmations':a['confirmations'],'tx':a['txid'],'amount':a['amount']}) - result.append({'nb_txs':nbtxs,'address':address,'txs':txs}) - return {'data':result} - - def get_tx_info(self, txhashes, query_params): - #TODO: handle more than one tx hash - res = json.loads(self.rpc(['getrawtransaction', txhashes[0], '1'])) - if not query_params[0]: - return {'data':{'tx':{'hex':res['hex']}}} - tx = btc.deserialize(res['hex']) - #build vout list - vouts = [] - n=0 - for o in tx['outs']: - vouts.append({'n':n,'amount':o['value'],'address':btc.script_to_address(o['script'],0x6f)}) - n+=1 - - return {'data':{'vouts':vouts}} - - def get_balance_at_addr(self, addresses, query_params): - #NB This will NOT return coinbase coins (but wont matter in our use case). - #In order to have the Bitcoin RPC read balances at addresses - #it doesn't own, we must import the addresses as watch-only - #Note that this is a 0.10 feature; won't work with older bitcoin clients. - #TODO : there can be a performance issue with rescanning here. - #TODO: This code is WRONG, reports *received* coins in total, not current balance. - #allow importaddress to fail in case the address is already in the wallet - res = [] - for address in addresses: - self.rpc(['importaddress', address,'watchonly'],[4]) - res.append({'address':address,'balance':\ - int(Decimal(1e8) * Decimal(self.rpc(['getreceivedbyaddress', address])))}) - return {'data':res} -''' + return self.rpc(['sendrawtransaction', txhex]).strip() #class for regtest chain access #running on local daemon. Only #to be instantiated after network is up #with > 100 blocks. class RegtestBitcoinCoreInterface(BitcoinCoreInterface): - def __init__(self, bitcoin_cli_cmd): - super(BitcoinCoreInterface, self).__init__() - #self.command_params = ['bitcoin-cli', '-port='+str(port), '-rpcport='+str(rpcport),'-testnet'] - self.command_params = bitcoin_cli_cmd + ['-regtest'] - #quick check that it's up else quit - try: - res = self.rpc(['getbalance']) - except Exception as e: - print e - - def send_tx(self, tx_hex, query_params): - super(RegtestBitcoinCoreInterface, self).send_tx(tx_hex, query_params) - self.tick_forward_chain(1) - - def tick_forward_chain(self, n): - '''Special method for regtest only; - instruct to mine n blocks.''' - self.rpc(['setgenerate','true', str(n)]) - - def grab_coins(self, receiving_addr, amt=50): - ''' - NOTE! amt is passed in Coins, not Satoshis! - Special method for regtest only: - take coins from bitcoind's own wallet - and put them in the receiving addr. - Return the txid. - ''' - if amt > 500: - raise Exception("too greedy") - ''' - if amt > self.current_balance: - #mine enough to get to the reqd amt - reqd = int(amt - self.current_balance) - reqd_blocks = str(int(reqd/50) +1) - if self.rpc(['setgenerate','true', reqd_blocks]): - raise Exception("Something went wrong") - ''' - #now we do a custom create transaction and push to the receiver - txid = self.rpc(['sendtoaddress', receiving_addr, str(amt)]) - if not txid: - raise Exception("Failed to broadcast transaction") - #confirm - self.tick_forward_chain(1) - return txid + def __init__(self, bitcoin_cli_cmd): + super(BitcoinCoreInterface, self).__init__(bitcoin_cli_cmd, False) + self.command_params = bitcoin_cli_cmd + ['-regtest'] + + def pushtx(self, txhex): + ret = super(RegtestBitcoinCoreInterface, self).send_tx(txhex) + self.tick_forward_chain(1) + return ret + + def tick_forward_chain(self, n): + '''Special method for regtest only; + instruct to mine n blocks.''' + self.rpc(['setgenerate','true', str(n)]) + + def grab_coins(self, receiving_addr, amt=50): + ''' + NOTE! amt is passed in Coins, not Satoshis! + Special method for regtest only: + take coins from bitcoind's own wallet + and put them in the receiving addr. + Return the txid. + ''' + if amt > 500: + raise Exception("too greedy") + ''' + if amt > self.current_balance: + #mine enough to get to the reqd amt + reqd = int(amt - self.current_balance) + reqd_blocks = str(int(reqd/50) +1) + if self.rpc(['setgenerate','true', reqd_blocks]): + raise Exception("Something went wrong") + ''' + #now we do a custom create transaction and push to the receiver + txid = self.rpc(['sendtoaddress', receiving_addr, str(amt)]) + if not txid: + raise Exception("Failed to broadcast transaction") + #confirm + self.tick_forward_chain(1) + return txid def main(): - myBCI = RegtestBitcoinCoreInterface() - #myBCI.send_tx('stuff') - print myBCI.get_utxos_from_addr(["n4EjHhGVS4Rod8ociyviR3FH442XYMWweD"]) - print myBCI.get_balance_at_addr(["n4EjHhGVS4Rod8ociyviR3FH442XYMWweD"]) - txid = myBCI.grab_coins('mygp9fsgEJ5U7jkPpDjX9nxRj8b5nC3Hnd',23) - print txid - print myBCI.get_balance_at_addr(['mygp9fsgEJ5U7jkPpDjX9nxRj8b5nC3Hnd']) - print myBCI.get_utxos_from_addr(['mygp9fsgEJ5U7jkPpDjX9nxRj8b5nC3Hnd']) + #TODO some useful quick testing here, so people know if they've set it up right + myBCI = RegtestBitcoinCoreInterface() + #myBCI.send_tx('stuff') + print myBCI.get_utxos_from_addr(["n4EjHhGVS4Rod8ociyviR3FH442XYMWweD"]) + print myBCI.get_balance_at_addr(["n4EjHhGVS4Rod8ociyviR3FH442XYMWweD"]) + txid = myBCI.grab_coins('mygp9fsgEJ5U7jkPpDjX9nxRj8b5nC3Hnd',23) + print txid + print myBCI.get_balance_at_addr(['mygp9fsgEJ5U7jkPpDjX9nxRj8b5nC3Hnd']) + print myBCI.get_utxos_from_addr(['mygp9fsgEJ5U7jkPpDjX9nxRj8b5nC3Hnd']) if __name__ == '__main__': main() diff --git a/joinmarket.cfg b/joinmarket.cfg index 3d1683ec..c0faa4d2 100644 --- a/joinmarket.cfg +++ b/joinmarket.cfg @@ -1,5 +1,5 @@ [BLOCKCHAIN] -blockchain_source = json-rpc +blockchain_source = blockr #options: blockr, json-rpc, regtest network = testnet bitcoin_cli_cmd = bitcoin-cli diff --git a/wallet-tool.py b/wallet-tool.py index bbd39658..d55cf61e 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -41,7 +41,6 @@ method = ('display' if len(args) == 1 else args[1].lower()) -load_program_config() #seed = '256 bits of randomness' diff --git a/yield-generator.py b/yield-generator.py index 1002bb45..e3a82f0c 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -84,7 +84,6 @@ def main(): common.load_program_config() import sys seed = sys.argv[1] #btc.sha256('dont use brainwallets except for holding testnet coins') - load_program_config() wallet = Wallet(seed, max_mix_depth = mix_levels) common.bc_interface.sync_wallet(wallet) wallet.print_debug_wallet_info() From 4b202a5f2efad47266b1a740d01c5dc4822a79a6 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 15 Mar 2015 15:14:06 +0000 Subject: [PATCH 160/409] added some notes --- blockchaininterface.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/blockchaininterface.py b/blockchaininterface.py index 0b3966dc..81f421d5 100644 --- a/blockchaininterface.py +++ b/blockchaininterface.py @@ -274,6 +274,12 @@ def run(self): httpd.btcinterface = self.btcinterface httpd.serve_forever() +#must run bitcoind with -txindex=1 -server +#-walletnotify="wget -spider -q http://localhost:62602/walletnotify?%s" +#and make sure wget is installed + +#TODO must add the tx addresses as watchonly if case we ever broadcast a tx +# with addresses not belonging to us class BitcoinCoreInterface(BlockchainInterface): def __init__(self, bitcoin_cli_cmd, testnet = False): super(BitcoinCoreInterface, self).__init__() From 05699af3123275092e195f4a1d9eb2c178abc9d6 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 15 Mar 2015 18:45:05 +0000 Subject: [PATCH 161/409] bugfix --- blockchaininterface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blockchaininterface.py b/blockchaininterface.py index 81f421d5..b8c63847 100644 --- a/blockchaininterface.py +++ b/blockchaininterface.py @@ -275,7 +275,7 @@ def run(self): httpd.serve_forever() #must run bitcoind with -txindex=1 -server -#-walletnotify="wget -spider -q http://localhost:62602/walletnotify?%s" +#-walletnotify="wget --spider -q http://localhost:62602/walletnotify?%s" #and make sure wget is installed #TODO must add the tx addresses as watchonly if case we ever broadcast a tx From eeca053c4eb4b56c18c9c8ea95a08336765d0a2e Mon Sep 17 00:00:00 2001 From: Belcher Date: Tue, 17 Mar 2015 02:21:34 +0000 Subject: [PATCH 162/409] tumbler.py first version --- blockchaininterface.py | 9 +- common.py | 1 + patientsendpayment.py | 2 +- sendpayment.py | 4 +- taker.py | 2 +- tumbler.py | 219 +++++++++++++++++++++++++++++++++++------ 6 files changed, 202 insertions(+), 35 deletions(-) diff --git a/blockchaininterface.py b/blockchaininterface.py index b8c63847..214eb8d0 100644 --- a/blockchaininterface.py +++ b/blockchaininterface.py @@ -299,15 +299,16 @@ def rpc(self, args): raise #something here def add_watchonly_addresses(self, addr_list, wallet_name): - common.debug('importing ' + str(len(addr_list)) + ' into account ' + wallet_name) + common.debug('importing ' + str(len(addr_list)) + ' addresses into account ' + wallet_name) for addr in addr_list: self.rpc(['importaddress', addr, wallet_name, 'false']) print 'now restart bitcoind with -rescan' sys.exit(0) def sync_wallet(self, wallet, gaplimit=6): + common.debug('requesting wallet history') wallet_name = 'joinmarket-wallet-' + btc.dbl_sha256(wallet.keys[0][0])[:6] - addr_req_count = 50 + addr_req_count = 20 wallet_addr_list = [] for mix_depth in range(wallet.max_mix_depth): for forchange in [0, 1]: @@ -356,7 +357,7 @@ def sync_wallet(self, wallet, gaplimit=6): if len(too_few_addr_mix_change) > 0: common.debug('too few addresses in ' + str(too_few_addr_mix_change)) for mix_depth, forchange in too_few_addr_mix_change: - wallet_addr_list += [wallet.get_new_addr(mix_depth, forchange) for i in range(addr_req_count)] + wallet_addr_list += [wallet.get_new_addr(mix_depth, forchange) for i in range(addr_req_count*3)] self.add_watchonly_addresses(wallet_addr_list, wallet_name) return @@ -364,6 +365,8 @@ def sync_wallet(self, wallet, gaplimit=6): for u in unspent_list: if u['account'] != wallet_name: continue + if u['address'] not in wallet.addr_cache: + continue wallet.unspent[u['txid'] + ':' + str(u['vout'])] = {'address': u['address'], 'value': int(u['amount']*1e8)} diff --git a/common.py b/common.py index e5f2a13d..c7f5f494 100644 --- a/common.py +++ b/common.py @@ -167,6 +167,7 @@ def get_utxo_list_by_mixdepth(self): ''' returns a list of utxos sorted by different mix levels ''' + pprint.pprint(self.unspent) mix_utxo_list = {} for utxo, addrvalue in self.unspent.iteritems(): mixdepth = self.addr_cache[addrvalue['address']][0] diff --git a/patientsendpayment.py b/patientsendpayment.py index 22d2f874..98fd290b 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -16,7 +16,7 @@ def __init__(self, tmaker): self.tmaker = tmaker self.finished = False - def finishcallback(self): + def finishcallback(self, coinjointx): self.tmaker.msgchan.shutdown() def run(self): diff --git a/sendpayment.py b/sendpayment.py index 7ab463f4..76201ae7 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -17,7 +17,7 @@ def __init__(self, taker): self.daemon = True self.taker = taker - def finishcallback(self): + def finishcallback(self, coinjointx): self.taker.msgchan.shutdown() def run(self): @@ -88,7 +88,7 @@ def main(): amount = int(args[1]) destaddr = args[2] - common.load_program_config() + load_program_config() import binascii, os common.nickname = 'payer-' +binascii.hexlify(os.urandom(4)) diff --git a/taker.py b/taker.py index 5a4997d7..8ed6b5d0 100644 --- a/taker.py +++ b/taker.py @@ -148,7 +148,7 @@ def add_signature(self, sigb64): txid = common.bc_interface.pushtx(btc.serialize(self.latest_tx)) debug('pushed tx ' + str(txid)) if self.finishcallback != None: - self.finishcallback() + self.finishcallback(self) class CoinJoinerPeer(object): def __init__(self, msgchan): diff --git a/tumbler.py b/tumbler.py index 2c086d53..ff167e26 100644 --- a/tumbler.py +++ b/tumbler.py @@ -1,9 +1,15 @@ +import taker as takermodule +import common +from common import * +from irc import IRCMessageChannel + from optparse import OptionParser -import datetime +import datetime, threading, binascii import numpy as np from pprint import pprint -import common + +orderwaittime = 5 def lower_bounded_int(thelist, lowerbound): return [int(l) if int(l) >= lowerbound else lowerbound for l in thelist] @@ -25,24 +31,147 @@ def generate_tumbler_tx(destaddrs, options): #assume that the sizes of outputs will follow a power law amount_ratios = 1.0 - np.random.power(options.amountpower, txcount) amount_ratios /= sum(amount_ratios) - #transaction times are uncorrelated and therefore follow poisson - blockheight_waits = np.random.poisson(options.timelambda, txcount) + #transaction times are uncorrelated + #time between events in a poisson process followed exp + waits = np.random.exponential(options.timelambda, txcount) #number of makers to use follows a normal distribution makercounts = np.random.normal(options.makercountrange[0], options.makercountrange[1], txcount) makercounts = lower_bounded_int(makercounts, 2) - for amount_ratio, blockheight_wait, makercount in zip(amount_ratios, blockheight_waits, makercounts): - tx = {'amount_ratio': amount_ratio, 'blockheight_wait': blockheight_wait, - 'srcmixdepth': m + options.mixdepthsrc, 'makercount': makercount} + for amount_ratio, wait, makercount in zip(amount_ratios, waits, makercounts): + tx = {'amount_ratio': amount_ratio, 'wait': round(wait, 2), + 'srcmixdepth': m + options.mixdepthsrc, 'makercount': makercount, 'dest': 'internal'} tx_list.append(tx) - pprint(tx_list) - block_count = sum([tx['blockheight_wait'] for tx in tx_list]) - print 'requires ' + str(block_count) + ' blocks' - print('estimated time taken ' + str(block_count*10) + - ' minutes or ' + str(block_count/6.0) + ' hours') - maker_count = sum([tx['makercount'] for tx in tx_list]) - relorder_fee = 0.001 - print('uses ' + str(maker_count) + ' makers, at ' + str(relorder_fee*100) + '% per maker, estimated total cost ' - + str(round((1 - (1 - relorder_fee)**maker_count) * 100, 3)) + '%') + + total_dest_addr = len(destaddrs) + options.addrask + external_dest_addrs = destaddrs + ['addrask']*options.addrask + if total_dest_addr >= options.mixdepthcount: + print 'not enough mixing depths to pay to all destination addresses' + return None + for i, srcmix in enumerate(range(options.mixdepthcount - total_dest_addr, options.mixdepthcount)): + for tx in reversed(tx_list): + if tx['srcmixdepth'] == srcmix: + tx['dest'] = external_dest_addrs[i] + break + if total_dest_addr - i != 1: + continue + tx_list_remove = [] + for tx in tx_list: + if tx['srcmixdepth'] == srcmix: + if tx['dest'] == 'internal': + tx_list_remove.append(tx) + else: + tx['amount_ratio'] = 1.0 + [tx_list.remove(t) for t in tx_list_remove] + return tx_list + +#thread which does the buy-side algorithm +# chooses which coinjoins to initiate and when +class TumblerThread(threading.Thread): + def __init__(self, taker): + threading.Thread.__init__(self) + self.daemon = True + self.taker = taker + + def unconfirm_callback(self, txd, txid): + pass + + def confirm_callback(self, txd, txid, confirmations): + self.taker.wallet.add_new_utxos(txd, txid) + self.lockcond.acquire() + self.lockcond.notify() + self.lockcond.release() + + def finishcallback(self, coinjointx): + common.bc_interface.add_tx_notify(coinjointx.latest_tx, + self.unconfirm_callback, self.confirm_callback) + self.taker.wallet.remove_old_utxos(coinjointx.latest_tx) + + def send_tx(self, tx, sweep, i, l): + total_value = 0 + all_utxos = self.taker.wallet.get_utxo_list_by_mixdepth()[tx['srcmixdepth']] + for utxo in all_utxos: + total_value += self.taker.wallet.unspent[utxo]['value'] + + destaddr = None + changeaddr = None + if tx['dest'] == 'internal': + destaddr = self.taker.wallet.get_receive_addr(tx['srcmixdepth'] + 1) + changeaddr = self.taker.wallet.get_change_addr(tx['srcmixdepth']) + elif tx['dest'] == 'addrask': + destaddr = raw_input('insert new address: ') + changeaddr = self.taker.wallet.get_change_addr(tx['srcmixdepth']) + else: + destaddr = tx['dest'] + changeaddr = self.taker.wallet.get_change_addr(tx['srcmixdepth']) + + if sweep: + + orders, cjamount = choose_sweep_order(self.taker.db, total_value, self.taker.txfee, tx['makercount']) + self.taker.cjtx = takermodule.CoinJoinTX(self.taker.msgchan, self.taker.wallet, self.taker.db, + cjamount, orders, all_utxos, destaddr, None, self.taker.txfee, self.finishcallback) + else: + amount = int(tx['amount_ratio'] * total_value) + print 'coinjoining ' + str(amount) + orders, total_cj_fee = choose_order(self.taker.db, amount, tx['makercount']) + print 'chosen orders to fill ' + str(orders) + ' totalcjfee=' + str(total_cj_fee) + total_amount = amount + total_cj_fee + self.taker.txfee + print 'total amount spent = ' + str(total_amount) + + utxos = self.taker.wallet.select_utxos(tx['srcmixdepth'], total_amount) + self.taker.cjtx = takermodule.CoinJoinTX(self.taker.msgchan, self.taker.wallet, + self.taker.db, amount, orders, utxos, destaddr, changeaddr, self.taker.txfee, + self.finishcallback) + + print 'that was %d tx out of %d' % (i, l) + self.lockcond.acquire() + self.lockcond.wait() + self.lockcond.release() + debug('tx confirmed, waiting for ' + str(tx['wait']) + ' minutes') + time.sleep(tx['wait'] * 60) + print 'woken' + + def run(self): + print 'waiting for all orders to certainly arrive' + time.sleep(orderwaittime) + + maker_count = sum([tx['makercount'] for tx in self.taker.tx_list]) + relorder_fee = 0.01 + print('uses ' + str(maker_count) + ' makers, at ' + str(relorder_fee*100) + '% per maker, estimated total cost ' + + str(round((1 - (1 - relorder_fee)**maker_count) * 100, 3)) + '%') + + self.lockcond = threading.Condition() + + for i, tx in enumerate(self.taker.tx_list): + sweep = True + for later_tx in self.taker.tx_list[i + 1:]: + if later_tx['srcmixdepth'] == tx['srcmixdepth']: + sweep = False + self.send_tx(tx, sweep, i, len(self.taker.tx_list)) + + print 'total finished' + self.taker.msgchan.shutdown() + + ''' + crow = self.taker.db.execute('SELECT COUNT(DISTINCT counterparty) FROM orderbook;').fetchone() + counterparty_count = crow['COUNT(DISTINCT counterparty)'] + if counterparty_count < self.taker.makercount: + print 'not enough counterparties to fill order, ending' + self.taker.msgchan.shutdown() + return + ''' + + +class Tumbler(takermodule.Taker): + def __init__(self, msgchan, wallet, tx_list, txfee, maxcjfee): + takermodule.Taker.__init__(self, msgchan) + self.wallet = wallet + self.tx_list = tx_list + self.maxcjfee = maxcjfee + self.txfee = txfee + + def on_welcome(self): + takermodule.Taker.on_welcome(self) + TumblerThread(self).start() def main(): parser = OptionParser(usage='usage: %prog [options] [seed] [tumble-file / destaddr...]', @@ -56,6 +185,8 @@ def main(): help='mixing depth to spend from, default=0', default=0) parser.add_option('-f', '--txfee', type='int', dest='txfee', default=10000, help='miner fee contribution, in satoshis, default=10000') + parser.add_option('-x', '--maxcjfee', type='float', dest='maxcjfee', + default=0.02, help='maximum coinjoin fee the tumbler is willing to pay for a single coinjoin. default=0.02 (2%)') parser.add_option('-a', '--addrask', type='int', dest='addrask', default=2, help='How many more addresses to ask for in the terminal. Should ' 'be similar to --txcountparams. default=2') @@ -64,17 +195,17 @@ def main(): help='Input the range of makers to use. e.g. 3-5 will random use between ' '3 and 5 makers inclusive, default=3 4', default=(3, 1)) parser.add_option('-M', '--mixdepthcount', type='int', dest='mixdepthcount', - help='how many mixing depths to mix through', default=3) + help='How many mixing depths to mix through', default=4) parser.add_option('-c', '--txcountparams', type='float', nargs=2, dest='txcountparams', default=(5, 1), help='The number of transactions to take coins from one mixing depth to the next, it is' ' randomly chosen following a normal distribution. Should be similar to --addrask. ' - 'This option controlled the parameters of that normal curve. (mean, standard deviation). default=(3, 1)') + 'This option controlls the parameters of that normal curve. (mean, standard deviation). default=(3, 1)') parser.add_option('--amountpower', type='float', dest='amountpower', default=100.0, - help='the output amounts follow a power law distribution, this is the power, default=100.0') + help='The output amounts follow a power law distribution, this is the power, default=100.0') parser.add_option('-l', '--timelambda', type='float', dest='timelambda', default=2, - help='the number of blocks to wait between transactions is randomly chosen ' - ' following a poisson distribution. This parameter is the lambda of that ' - ' distribution. default=2 blocks') + help='Average the number of minutes to wait between transactions. Randomly chosen ' + ' following an exponential distribution, which describes the time between uncorrelated' + ' events. default=5') parser.add_option('-w', '--wait-time', action='store', type='float', dest='waittime', help='wait time in seconds to allow orders to arrive, default=5', default=5) @@ -91,23 +222,55 @@ def main(): if len(destaddrs) + options.addrask <= 1: print '='*50 print 'WARNING: You are only using one destination address' - print 'this is almost useless for privacy' + print 'this is very bad for privacy' print '='*50 print 'seed=' + seed print 'destaddrs=' + str(destaddrs) print str(options) - generate_tumbler_tx(destaddrs, options) + tx_list = generate_tumbler_tx(destaddrs, options) + + pprint(tx_list) + total_wait = sum([tx['wait'] for tx in tx_list]) + print 'waits in total for ' + str(len(tx_list)) + ' blocks and ' + str(total_wait) + ' minutes' + total_block_and_wait = len(tx_list)*10 + total_wait + print('estimated time taken ' + str(total_block_and_wait) + + ' minutes or ' + str(round(total_block_and_wait/60.0, 2)) + ' hours') + + ret = raw_input('tumble with these tx? (y/n):') + if ret[0] != 'y': + return - #a couple of overarching modes + #a couple of modes #im-running-from-the-nsa, takes about 80 hours, costs a lot - #python tumbler.py -a 10 -N 10 5 -c 10 5 -l 5 -M 10 seed 1xxx + #python tumbler.py -a 10 -N 10 5 -c 10 5 -l 50 -M 10 seed 1xxx # #quick and cheap, takes about 90 minutes - #python tumbler.py -N 2 1 -c 3 0.001 -l 2 -M 2 seed 1xxx 1yyy + #python tumbler.py -N 2 1 -c 3 0.001 -l 10 -M 3 -a 1 seed 1xxx # #default, good enough for most, takes about 5 hours - #python tumbler.py seed 1 + #python tumbler.py seed 1xxx + # + #for quick testing + #python tumbler.py -N 2 1 -c 3 0.001 -l 0.1 -M 3 -a 0 seed 1xxx 1yyy + wallet = Wallet(seed, max_mix_depth = options.mixdepthsrc + options.mixdepthcount) + common.bc_interface.sync_wallet(wallet) + wallet.print_debug_wallet_info() + + common.nickname = 'tumbler-'+binascii.hexlify(os.urandom(4)) + irc = IRCMessageChannel(common.nickname) + tumbler = Tumbler(irc, wallet, tx_list, options.txfee, options.maxcjfee) + try: + debug('connecting to irc') + irc.run() + except: + debug('CRASHING, DUMPING EVERYTHING') + debug('wallet seed = ' + seed) + debug_dump_object(wallet, ['addr_cache']) + debug_dump_object(tumbler) + import traceback + debug(traceback.format_exc()) + if __name__ == "__main__": main() From 4ab74614681903b6e8469d868e5d1537e3f584b7 Mon Sep 17 00:00:00 2001 From: Belcher Date: Tue, 17 Mar 2015 22:44:15 +0000 Subject: [PATCH 163/409] crashing bugfix --- common.py | 3 --- irc.py | 31 ++++++++++--------------------- yield-generator.py | 3 ++- 3 files changed, 12 insertions(+), 25 deletions(-) diff --git a/common.py b/common.py index c7f5f494..ce369b4e 100644 --- a/common.py +++ b/common.py @@ -8,12 +8,9 @@ from ConfigParser import SafeConfigParser import os nickname = '' -COMMAND_PREFIX = '!' MAX_PRIVMSG_LEN = 400 bc_interface = None ordername_list = ["absorder", "relorder"] -encrypted_commands = ["auth", "ioauth", "tx", "sig"] -plaintext_commands = ["fill", "error", "pubkey", "orderbook", "relorder", "absorder"] debug_file_handle = None config = SafeConfigParser() diff --git a/irc.py b/irc.py index e8bc240e..229f0065 100644 --- a/irc.py +++ b/irc.py @@ -7,8 +7,10 @@ import base64, os import enc_wrapper +COMMAND_PREFIX = '!' PING_INTERVAL = 40 PING_TIMEOUT = 10 +joinmarket_irc_commands = ["auth", "ioauth", "tx", "sig", "fill", "error", "pubkey", "orderbook", "relorder", "absorder"] def get_irc_text(line): return line[line[1:].find(':') + 2:] @@ -135,7 +137,7 @@ def __privmsg(self, nick, cmd, message): for m in message_chunks: trailer = ' ~' if m==message_chunks[-1] else ' ;' header = "PRIVMSG " + nick + " :" - if m==message_chunks[0]: header += '!'+cmd + ' ' + if m==message_chunks[0]: header += COMMAND_PREFIX + cmd + ' ' self.send_raw(header + m + trailer) def send_raw(self, line): @@ -255,26 +257,9 @@ def __get_encryption_box(self, cmd, nick): If so, retrieve the appropriate crypto_box object and return. Sending/receiving flag enables us to check which command strings correspond to which - type of object (maker/taker).''' - - if cmd in plaintext_commands: - return None - elif cmd not in encrypted_commands: - raise Exception("Invalid command type: " + cmd) + type of object (maker/taker).''' #old doc, dont trust return self.cjpeer.get_crypto_box_from_nick(nick) - ''' - maker_strings = ['tx','auth'] if not sending else ['ioauth','sig'] - taker_strings = ['ioauth','sig'] if not sending else ['tx','auth'] - - if cmd in maker_strings: - return self.active_orders[nick].crypto_box - elif cmd in taker_strings: - return self.cjtx.crypto_boxes[nick][1] - else: - raise Exception("Invalid command type: " + cmd) - ''' - def __handle_privmsg(self, source, target, message): nick = get_irc_nick(source) @@ -288,14 +273,18 @@ def __handle_privmsg(self, source, target, message): if target == self.nick: if nick not in self.built_privmsg: + if message[0] != COMMAND_PREFIX: + return #new message starting - cmd_string = ''.join(message.split(' ')[0][1:]) + cmd_string = message[1:].split(' ')[0] + if self.built_privmsg not in joinmarket_irc_commands: + return self.built_privmsg[nick] = [cmd_string, message[:-2]] else: self.built_privmsg[nick][1] += message[:-2] box = self.__get_encryption_box(self.built_privmsg[nick][0], nick) if message[-1]==';': - self.waiting[nick]=True + self.waiting[nick]=True elif message[-1]=='~': self.waiting[nick]=False if box: diff --git a/yield-generator.py b/yield-generator.py index e3a82f0c..6883a641 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -97,8 +97,9 @@ def main(): except: debug('CRASHING, DUMPING EVERYTHING') debug('wallet seed = ' + seed) - debug_dump_object(wallet, ['addr_cache']) + debug_dump_object(wallet, ['addr_cache', 'keys']) debug_dump_object(maker) + debug_dump_object(irc) import traceback debug(traceback.format_exc()) From 3441276d9070583c85b7e9151b80bea06097aa83 Mon Sep 17 00:00:00 2001 From: Belcher Date: Tue, 17 Mar 2015 23:07:13 +0000 Subject: [PATCH 164/409] bugfixes to fix the other bugfixes --- irc.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/irc.py b/irc.py index 229f0065..06a9fce5 100644 --- a/irc.py +++ b/irc.py @@ -10,7 +10,8 @@ COMMAND_PREFIX = '!' PING_INTERVAL = 40 PING_TIMEOUT = 10 -joinmarket_irc_commands = ["auth", "ioauth", "tx", "sig", "fill", "error", "pubkey", "orderbook", "relorder", "absorder"] +encrypted_commands = ["auth", "ioauth", "tx", "sig"] +plaintext_commands = ["fill", "error", "pubkey", "orderbook", "relorder", "absorder"] def get_irc_text(line): return line[line[1:].find(':') + 2:] @@ -258,8 +259,10 @@ def __get_encryption_box(self, cmd, nick): and return. Sending/receiving flag enables us to check which command strings correspond to which type of object (maker/taker).''' #old doc, dont trust - - return self.cjpeer.get_crypto_box_from_nick(nick) + if cmd in plaintext_commands: + return None + else: + return self.cjpeer.get_crypto_box_from_nick(nick) def __handle_privmsg(self, source, target, message): nick = get_irc_nick(source) @@ -277,7 +280,8 @@ def __handle_privmsg(self, source, target, message): return #new message starting cmd_string = message[1:].split(' ')[0] - if self.built_privmsg not in joinmarket_irc_commands: + if cmd_string not in plaintext_commands + encrypted_commands: + debug('cmd not in cmd_list, line="' + message + '"') return self.built_privmsg[nick] = [cmd_string, message[:-2]] else: @@ -299,7 +303,8 @@ def __handle_privmsg(self, source, target, message): debug("< Date: Wed, 18 Mar 2015 22:05:03 +0000 Subject: [PATCH 165/409] added display private key option to wallettool --- wallet-tool.py | 35 ++++++----------------------------- 1 file changed, 6 insertions(+), 29 deletions(-) diff --git a/wallet-tool.py b/wallet-tool.py index d55cf61e..517907ba 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -1,6 +1,6 @@ import bitcoin as btc -from common import Wallet, get_signed_tx, load_program_config +from common import Wallet, get_signed_tx, load_program_config, get_addr_vbyte import common import sys @@ -44,35 +44,10 @@ #seed = '256 bits of randomness' -print_privkey = options.showprivkey - - wallet = Wallet(seed, options.maxmixdepth) common.bc_interface.sync_wallet(wallet, options.gaplimit) -if method == 'display': - total_balance = 0 - for m in range(wallet.max_mix_depth): - print 'mixing depth %d m/0/%d/' % (m, m) - balance_depth = 0 - for forchange in [0, 1]: - print(' ' + ('receive' if forchange==0 else 'change') + - ' addresses m/0/%d/%d/' % (m, forchange)) - for k in range(wallet.index[m][forchange] + options.gaplimit): - addr = wallet.get_addr(m, forchange, k) - balance = 0.0 - for addrvalue in wallet.unspent.values(): - if addr == addrvalue['address']: - balance += addrvalue['value'] - balance_depth += balance - used = ('used' if k < wallet.index[m][forchange] else ' new') - if balance > 0 or used == ' new': - print ' m/0/%d/%d/%02d %s %s %.8fbtc' % (m, forchange, k, addr, used, balance/1e8) - print 'for mixdepth=%d balance=%.8fbtc' % (m, balance_depth/1e8) - total_balance += balance_depth - print 'total balance = %.8fbtc' % (total_balance/1e8) - -if method == 'displayall': +if method == 'display' or method == 'displayall': total_balance = 0 for m in range(wallet.max_mix_depth): print 'mixing depth %d m/0/%d/' % (m, m) @@ -88,11 +63,13 @@ balance += addrvalue['value'] balance_depth += balance used = ('used' if k < wallet.index[m][forchange] else ' new') - print ' m/0/%d/%d/%02d %s %s %.8fbtc' % (m, forchange, k, addr, used, balance/1e8) + privkey = btc.encode_privkey(wallet.get_key(m, forchange, k), 'wif_compressed', + get_addr_vbyte()) if options.showprivkey else '' + if method == 'displayall' or balance > 0 or used == ' new': + print ' m/0/%d/%d/%02d %s %s %.8fbtc %s' % (m, forchange, k, addr, used, balance/1e8, privkey) print 'for mixdepth=%d balance=%.8fbtc' % (m, balance_depth/1e8) total_balance += balance_depth print 'total balance = %.8fbtc' % (total_balance/1e8) - elif method == 'summary': total_balance = 0 for m in range(wallet.max_mix_depth): From c420ca4b49a7c414865e68fbfda29b1830a91ad6 Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 20 Mar 2015 00:12:35 +0000 Subject: [PATCH 166/409] added debug, easier nick selection in yieldgen --- taker.py | 1 + yield-generator.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/taker.py b/taker.py index 8ed6b5d0..d0abb12b 100644 --- a/taker.py +++ b/taker.py @@ -79,6 +79,7 @@ def recv_txio(self, nick, utxo_list, cj_pub, change_addr): self.outputs.append({'address': cj_addr, 'value': self.cj_amount}) self.cjfee_total += real_cjfee if len(self.nonrespondants) > 0: + debug('nonrespondants = ' + str(self.nonrespondants)) return debug('got all parts, enough to build a tx cjfeetotal=' + str(self.cjfee_total)) diff --git a/yield-generator.py b/yield-generator.py index 6883a641..fdd7c1fc 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -11,8 +11,9 @@ from socket import gethostname txfee = 1000 -cjfee = '0.01' # 1% fee +cjfee = '0.002' # 1% fee mix_levels = 5 +nickname = 'yigen-'+binascii.hexlify(os.urandom(4)) nickserv_password = '' minsize = int(2 * txfee / float(cjfee)) #minimum size is such that you always net profit at least the miners fee @@ -88,7 +89,7 @@ def main(): common.bc_interface.sync_wallet(wallet) wallet.print_debug_wallet_info() - common.nickname = 'yigen-'+binascii.hexlify(os.urandom(4)) + common.nickname = nickname irc = IRCMessageChannel(common.nickname) maker = YieldGenerator(irc, wallet) try: From 8aefc96873d91499b2e81c710e4739751200bf3b Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 20 Mar 2015 00:31:15 +0000 Subject: [PATCH 167/409] added debug line to irc.py --- irc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/irc.py b/irc.py index 06a9fce5..43278b5f 100644 --- a/irc.py +++ b/irc.py @@ -273,7 +273,7 @@ def __handle_privmsg(self, source, target, message): ctcp = message[1:endindex + 1] #self.send_raw('PRIVMSG ' + nick + ' :\x01VERSION #TODO ctcp version here, since some servers dont let you get on without - + debug('target="' + target + '" self.nick="' + self.nick + '"') if target == self.nick: if nick not in self.built_privmsg: if message[0] != COMMAND_PREFIX: From 4b935e2d89a9e8d646836bbb4c15bc86ec7bb74d Mon Sep 17 00:00:00 2001 From: Belcher Date: Thu, 26 Mar 2015 01:03:40 +0000 Subject: [PATCH 168/409] split sync_wallet() into sync_addresses() + sync_unspent(), removed TestTaker --- blockchaininterface.py | 27 ++++++++++---- taker.py | 83 ------------------------------------------ 2 files changed, 20 insertions(+), 90 deletions(-) diff --git a/blockchaininterface.py b/blockchaininterface.py index 214eb8d0..2891f9f9 100644 --- a/blockchaininterface.py +++ b/blockchaininterface.py @@ -32,9 +32,18 @@ class BlockchainInterface(object): def __init__(self): pass - @abc.abstractmethod def sync_wallet(self, wallet, gaplimit=6): - '''Finds used addresses and utxos, puts in wallet.index and wallet.unspent''' + self.sync_addresses(wallet, gaplimit) + self.sync_unspent(wallet) + + @abc.abstractmethod + def sync_addresses(self, wallet, gaplimit=6): + '''Finds which addresses have been used and sets wallet.index appropriately''' + pass + + @abc.abstractmethod + def sync_unspent(self, wallet): + '''Finds the unspent transaction outputs belonging to this wallet, sets wallet.unspent''' pass @abc.abstractmethod @@ -57,8 +66,8 @@ def __init__(self, testnet = False): super(BlockrInterface, self).__init__() self.network = 'testnet' if testnet else 'btc' #see bci.py in bitcoin module self.blockr_domain = 'tbtc' if testnet else 'btc' - - def sync_wallet(self, wallet, gaplimit=6): + + def sync_addresses(self, wallet, gaplimit=6): common.debug('downloading wallet history') #sets Wallet internal indexes to be at the next unused address addr_req_count = 20 @@ -86,7 +95,10 @@ def sync_wallet(self, wallet, gaplimit=6): else: wallet.index[mix_depth][forchange] = wallet.addr_cache[last_used_addr][2] + 1 + def sync_unspent(self, wallet): + wallet.unspent = {} #finds utxos in the wallet + addr_req_count = 20 addrs = {} for m in range(wallet.max_mix_depth): @@ -97,8 +109,6 @@ def sync_wallet(self, wallet, gaplimit=6): common.debug('no tx used') return - #TODO handle the case where there are so many addresses it cant - # fit into one api call (>50 or so) i = 0 addrkeys = addrs.keys() while i < len(addrkeys): @@ -305,7 +315,7 @@ def add_watchonly_addresses(self, addr_list, wallet_name): print 'now restart bitcoind with -rescan' sys.exit(0) - def sync_wallet(self, wallet, gaplimit=6): + def sync_addresses(self, wallet, gaplimit=6): common.debug('requesting wallet history') wallet_name = 'joinmarket-wallet-' + btc.dbl_sha256(wallet.keys[0][0])[:6] addr_req_count = 20 @@ -361,6 +371,9 @@ def sync_wallet(self, wallet, gaplimit=6): self.add_watchonly_addresses(wallet_addr_list, wallet_name) return + def sync_unspent(self, wallet): + wallet_name = 'joinmarket-wallet-' + btc.dbl_sha256(wallet.keys[0][0])[:6] + wallet.unspent = {} unspent_list = json.loads(self.rpc(['listunspent'])) for u in unspent_list: if u['account'] != wallet_name: diff --git a/taker.py b/taker.py index d0abb12b..8177aa22 100644 --- a/taker.py +++ b/taker.py @@ -231,89 +231,6 @@ def on_ioauth(self, nick, utxo_list, cj_pub, change_addr, btc_sig): def on_sig(self, nick, sig): self.cjtx.add_signature(sig) -my_tx_fee = 10000 - -class TestTaker(Taker): - def __init__(self, msgchan, wallet): - Taker.__init__(self, msgchan) - self.wallet = wallet - #TODO this is for testing/debugging, should be removed - self.msgchan.debug_on_pubmsg_cmd = self.debug_on_pubmsg_cmd - - def finish_callback(self): - removed_utxos = self.wallet.remove_old_utxos(self.cjtx.latest_tx) - added_utxos = self.wallet.add_new_utxos(self.cjtx.latest_tx, btc.txhash(btc.serialize(self.cjtx.latest_tx))) - debug('tx published, added_utxos=\n' + pprint.pformat(added_utxos)) - debug('removed_utxos=\n' + pprint.pformat(removed_utxos)) - - def debug_on_pubmsg_cmd(self, nick, chunks): - if chunks[0] == '%go': - #!%go [counterparty] [oid] [amount] - cp = chunks[1] - oid = chunks[2] - amt = chunks[3] - #this testing command implements a very dumb algorithm. - #just take 1 utxo from anywhere and output it to a level 1 - #change address. - utxo_dict = self.wallet.get_utxo_list_by_mixdepth() - utxo_list = [x for v in utxo_dict.itervalues() for x in v] - unspent = [{'utxo': utxo, 'value': self.wallet.unspent[utxo]['value']} \ - for utxo in utxo_list] - inputs = btc.select(unspent, amt) - utxos = [i['utxo'] for i in inputs] - print 'making cjtx' - self.cjtx = CoinJoinTX(self, int(amt), {cp: oid}, - utxos, self.wallet.get_receive_addr(mixing_depth=1), - self.wallet.get_change_addr(mixing_depth=0), my_tx_fee, self.finish_callback) - elif chunks[0] == '%unspent': - from pprint import pprint - pprint(self.wallet.unspent) - elif chunks[0] == '%fill': - #!fill [counterparty] [oid] [amount] [utxo] - counterparty = chunks[1] - oid = int(chunks[2]) - amount = chunks[3] - my_utxo = chunks[4] - print 'making cjtx' - self.cjtx = CoinJoinTX(self.msgchan, self.wallet, self.db, int(amount), {counterparty: oid}, - [my_utxo], self.wallet.get_receive_addr(mixing_depth=1), - self.wallet.get_change_addr(mixing_depth=0), my_tx_fee, self.finish_callback) - elif chunks[0] == '%2fill': - #!2fill [amount] [utxo] [counterparty1] [oid1] [counterparty2] [oid2] - amount = int(chunks[1]) - my_utxo = chunks[2] - cp1 = chunks[3] - oid1 = int(chunks[4]) - cp2 = chunks[5] - oid2 = int(chunks[6]) - print 'creating cjtx' - self.cjtx = CoinJoinTX(self.msgchan, self.wallet, self.db, amount, {cp1: oid1, cp2: oid2}, - [my_utxo], self.wallet.get_receive_addr(mixing_depth=1), - self.wallet.get_change_addr(mixing_depth=0), my_tx_fee, self.finish_callback) - -def main(): - import sys - seed = sys.argv[1] #btc.sha256('your brainwallet goes here') - from socket import gethostname - nickname = 'taker-' + btc.sha256(gethostname())[:6] - - wallet = Wallet(seed, max_mix_depth=5) - wallet.sync_wallet() - - from irc import IRCMessageChannel - irc = IRCMessageChannel(nickname) - taker = TestTaker(irc, wallet) - try: - print 'connecting to irc' - irc.run() - except: - debug('CRASHING, DUMPING EVERYTHING') - debug('wallet seed = ' + seed) - debug_dump_object(wallet, ['addr_cache']) - debug_dump_object(taker) - import traceback - traceback.print_exc() - if __name__ == "__main__": main() print('done') From 99dfb1970e83389b6bd05c633c4d25ff6be0845b Mon Sep 17 00:00:00 2001 From: Belcher Date: Thu, 26 Mar 2015 15:35:04 +0000 Subject: [PATCH 169/409] created taker.start_cj() --- patientsendpayment.py | 5 ++--- sendpayment.py | 5 ++--- taker.py | 7 ++++++- tumbler.py | 9 ++++----- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/patientsendpayment.py b/patientsendpayment.py index 98fd290b..d364b4d2 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -37,9 +37,8 @@ def run(self): print 'total amount spent = ' + str(total_amount) utxos = self.tmaker.wallet.select_utxos(self.tmaker.mixdepth, total_amount) - self.tmaker.cjtx = taker.CoinJoinTX(self.tmaker.msgchan, self.tmaker.wallet, - self.tmaker.db, self.tmaker.amount, orders, utxos, self.tmaker.destaddr, - self.tmaker.wallet.get_change_addr(self.tmaker.mixdepth), + self.tmaker.start_cj(self.tmaker.wallet, self.tmaker.amount, orders, utxos, + self.tmaker.destaddr, self.tmaker.wallet.get_change_addr(self.tmaker.mixdepth), self.tmaker.txfee, self.finishcallback) class PatientSendPayment(maker.Maker, taker.Taker): diff --git a/sendpayment.py b/sendpayment.py index 76201ae7..316606fb 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -37,7 +37,7 @@ def run(self): for utxo in utxo_list: total_value += self.taker.wallet.unspent[utxo]['value'] orders, cjamount = choose_sweep_order(self.taker.db, total_value, self.taker.txfee, self.taker.makercount) - self.taker.cjtx = takermodule.CoinJoinTX(self.taker.msgchan, self.taker.wallet, self.taker.db, cjamount, orders, utxo_list, + self.taker.start_cj(self.taker.wallet, cjamount, orders, utxo_list, self.taker.destaddr, None, self.taker.txfee, self.finishcallback) else: orders, total_cj_fee = choose_order(self.taker.db, self.taker.amount, self.taker.makercount) @@ -46,8 +46,7 @@ def run(self): print 'total amount spent = ' + str(total_amount) utxos = self.taker.wallet.select_utxos(self.taker.mixdepth, total_amount) - self.taker.cjtx = takermodule.CoinJoinTX(self.taker.msgchan, self.taker.wallet, - self.taker.db, self.taker.amount, orders, utxos, self.taker.destaddr, + self.taker.start_cjd(self.taker.wallet, self.taker.amount, orders, utxos, self.taker.destaddr, self.taker.wallet.get_change_addr(self.taker.mixdepth), self.taker.txfee, self.finishcallback) diff --git a/taker.py b/taker.py index 8177aa22..b0af980c 100644 --- a/taker.py +++ b/taker.py @@ -213,7 +213,12 @@ def __init__(self, msgchan): #maybe a start_cj_tx() method is needed def get_crypto_box_from_nick(self, nick): - return self.cjtx.crypto_boxes[nick][1] + return self.cjtx.crypto_boxes[nick][1] + + def start_cj(self, wallet, cj_amount, orders, my_utxos, my_cj_addr, my_change_addr, + my_txfee, finishcallback=None): + self.cjtx = CoinJoinTX(self.msgchan, wallet, self.db, cj_amount, orders, + my_utxos, my_cj_addr, my_change_addr, my_txfee, finishcallback) def on_error(self): pass #TODO implement diff --git a/tumbler.py b/tumbler.py index ff167e26..d5dc74a5 100644 --- a/tumbler.py +++ b/tumbler.py @@ -107,8 +107,8 @@ def send_tx(self, tx, sweep, i, l): if sweep: orders, cjamount = choose_sweep_order(self.taker.db, total_value, self.taker.txfee, tx['makercount']) - self.taker.cjtx = takermodule.CoinJoinTX(self.taker.msgchan, self.taker.wallet, self.taker.db, - cjamount, orders, all_utxos, destaddr, None, self.taker.txfee, self.finishcallback) + self.taker.start_cj(self.taker.wallet, cjamount, orders, all_utxos, destaddr, + None, self.taker.txfee, self.finishcallback) else: amount = int(tx['amount_ratio'] * total_value) print 'coinjoining ' + str(amount) @@ -118,9 +118,8 @@ def send_tx(self, tx, sweep, i, l): print 'total amount spent = ' + str(total_amount) utxos = self.taker.wallet.select_utxos(tx['srcmixdepth'], total_amount) - self.taker.cjtx = takermodule.CoinJoinTX(self.taker.msgchan, self.taker.wallet, - self.taker.db, amount, orders, utxos, destaddr, changeaddr, self.taker.txfee, - self.finishcallback) + self.taker.start_cj(self.taker.wallet, amount, orders, utxos, destaddr, + changeaddr, self.taker.txfee, self.finishcallback) print 'that was %d tx out of %d' % (i, l) self.lockcond.acquire() From 3cc52529fffacd7d2d1599a543ea1edf91c32996 Mon Sep 17 00:00:00 2001 From: Belcher Date: Thu, 26 Mar 2015 15:40:07 +0000 Subject: [PATCH 170/409] bugfix in sweep warnings --- taker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taker.py b/taker.py index b0af980c..aadc6738 100644 --- a/taker.py +++ b/taker.py @@ -92,7 +92,7 @@ def recv_txio(self, nick, utxo_list, cj_pub, change_addr): print 'fee breakdown for me totalin=%d txfee=%d cjfee_total=%d => changevalue=%d' % (my_total_in, self.my_txfee, self.cjfee_total, my_change_value) if self.my_change_addr == None: - if my_change_value != 0 or abs(my_change_value) != 1: + if my_change_value != 0 and abs(my_change_value) != 1: #seems you wont always get exactly zero because of integer rounding # so 1 satoshi extra or fewer being spent as miner fees is acceptable print 'WARNING CHANGE NOT BEING USED\nCHANGEVALUE = ' + str(my_change_value) From 0a9e729f4c4e367b0e1fc32fc0ff0e861df1e5b7 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sat, 28 Mar 2015 22:28:39 +0000 Subject: [PATCH 171/409] taker no longer uses wallet.unspent --- .gitignore | 1 + common.py | 29 +++++++++++++++-------------- maker.py | 6 ++++-- patientsendpayment.py | 16 +++++----------- sendpayment.py | 9 +++------ taker.py | 28 +++++++++++++--------------- tumbler.py | 7 ++----- 7 files changed, 43 insertions(+), 53 deletions(-) diff --git a/.gitignore b/.gitignore index c9b568f7..bd95d0df 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.pyc *.swp +*.log diff --git a/common.py b/common.py index ce369b4e..2c86b0b7 100644 --- a/common.py +++ b/common.py @@ -144,7 +144,7 @@ def remove_old_utxos(self, tx): continue removed_utxos[utxo] = self.unspent[utxo] del self.unspent[utxo] - debug('removed utxos, wallet now is \n' + pprint.pformat(self.get_utxo_list_by_mixdepth())) + debug('removed utxos, wallet now is \n' + pprint.pformat(self.get_utxos_by_mixdepth())) return removed_utxos def add_new_utxos(self, tx, txid): @@ -157,40 +157,41 @@ def add_new_utxos(self, tx, txid): utxo = txid + ':' + str(index) added_utxos[utxo] = addrdict self.unspent[utxo] = addrdict - debug('added utxos, wallet now is \n' + pprint.pformat(self.get_utxo_list_by_mixdepth())) + debug('added utxos, wallet now is \n' + pprint.pformat(self.get_utxos_by_mixdepth())) return added_utxos - def get_utxo_list_by_mixdepth(self): + def get_utxos_by_mixdepth(self): ''' returns a list of utxos sorted by different mix levels ''' - pprint.pprint(self.unspent) + debug('wallet.unspent = \n' + pprint.pformat(self.unspent)) mix_utxo_list = {} + for m in range(self.max_mix_depth): + mix_utxo_list[m] = {} for utxo, addrvalue in self.unspent.iteritems(): mixdepth = self.addr_cache[addrvalue['address']][0] if mixdepth not in mix_utxo_list: - mix_utxo_list[mixdepth] = [] - mix_utxo_list[mixdepth].append(utxo) + mix_utxo_list[mixdepth] = {} + mix_utxo_list[mixdepth][utxo] = addrvalue return mix_utxo_list def get_balance_by_mixdepth(self): - mix_utxo_list = self.get_utxo_list_by_mixdepth() mix_balance = {} - for mixdepth, utxo_list in mix_utxo_list.iteritems(): - total_value = 0 - for utxo in utxo_list: - total_value += self.unspent[utxo]['value'] - mix_balance[mixdepth] = total_value + for m in range(self.max_mix_depth): + mix_balance[m] = 0 + for mixdepth, utxos in self.get_utxos_by_mixdepth().iteritems(): + mix_balance[mixdepth] = sum([addrval['value'] for addrval in utxos.values()]) return mix_balance def select_utxos(self, mixdepth, amount): - utxo_list = self.get_utxo_list_by_mixdepth()[mixdepth] + utxo_list = self.get_utxos_by_mixdepth()[mixdepth] unspent = [{'utxo': utxo, 'value': self.unspent[utxo]['value']} for utxo in utxo_list] inputs = btc.select(unspent, amount) debug('for mixdepth=' + str(mixdepth) + ' amount=' + str(amount) + ' selected:') debug(pprint.pformat(inputs)) - return [i['utxo'] for i in inputs] + return dict([(i['utxo'], {'value': i['value'], 'address': + self.unspent[i['utxo']]['address']}) for i in inputs]) def print_debug_wallet_info(self): debug('printing debug wallet information') diff --git a/maker.py b/maker.py index fc6b2019..5a242f19 100644 --- a/maker.py +++ b/maker.py @@ -209,8 +209,10 @@ def on_nick_leave(self, nick): def modify_orders(self, to_cancel, to_announce): debug('modifying orders. to_cancel=' + str(to_cancel) + '\nto_announce=' + str(to_announce)) for oid in to_cancel: - order = [o for o in self.orderlist if o['oid'] == oid][0] - self.orderlist.remove(order) + order = [o for o in self.orderlist if o['oid'] == oid] + if len(order) == 0: + debug('didnt cancel order which doesnt exist, oid=' + str(oid)) + self.orderlist.remove(order[0]) if len(to_cancel) > 0: self.msgchan.cancel_orders(to_cancel) if len(to_announce) > 0: diff --git a/patientsendpayment.py b/patientsendpayment.py index d364b4d2..b2751251 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -1,5 +1,6 @@ from common import * +import common import taker import maker from irc import IRCMessageChannel @@ -7,7 +8,7 @@ from optparse import OptionParser from datetime import timedelta -import threading, time +import threading, time, binascii, os class TakerThread(threading.Thread): def __init__(self, tmaker): @@ -88,10 +89,7 @@ def on_tx_unconfirmed(self, cjorder, balance, removed_utxos): print 'finished sending, exiting..' self.msgchan.shutdown() return ([], []) - utxo_list = self.wallet.get_utxo_list_by_mixdepth()[self.mixdepth] - available_balance = 0 - for utxo in utxo_list: - available_balance += self.wallet.unspent[utxo]['value'] + available_balance = self.wallet.get_balance_by_mixdepth()[self.mixdepth] if available_balance > self.amount: order = {'oid': 0, 'ordertype': 'absorder', 'minsize': 0, 'maxsize': self.amount, 'txfee': self.txfee, 'cjfee': self.cjfee} @@ -142,17 +140,13 @@ def main(): str(timedelta(hours=options.waittime)), options.makercount) wallet = Wallet(seed, options.mixdepth + 1) - wallet.sync_wallet() + common.bc_interface.sync_wallet(wallet) - utxo_list = wallet.get_utxo_list_by_mixdepth()[options.mixdepth] - available_balance = 0 - for utxo in utxo_list: - available_balance += wallet.unspent[utxo]['value'] + available_balance = wallet.get_balance_by_mixdepth()[options.mixdepth] if available_balance < amount: print 'not enough money at mixdepth=%d, exiting' % (options.mixdepth) return - import common, binascii, os common.nickname = 'ppayer-' +binascii.hexlify(os.urandom(4)) irc = IRCMessageChannel(common.nickname) diff --git a/sendpayment.py b/sendpayment.py index 316606fb..df1daa87 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -7,7 +7,7 @@ import bitcoin as btc from optparse import OptionParser -import threading +import threading, pprint #thread which does the buy-side algorithm # chooses which coinjoins to initiate and when @@ -32,10 +32,7 @@ def run(self): return if self.taker.amount == 0: - total_value = 0 - utxo_list = self.taker.wallet.get_utxo_list_by_mixdepth()[self.taker.mixdepth] - for utxo in utxo_list: - total_value += self.taker.wallet.unspent[utxo]['value'] + total_value = self.wallet.get_balance_by_mixdepth()[self.taker.mixdepth] orders, cjamount = choose_sweep_order(self.taker.db, total_value, self.taker.txfee, self.taker.makercount) self.taker.start_cj(self.taker.wallet, cjamount, orders, utxo_list, self.taker.destaddr, None, self.taker.txfee, self.finishcallback) @@ -46,7 +43,7 @@ def run(self): print 'total amount spent = ' + str(total_amount) utxos = self.taker.wallet.select_utxos(self.taker.mixdepth, total_amount) - self.taker.start_cjd(self.taker.wallet, self.taker.amount, orders, utxos, self.taker.destaddr, + self.taker.start_cj(self.taker.wallet, self.taker.amount, orders, utxos, self.taker.destaddr, self.taker.wallet.get_change_addr(self.taker.mixdepth), self.taker.txfee, self.finishcallback) diff --git a/taker.py b/taker.py index aadc6738..42e90316 100644 --- a/taker.py +++ b/taker.py @@ -9,7 +9,7 @@ class CoinJoinTX(object): #soon the taker argument will be removed and just be replaced by wallet or some other interface - def __init__(self, msgchan, wallet, db, cj_amount, orders, my_utxos, my_cj_addr, + def __init__(self, msgchan, wallet, db, cj_amount, orders, input_utxos, my_cj_addr, my_change_addr, my_txfee, finishcallback=None): ''' if my_change is None then there wont be a change address @@ -23,8 +23,8 @@ def __init__(self, msgchan, wallet, db, cj_amount, orders, my_utxos, my_cj_addr, self.cj_amount = cj_amount self.active_orders = dict(orders) self.nonrespondants = list(orders.keys()) - self.my_utxos = my_utxos - self.utxos = {None: my_utxos} #None means they belong to me + self.input_utxos = input_utxos + self.utxos = {None: input_utxos.keys()} #None means they belong to me self.finishcallback = finishcallback self.my_txfee = my_txfee self.outputs = [{'address': my_cj_addr, 'value': self.cj_amount}] @@ -35,7 +35,7 @@ def __init__(self, msgchan, wallet, db, cj_amount, orders, my_utxos, my_cj_addr, self.kp = enc_wrapper.init_keypair() self.crypto_boxes = {} #find the btc pubkey of the first utxo being used - self.signing_btc_add = wallet.unspent[self.my_utxos[0]]['address'] + self.signing_btc_add = self.input_utxos.itervalues().next()['address'] self.signing_btc_pub = btc.privtopub(wallet.get_key_from_addr(self.signing_btc_add)) self.msgchan.fill_orders(orders, cj_amount, self.kp.hex_pk()) @@ -45,8 +45,8 @@ def start_encryption(self, nick, maker_pk): self.crypto_boxes[nick] = [maker_pk, enc_wrapper.as_init_encryption(\ self.kp, enc_wrapper.init_pubkey(maker_pk))] #send authorisation request - my_btc_priv = self.wallet.get_key_from_addr(\ - self.wallet.unspent[self.my_utxos[0]]['address']) + my_btc_addr = self.input_utxos.itervalues().next()['address'] + my_btc_priv = self.wallet.get_key_from_addr(my_btc_addr) my_btc_pub = btc.privtopub(my_btc_priv) my_btc_sig = btc.ecdsa_sign(self.kp.hex_pk(), my_btc_priv) self.msgchan.send_auth(nick, my_btc_pub, my_btc_sig) @@ -84,9 +84,9 @@ def recv_txio(self, nick, utxo_list, cj_pub, change_addr): debug('got all parts, enough to build a tx cjfeetotal=' + str(self.cjfee_total)) my_total_in = 0 - for u in self.my_utxos: - usvals = self.wallet.unspent[u] - my_total_in += int(usvals['value']) + for u, va in self.input_utxos.iteritems(): + my_total_in += va['value'] + #my_total_in = sum([va['value'] for u, va in self.input_utxos.iteritems()]) my_change_value = my_total_in - self.cj_amount - self.cjfee_total - self.my_txfee print 'fee breakdown for me totalin=%d txfee=%d cjfee_total=%d => changevalue=%d' % (my_total_in, @@ -108,11 +108,9 @@ def recv_txio(self, nick, utxo_list, cj_pub, change_addr): #now sign it ourselves here for index, ins in enumerate(btc.deserialize(tx)['ins']): utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) - if utxo not in self.my_utxos: + if utxo not in self.input_utxos.keys(): continue - if utxo not in self.wallet.unspent: - continue - addr = self.wallet.unspent[utxo]['address'] + addr = self.input_utxos[utxo]['address'] tx = btc.sign(tx, index, self.wallet.get_key_from_addr(addr)) self.latest_tx = btc.deserialize(tx) @@ -215,10 +213,10 @@ def __init__(self, msgchan): def get_crypto_box_from_nick(self, nick): return self.cjtx.crypto_boxes[nick][1] - def start_cj(self, wallet, cj_amount, orders, my_utxos, my_cj_addr, my_change_addr, + def start_cj(self, wallet, cj_amount, orders, input_utxos, my_cj_addr, my_change_addr, my_txfee, finishcallback=None): self.cjtx = CoinJoinTX(self.msgchan, wallet, self.db, cj_amount, orders, - my_utxos, my_cj_addr, my_change_addr, my_txfee, finishcallback) + input_utxos, my_cj_addr, my_change_addr, my_txfee, finishcallback) def on_error(self): pass #TODO implement diff --git a/tumbler.py b/tumbler.py index d5dc74a5..11017aa3 100644 --- a/tumbler.py +++ b/tumbler.py @@ -87,11 +87,6 @@ def finishcallback(self, coinjointx): self.taker.wallet.remove_old_utxos(coinjointx.latest_tx) def send_tx(self, tx, sweep, i, l): - total_value = 0 - all_utxos = self.taker.wallet.get_utxo_list_by_mixdepth()[tx['srcmixdepth']] - for utxo in all_utxos: - total_value += self.taker.wallet.unspent[utxo]['value'] - destaddr = None changeaddr = None if tx['dest'] == 'internal': @@ -104,6 +99,7 @@ def send_tx(self, tx, sweep, i, l): destaddr = tx['dest'] changeaddr = self.taker.wallet.get_change_addr(tx['srcmixdepth']) + total_value = self.wallet.get_balance_by_mixdepth()[tx['srcmixdepth']] if sweep: orders, cjamount = choose_sweep_order(self.taker.db, total_value, self.taker.txfee, tx['makercount']) @@ -111,6 +107,7 @@ def send_tx(self, tx, sweep, i, l): None, self.taker.txfee, self.finishcallback) else: amount = int(tx['amount_ratio'] * total_value) + #ERROR TODO this is wrong, should be the ratio times initial amount, not running total amount print 'coinjoining ' + str(amount) orders, total_cj_fee = choose_order(self.taker.db, amount, tx['makercount']) print 'chosen orders to fill ' + str(orders) + ' totalcjfee=' + str(total_cj_fee) From 94c1af2d996cc7646948daa9692d2d510db2df2c Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 29 Mar 2015 00:40:21 +0000 Subject: [PATCH 172/409] fixed bug in tumbler where it wouldnt follow the amount ratio --- tumbler.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/tumbler.py b/tumbler.py index 11017aa3..3896f032 100644 --- a/tumbler.py +++ b/tumbler.py @@ -86,35 +86,30 @@ def finishcallback(self, coinjointx): self.unconfirm_callback, self.confirm_callback) self.taker.wallet.remove_old_utxos(coinjointx.latest_tx) - def send_tx(self, tx, sweep, i, l): + def send_tx(self, tx, balance, sweep, i, l): destaddr = None - changeaddr = None if tx['dest'] == 'internal': destaddr = self.taker.wallet.get_receive_addr(tx['srcmixdepth'] + 1) - changeaddr = self.taker.wallet.get_change_addr(tx['srcmixdepth']) elif tx['dest'] == 'addrask': destaddr = raw_input('insert new address: ') - changeaddr = self.taker.wallet.get_change_addr(tx['srcmixdepth']) else: destaddr = tx['dest'] - changeaddr = self.taker.wallet.get_change_addr(tx['srcmixdepth']) - total_value = self.wallet.get_balance_by_mixdepth()[tx['srcmixdepth']] if sweep: - + total_value = self.wallet.get_balance_by_mixdepth()[tx['srcmixdepth']] orders, cjamount = choose_sweep_order(self.taker.db, total_value, self.taker.txfee, tx['makercount']) self.taker.start_cj(self.taker.wallet, cjamount, orders, all_utxos, destaddr, None, self.taker.txfee, self.finishcallback) else: - amount = int(tx['amount_ratio'] * total_value) - #ERROR TODO this is wrong, should be the ratio times initial amount, not running total amount + amount = int(tx['amount_ratio'] * balance) + changeaddr = self.taker.wallet.get_change_addr(tx['srcmixdepth']) print 'coinjoining ' + str(amount) orders, total_cj_fee = choose_order(self.taker.db, amount, tx['makercount']) print 'chosen orders to fill ' + str(orders) + ' totalcjfee=' + str(total_cj_fee) total_amount = amount + total_cj_fee + self.taker.txfee print 'total amount spent = ' + str(total_amount) - utxos = self.taker.wallet.select_utxos(tx['srcmixdepth'], total_amount) + utxos = self.taker.wallet.select_utxos(tx['srcmixdepth'], amount) self.taker.start_cj(self.taker.wallet, amount, orders, utxos, destaddr, changeaddr, self.taker.txfee, self.finishcallback) @@ -137,12 +132,16 @@ def run(self): self.lockcond = threading.Condition() + self.balance_by_mixdepth = {} for i, tx in enumerate(self.taker.tx_list): + if tx['srcmixdepth'] not in self.balance_by_mixdepth: + self.balance_by_mixdepth[tx['srcmixdepth']] = self.wallet.get_balance_by_mixdepth()[tx['srcmixdepth']] sweep = True for later_tx in self.taker.tx_list[i + 1:]: if later_tx['srcmixdepth'] == tx['srcmixdepth']: sweep = False - self.send_tx(tx, sweep, i, len(self.taker.tx_list)) + self.send_tx(tx, sweep, self.balance_by_mixdepth[tx['srcmixdepth']], + i, len(self.taker.tx_list)) print 'total finished' self.taker.msgchan.shutdown() From bd3a6fec323aadf1f3bca3cf69f5e139182bfe5d Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 29 Mar 2015 00:46:02 +0000 Subject: [PATCH 173/409] removed some hangover code --- taker.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/taker.py b/taker.py index 42e90316..f433b96f 100644 --- a/taker.py +++ b/taker.py @@ -34,10 +34,6 @@ def __init__(self, msgchan, wallet, db, cj_amount, orders, input_utxos, my_cj_ad #create DH keypair on the fly for this Tx object self.kp = enc_wrapper.init_keypair() self.crypto_boxes = {} - #find the btc pubkey of the first utxo being used - self.signing_btc_add = self.input_utxos.itervalues().next()['address'] - self.signing_btc_pub = btc.privtopub(wallet.get_key_from_addr(self.signing_btc_add)) - self.msgchan.fill_orders(orders, cj_amount, self.kp.hex_pk()) def start_encryption(self, nick, maker_pk): if nick not in self.active_orders.keys(): From 535627b053e4c765eecc981a366f581e31028b91 Mon Sep 17 00:00:00 2001 From: Belcher Date: Mon, 30 Mar 2015 20:23:55 +0100 Subject: [PATCH 174/409] adjustments in maker's handling of unspent in preparation for sync_unspent() more often --- blockchaininterface.py | 12 +++++++++++- maker.py | 13 +++++-------- taker.py | 1 + 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/blockchaininterface.py b/blockchaininterface.py index 2891f9f9..5e9647d5 100644 --- a/blockchaininterface.py +++ b/blockchaininterface.py @@ -66,6 +66,7 @@ def __init__(self, testnet = False): super(BlockrInterface, self).__init__() self.network = 'testnet' if testnet else 'btc' #see bci.py in bitcoin module self.blockr_domain = 'tbtc' if testnet else 'btc' + self.last_sync_unspent = 0 def sync_addresses(self, wallet, gaplimit=6): common.debug('downloading wallet history') @@ -96,6 +97,11 @@ def sync_addresses(self, wallet, gaplimit=6): wallet.index[mix_depth][forchange] = wallet.addr_cache[last_used_addr][2] + 1 def sync_unspent(self, wallet): + st = time.time() + rate_limit_time = 10*60 #dont refresh unspent dict more often than 10 minutes + if st - self.last_sync_unspent < rate_limit_time: + common.debug('blockr sync_unspent() happened too recently (%dsec), skipping' % (st - self.last_sync_unspent)) + return wallet.unspent = {} #finds utxos in the wallet addr_req_count = 20 @@ -108,7 +114,6 @@ def sync_unspent(self, wallet): if len(addrs) == 0: common.debug('no tx used') return - i = 0 addrkeys = addrs.keys() while i < len(addrkeys): @@ -129,6 +134,8 @@ def sync_unspent(self, wallet): for u in dat['unspent']: wallet.unspent[u['tx']+':'+str(u['n'])] = {'address': dat['address'], 'value': int(u['amount'].replace('.', ''))} + self.last_sync_unspent = time.time() + common.debug('blockr sync_unspent took ' + str((self.last_sync_unspent - st)) + 'sec') def add_tx_notify(self, txd, unconfirmfun, confirmfun): unconfirm_timeout = 5*60 #seconds @@ -372,6 +379,7 @@ def sync_addresses(self, wallet, gaplimit=6): return def sync_unspent(self, wallet): + st = time.time() wallet_name = 'joinmarket-wallet-' + btc.dbl_sha256(wallet.keys[0][0])[:6] wallet.unspent = {} unspent_list = json.loads(self.rpc(['listunspent'])) @@ -382,6 +390,8 @@ def sync_unspent(self, wallet): continue wallet.unspent[u['txid'] + ':' + str(u['vout'])] = {'address': u['address'], 'value': int(u['amount']*1e8)} + et = time.time() + common.debug('bitcoind sync_unspent took ' + str((et - st)) + 'sec') def add_tx_notify(self, txd, unconfirmfun, confirmfun): if not self.notifythread: diff --git a/maker.py b/maker.py index 5a242f19..472c67be 100644 --- a/maker.py +++ b/maker.py @@ -49,7 +49,7 @@ def auth_counterparty(self, nick, i_utxo_pubkey, btc_sig): btc_key = self.maker.wallet.get_key_from_addr(self.cj_addr) btc_pub = btc.privtopub(btc_key) btc_sig = btc.ecdsa_sign(self.kp.hex_pk(), btc_key) - self.maker.msgchan.send_ioauth(nick, self.utxos, btc_pub, self.change_addr, btc_sig) + self.maker.msgchan.send_ioauth(nick, self.utxos.keys(), btc_pub, self.change_addr, btc_sig) return True def recv_tx(self, nick, txhex): @@ -67,9 +67,9 @@ def recv_tx(self, nick, txhex): sigs = [] for index, ins in enumerate(self.tx['ins']): utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) - if utxo not in self.maker.wallet.unspent: + if utxo not in self.utxos: continue - addr = self.maker.wallet.unspent[utxo]['address'] + addr = self.utxos[utxo]['address'] txs = btc.sign(txhex, index, self.maker.wallet.get_key_from_addr(addr)) sigs.append(base64.b64encode(btc.deserialize(txs)['ins'][index]['script'].decode('hex'))) #len(sigs) > 0 guarenteed since i did verify_unsigned_tx() @@ -108,17 +108,14 @@ def verify_unsigned_tx(self, txd): in [get_addr_from_utxo(i['outpoint']['hash'], i['outpoint']['index']) \ for i in txd['ins']]: return False, "authenticating bitcoin address is not contained" - my_utxo_set = set(self.utxos) + my_utxo_set = set(self.utxos.keys()) wallet_utxos = set(self.maker.wallet.unspent) if not tx_utxo_set.issuperset(my_utxo_set): return False, 'my utxos are not contained' if not wallet_utxos.issuperset(my_utxo_set): return False, 'my utxos already spent' - my_total_in = 0 - for u in self.utxos: - usvals = self.maker.wallet.unspent[u] - my_total_in += usvals['value'] + my_total_in = sum([va['value'] for va in self.utxos.values()]) real_cjfee = calc_cj_fee(self.ordertype, self.cjfee, self.cj_amount) expected_change_value = (my_total_in - self.cj_amount - self.txfee + real_cjfee) diff --git a/taker.py b/taker.py index f433b96f..4ee5f847 100644 --- a/taker.py +++ b/taker.py @@ -34,6 +34,7 @@ def __init__(self, msgchan, wallet, db, cj_amount, orders, input_utxos, my_cj_ad #create DH keypair on the fly for this Tx object self.kp = enc_wrapper.init_keypair() self.crypto_boxes = {} + self.msgchan.fill_orders(orders, cj_amount, self.kp.hex_pk()) def start_encryption(self, nick, maker_pk): if nick not in self.active_orders.keys(): From 88a1a3a8c4d1d359312b079543a2063873c96c84 Mon Sep 17 00:00:00 2001 From: Belcher Date: Tue, 31 Mar 2015 21:11:30 +0100 Subject: [PATCH 175/409] maker now does sync_unspent() periodically, alerts in json-rpc mode are handled correctly --- blockchaininterface.py | 15 ++++++--------- common.py | 3 +++ irc.py | 1 - maker.py | 8 ++++---- patientsendpayment.py | 4 ++-- sendpayment.py | 3 ++- yield-generator.py | 2 +- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/blockchaininterface.py b/blockchaininterface.py index 5e9647d5..a5dfdf6e 100644 --- a/blockchaininterface.py +++ b/blockchaininterface.py @@ -138,9 +138,9 @@ def sync_unspent(self, wallet): common.debug('blockr sync_unspent took ' + str((self.last_sync_unspent - st)) + 'sec') def add_tx_notify(self, txd, unconfirmfun, confirmfun): - unconfirm_timeout = 5*60 #seconds + unconfirm_timeout = 10*60 #seconds unconfirm_poll_period = 5 - confirm_timeout = 120*60 + confirm_timeout = 2*60*60 confirm_poll_period = 5*60 class NotifyThread(threading.Thread): def __init__(self, blockr_domain, txd, unconfirmfun, confirmfun): @@ -183,7 +183,7 @@ def run(self): data = [data] for txinfo in data: outs = set([(sv['script'], sv['value']) for sv in btc.deserialize(txinfo['tx']['hex'])['outs']]) - print 'outs = ' + str(outs) + common.debug('unconfirm query outs = ' + str(outs)) if outs == self.tx_output_set: unconfirmed_txid = txinfo['tx']['txid'] unconfirmed_txhex = txinfo['tx']['hex'] @@ -217,7 +217,7 @@ def run(self): data = [data] for txinfo in data: outs = set([(sv['script'], sv['value']) for sv in btc.deserialize(txinfo['tx']['hex'])['outs']]) - print 'outs = ' + str(outs) + common.debug('confirm query outs = ' + str(outs)) if outs == self.tx_output_set: confirmed_txid = txinfo['tx']['txid'] confirmed_txhex = txinfo['tx']['hex'] @@ -244,7 +244,6 @@ def __init__(self, request, client_address, base_server): SimpleHTTPServer.SimpleHTTPRequestHandler.__init__(self, request, client_address, base_server) def do_HEAD(self): - print 'httpd received HEAD ' + self.path + ' request' pages = ('/walletnotify?', '/alertnotify?') if not self.path.startswith(pages): return @@ -252,7 +251,6 @@ def do_HEAD(self): txid = self.path[len(pages[0]):] txd = btc.deserialize(self.btcinterface.fetchtx(txid)) tx_output_set = set([(sv['script'], sv['value']) for sv in txd['outs']]) - print 'outs = ' + str(tx_output_set) unconfirmfun, confirmfun = None, None for tx_out, ucfun, cfun in self.btcinterface.txnotify_fun: @@ -271,9 +269,8 @@ def do_HEAD(self): self.btcinterface.txnotify_fun.remove((tx_out, unconfirmfun, confirmfun)) elif self.path.startswith('/alertnotify?'): - message = self.path[len(pages[1]):] - print 'got an alert, shit, shutting down. message=' + message - sys.exit(0) + common.alert_message = self.path[len(pages[1]):] + common.debug('Got an alert!\nMessage=' + common.alert_message) self.send_response(200) #self.send_header('Connection', 'close') self.end_headers() diff --git a/common.py b/common.py index 2c86b0b7..0d85e3d1 100644 --- a/common.py +++ b/common.py @@ -12,6 +12,7 @@ bc_interface = None ordername_list = ["absorder", "relorder"] debug_file_handle = None +alert_message = None config = SafeConfigParser() config_location = os.path.join(os.path.dirname(os.path.realpath(__file__)),'joinmarket.cfg') @@ -43,6 +44,8 @@ def debug(msg): if nickname and not debug_file_handle: debug_file_handle = open(nickname+'.log','ab') outmsg = datetime.datetime.now().strftime("[%Y/%m/%d %H:%M:%S] ") + msg + if alert_message: + print 'Alert Message: ' + alert_message print outmsg if nickname: #debugs before creating bot nick won't be handled like this debug_file_handle.write(outmsg + '\n') diff --git a/irc.py b/irc.py index 43278b5f..aa13bfe9 100644 --- a/irc.py +++ b/irc.py @@ -273,7 +273,6 @@ def __handle_privmsg(self, source, target, message): ctcp = message[1:endindex + 1] #self.send_raw('PRIVMSG ' + nick + ' :\x01VERSION #TODO ctcp version here, since some servers dont let you get on without - debug('target="' + target + '" self.nick="' + self.nick + '"') if target == self.nick: if nick not in self.built_privmsg: if message[0] != COMMAND_PREFIX: diff --git a/maker.py b/maker.py index 472c67be..2e6d8ef7 100644 --- a/maker.py +++ b/maker.py @@ -92,12 +92,12 @@ def unconfirm_callback(self, txd, txid): def confirm_callback(self, txd, txid, confirmations): self.maker.wallet_unspent_lock.acquire() try: - added_utxos = self.maker.wallet.add_new_utxos(self.tx, txid) + common.bc_interface.sync_unspent(self.maker.wallet) finally: self.maker.wallet_unspent_lock.release() - debug('tx in a block, added_utxos=\n' + pprint.pformat(added_utxos)) + debug('tx in a block') to_cancel, to_announce = self.maker.on_tx_confirmed(self, - confirmations, txid, added_utxos) + confirmations, txid) self.maker.modify_orders(to_cancel, to_announce) def verify_unsigned_tx(self, txd): @@ -284,7 +284,7 @@ def on_tx_unconfirmed(self, cjorder, txid, removed_utxos): #must return which orders to cancel or recreate # and i have to think about how that will work for both # the blockchain explorer api method and the bitcoid walletnotify - def on_tx_confirmed(self, cjorder, confirmations, txid, added_utxos): + def on_tx_confirmed(self, cjorder, confirmations, txid): to_announce = [] for i, out in enumerate(cjorder.tx['outs']): addr = btc.script_to_address(out['script'], get_addr_vbyte()) diff --git a/patientsendpayment.py b/patientsendpayment.py index b2751251..83eeb8f3 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -90,7 +90,7 @@ def on_tx_unconfirmed(self, cjorder, balance, removed_utxos): self.msgchan.shutdown() return ([], []) available_balance = self.wallet.get_balance_by_mixdepth()[self.mixdepth] - if available_balance > self.amount: + if available_balance >= self.amount: order = {'oid': 0, 'ordertype': 'absorder', 'minsize': 0, 'maxsize': self.amount, 'txfee': self.txfee, 'cjfee': self.cjfee} return ([], [order]) @@ -98,7 +98,7 @@ def on_tx_unconfirmed(self, cjorder, balance, removed_utxos): debug('not enough money left, have to wait until tx confirms') return ([0], []) - def on_tx_confirmed(self, cjorder, confirmations, txid, balance, added_utxos): + def on_tx_confirmed(self, cjorder, confirmations, txid, balance): if len(self.orderlist) == 0: order = {'oid': 0, 'ordertype': 'absorder', 'minsize': 0, 'maxsize': self.amount, 'txfee': self.txfee, 'cjfee': self.cjfee} diff --git a/sendpayment.py b/sendpayment.py index df1daa87..ca2655d0 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -32,7 +32,8 @@ def run(self): return if self.taker.amount == 0: - total_value = self.wallet.get_balance_by_mixdepth()[self.taker.mixdepth] + utxo_list = self.taker.wallet.get_utxos_by_mixdepth()[self.taker.mixdepth] + total_value = sum([va['value'] for va in utxo_list.values()]) orders, cjamount = choose_sweep_order(self.taker.db, total_value, self.taker.txfee, self.taker.makercount) self.taker.start_cj(self.taker.wallet, cjamount, orders, utxo_list, self.taker.destaddr, None, self.taker.txfee, self.finishcallback) diff --git a/yield-generator.py b/yield-generator.py index fdd7c1fc..721ad273 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -78,7 +78,7 @@ def on_tx_unconfirmed(self, cjorder, txid, removed_utxos): #announce new order, replacing the old order return ([], [neworders[0]]) - def on_tx_confirmed(self, cjorder, confirmations, txid, added_utxos): + def on_tx_confirmed(self, cjorder, confirmations, txid): return self.on_tx_unconfirmed(None, None, None) def main(): From 6ae4ae81006f8e47c9c7a06a3518304590efd01c Mon Sep 17 00:00:00 2001 From: Belcher Date: Tue, 31 Mar 2015 21:48:09 +0100 Subject: [PATCH 176/409] irc bot in the realname field says which blockchain interface it uses --- yield-generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yield-generator.py b/yield-generator.py index 721ad273..2c3eb453 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -90,7 +90,7 @@ def main(): wallet.print_debug_wallet_info() common.nickname = nickname - irc = IRCMessageChannel(common.nickname) + irc = IRCMessageChannel(common.nickname, realname='btcint=' + common.config.get("BLOCKCHAIN", "blockchain_source")) maker = YieldGenerator(irc, wallet) try: debug('connecting to irc') From a4ba0a5d96a8e8e24353266ebf8c955a6f2c66c0 Mon Sep 17 00:00:00 2001 From: Belcher Date: Wed, 1 Apr 2015 01:11:13 +0100 Subject: [PATCH 177/409] created depth chart in gui-taker, fixed kick bug in irc.py --- gui-taker.py | 53 ++++++++++++++++++++++++++++++++++++++++------ irc.py | 8 +++++-- yield-generator.py | 2 +- 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/gui-taker.py b/gui-taker.py index da13cf23..28d9cb34 100644 --- a/gui-taker.py +++ b/gui-taker.py @@ -7,7 +7,7 @@ from decimal import Decimal import io -import base64 +import base64, time tableheading = ''' @@ -23,15 +23,35 @@ ''' shutdownform = '' - shutdownpage = '

    Successfully Shut down

    ' +refresh_orderbook_form = '' + def calc_depth_data(db, value): pass def calc_order_size_data(db): return ordersizes +def create_depth_chart(db, cj_amount): + try: + import matplotlib.pyplot as plt + except ImportError: + return 'Install matplotlib to see graphs' + sqlorders = db.execute('SELECT * FROM orderbook;').fetchall() + orderfees = [calc_cj_fee(o['ordertype'], o['cjfee'], cj_amount)/1e8 + for o in sqlorders if cj_amount >= o['minsize'] and cj_amount <= o['maxsize']] + + if len(orderfees) == 0: + return 'No orders at amount ' + str(cj_amount/1e8) + fig = plt.figure() + plt.hist(orderfees, 30, histtype='bar', rwidth=0.8) + plt.grid() + plt.title('CoinJoin Orderbook Depth Chart for amount=' + str(cj_amount/1e8) + 'btc') + plt.xlabel('CoinJoin Fee / btc') + plt.ylabel('Frequency') + return get_graph_html(fig) + def create_size_histogram(db): try: import matplotlib.pyplot as plt @@ -99,7 +119,7 @@ def get_counterparty_count(self): def do_GET(self): #SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self) #print 'httpd received ' + self.path + ' request' - pages = ['/', '/ordersize'] + pages = ['/', '/ordersize', '/depth'] if self.path not in pages: return fd = open('orderbook.html', 'r') @@ -108,19 +128,30 @@ def do_GET(self): if self.path == '/': ordercount, ordertable = self.create_orderbook_table() replacements = { - 'PAGETITLE': 'Joinmarket Browser Interface', - 'MAINHEADING': 'Joinmarket Orderbook', + 'PAGETITLE': 'JoinMarket Browser Interface', + 'MAINHEADING': 'JoinMarket Orderbook', 'SECONDHEADING': (str(ordercount) + ' orders found by ' + self.get_counterparty_count() + ' counterparties'), - 'MAINBODY': shutdownform + tableheading + ordertable + '
    \n' + 'MAINBODY': refresh_orderbook_form + shutdownform + tableheading + ordertable + '\n' } elif self.path == '/ordersize': replacements = { - 'PAGETITLE': 'Joinmarket Browser Interface', + 'PAGETITLE': 'JoinMarket Browser Interface', 'MAINHEADING': 'Order Sizes', 'SECONDHEADING': 'Order Size Histogram', 'MAINBODY': create_size_histogram(self.taker.db) } + elif self.path.startswith('/depth'): + #if self.path[6] == '?': + # quantity = + cj_amounts = [10**cja for cja in range(4, 10, 1)] + mainbody = [create_depth_chart(self.taker.db, cja) for cja in cj_amounts] + replacements = { + 'PAGETITLE': 'JoinMarket Browser Interface', + 'MAINHEADING': 'Depth Chart', + 'SECONDHEADING': 'Orderbook Depth', + 'MAINBODY': '
    '.join(mainbody) + } orderbook_page = orderbook_fmt for key, rep in replacements.iteritems(): orderbook_page = orderbook_page.replace(key, rep) @@ -131,6 +162,9 @@ def do_GET(self): self.wfile.write(orderbook_page) def do_POST(self): + pages = ['/shutdown', '/refreshorderbook'] + if self.path not in pages: + return if self.path == '/shutdown': self.taker.msgchan.shutdown() self.send_response(200) @@ -139,6 +173,11 @@ def do_POST(self): self.end_headers() self.wfile.write(shutdownpage) self.base_server.__shutdown_request = True + elif self.path == '/refreshorderbook': + self.taker.msgchan.request_orderbook() + time.sleep(5) + self.path = '/' + self.do_GET() class HTTPDThread(threading.Thread): def __init__(self, taker): diff --git a/irc.py b/irc.py index aa13bfe9..ded1345d 100644 --- a/irc.py +++ b/irc.py @@ -347,8 +347,12 @@ def __handle_line(self, line): elif chunks[1] == 'KICK': target = chunks[3] nick = get_irc_nick(chunks[0]) - if self.on_nick_leave: - self.on_nick_leave(nick) + if target == self.nick: + self.give_up = True + raise IOError(get_irc_nick(chunks[0]) + ' has kicked us from the irc channel! Reason=' + get_irc_text(line)) + else: + if self.on_nick_leave: + self.on_nick_leave(target) elif chunks[1] == 'PART': nick = get_irc_nick(chunks[0]) if self.on_nick_leave: diff --git a/yield-generator.py b/yield-generator.py index 2c3eb453..0e4c8a4c 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -15,7 +15,7 @@ mix_levels = 5 nickname = 'yigen-'+binascii.hexlify(os.urandom(4)) nickserv_password = '' -minsize = int(2 * txfee / float(cjfee)) #minimum size is such that you always net profit at least the miners fee +minsize = int(1.2 * txfee / float(cjfee)) #minimum size is such that you always net profit at least 20% of the miner fee From 26b3154032404d8244dcf1e3fdce58dc0b0bde82 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 5 Apr 2015 01:32:25 +0100 Subject: [PATCH 178/409] improve speed of blockr sync_unspent() --- blockchaininterface.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/blockchaininterface.py b/blockchaininterface.py index a5dfdf6e..c32f1055 100644 --- a/blockchaininterface.py +++ b/blockchaininterface.py @@ -97,28 +97,23 @@ def sync_addresses(self, wallet, gaplimit=6): wallet.index[mix_depth][forchange] = wallet.addr_cache[last_used_addr][2] + 1 def sync_unspent(self, wallet): + #finds utxos in the wallet st = time.time() rate_limit_time = 10*60 #dont refresh unspent dict more often than 10 minutes if st - self.last_sync_unspent < rate_limit_time: common.debug('blockr sync_unspent() happened too recently (%dsec), skipping' % (st - self.last_sync_unspent)) return wallet.unspent = {} - #finds utxos in the wallet addr_req_count = 20 - addrs = {} - for m in range(wallet.max_mix_depth): - for forchange in [0, 1]: - for n in range(wallet.index[m][forchange]): - addrs[wallet.get_addr(m, forchange, n)] = m + addrs = wallet.addr_cache.keys() if len(addrs) == 0: common.debug('no tx used') return i = 0 - addrkeys = addrs.keys() - while i < len(addrkeys): - inc = min(len(addrkeys) - i, addr_req_count) - req = addrkeys[i:i + inc] + while i < len(addrs): + inc = min(len(addrs) - i, addr_req_count) + req = addrs[i:i + inc] i += inc #TODO send a pull request to pybitcointools From 6a4218acfcb87f0672bcd07c485b4d14085b8887 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 5 Apr 2015 01:58:54 +0100 Subject: [PATCH 179/409] first implementation of randomized weighted order choosing --- common.py | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/common.py b/common.py index 0d85e3d1..2854dfe8 100644 --- a/common.py +++ b/common.py @@ -2,8 +2,8 @@ import bitcoin as btc from decimal import Decimal from math import factorial -import sys, datetime, json, time, pprint -import threading +import sys, datetime, json, time, pprint, threading +import numpy as np import blockchaininterface from ConfigParser import SafeConfigParser import os @@ -222,6 +222,19 @@ def calc_total_input_value(utxos): input_sum += int(btc.deserialize(tx)['outs'][int(utxo[65:])]['value']) return input_sum +def weighted_order_choose(orders, n, feekey): + minfee = feekey(orders[0]) + M = 2*n + if len(orders) > M: + phi = feekey(orders[M]) - minfee + else: + phi = feekey(orders[-1]) - minfee + fee = np.array([feekey(o) for o in orders]) + weight = np.exp(-(1.0*fee - minfee) / phi) + weight /= sum(weight) + chosen_order_index = np.random.choice(len(orders), p=weight) + return orders[chosen_order_index] + def choose_order(db, cj_amount, n): sqlorders = db.execute('SELECT * FROM orderbook;').fetchall() @@ -232,10 +245,11 @@ def choose_order(db, cj_amount, n): total_cj_fee = 0 chosen_orders = [] for i in range(n): - chosen_order = orders[0] #choose the cheapest, later this will be chosen differently + chosen_order = weighted_order_choose(orders, n, lambda k: k[2]) orders = [o for o in orders if o[0] != chosen_order[0]] #remove all orders from that same counterparty chosen_orders.append(chosen_order) total_cj_fee += chosen_order[2] + debug('chosen orders = ' + str(chosen_orders)) chosen_orders = [o[:2] for o in chosen_orders] return dict(chosen_orders), total_cj_fee @@ -296,7 +310,7 @@ def calc_zero_change_cj_amount(ordercombo): raise RuntimeError('unknown order type: ' + str(ordertype)) cjamount = (my_total_input - my_tx_fee - sumabsfee) / (1 + sumrelfee) cjamount = int(cjamount.quantize(Decimal(1))) - return cjamount + return cjamount, int(sumabsfee + sumrelfee*cjamount) def is_amount_in_range(ordercombo, cjamount): for order in ordercombo: @@ -311,14 +325,15 @@ def is_amount_in_range(ordercombo, cjamount): ordercombos = create_combination(orderlist, n) ordercombos = [(c, calc_zero_change_cj_amount(c)) for c in ordercombos] - ordercombos = [oc for oc in ordercombos if is_amount_in_range(oc[0], oc[1])] - ordercombos = sorted(ordercombos, key=lambda k: k[1]) + ordercombos = [oc for oc in ordercombos if is_amount_in_range(oc[0], oc[1][0])] + ordercombos = sorted(ordercombos, key=lambda k: k[1][0], reverse=True) dbgprint = [([(o['counterparty'], o['oid']) for o in oc[0]], oc[1]) for oc in ordercombos] debug('considered order combinations') debug(pprint.pformat(dbgprint)) - ordercombo = ordercombos[-1] #choose the cheapest, i.e. highest cj_amount + + ordercombo = weighted_order_choose(ordercombos, n, lambda k: k[1][1]) orders = dict([(o['counterparty'], o['oid']) for o in ordercombo[0]]) - cjamount = ordercombo[1] + cjamount = ordercombo[1][0] debug('chosen orders = ' + str(orders)) debug('cj amount = ' + str(cjamount)) return orders, cjamount From 63372f43332ddd799dadec899e52a75b863b8ad8 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 5 Apr 2015 02:12:39 +0100 Subject: [PATCH 180/409] added explaination of weighting algo --- common.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/common.py b/common.py index 2854dfe8..7c457325 100644 --- a/common.py +++ b/common.py @@ -223,6 +223,19 @@ def calc_total_input_value(utxos): return input_sum def weighted_order_choose(orders, n, feekey): + ''' + Algorithm for choosing the weighting function + it is an exponential + P(f) = exp(-(f - fmin) / phi) + P(f) - probability of order being chosen + f - order fee + fmin - minimum fee in the order book + phi - scaling parameter, 63% of the distribution is within + + define number M, related to the number of counterparties in this coinjoin + phi has a value such that it contains up to the Mth order + unless M < orderbook size, then phi goes up to the last order + ''' minfee = feekey(orders[0]) M = 2*n if len(orders) > M: From b25907202a7121eac03c1ba74964546a614f2456 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 5 Apr 2015 21:39:17 +0100 Subject: [PATCH 181/409] taker checks the received utxos are unspent and mined into a block --- blockchaininterface.py | 47 +++++++++++++++++++++++++++++++++++++++++- taker.py | 8 +++++-- 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/blockchaininterface.py b/blockchaininterface.py index c32f1055..250a7266 100644 --- a/blockchaininterface.py +++ b/blockchaininterface.py @@ -61,6 +61,15 @@ def pushtx(self, txhex): '''pushes tx to the network, returns txhash''' pass + @abc.abstractmethod + def is_output_suitable(self, txout): + ''' + checks whether the txid:vout output is suitable to be used, + must be already mined into a block, and unspent + accepts list of txid:vout too + ''' + pass + class BlockrInterface(BlockchainInterface): def __init__(self, testnet = False): super(BlockrInterface, self).__init__() @@ -231,6 +240,32 @@ def pushtx(self, txhex): common.debug(data) return None return data['data'] + + def is_output_suitable(self, txout): + if not isinstance(txout, list): + txout = [txout] + txids = [h[:64] for h in txout] + blockr_url = 'http://' + self.blockr_domain + '.blockr.io/api/v1/tx/info/' + data = json.loads(btc.make_request(blockr_url + ','.join(txids)))['data'] + if not isinstance(data, list): + data = [data] + addrs = [] + for tx in data: + if tx['is_unconfirmed']: + return False, 'tx ' + tx['tx'] + ' unconfirmed' + for outs in tx['vouts']: + addrs.append(outs['address']) + common.debug('addrs ' + pprint.pformat(addrs)) + utxos = btc.blockr_unspent(addrs, self.network) + utxos = [u['output'] for u in utxos] + common.debug('unspents = ' + pprint.pformat(utxos)) + common.debug('txouts = ' + pprint.pformat(txout)) + txoutset = set(txout) + utxoset = set(utxos) + if not utxoset.issuperset(txoutset): + return False, 'some utxos already spent' + return True, 'success' + class NotifyRequestHeader(SimpleHTTPServer.SimpleHTTPRequestHandler): def __init__(self, request, client_address, base_server): @@ -381,7 +416,7 @@ def sync_unspent(self, wallet): if u['address'] not in wallet.addr_cache: continue wallet.unspent[u['txid'] + ':' + str(u['vout'])] = {'address': u['address'], - 'value': int(u['amount']*1e8)} + 'value': int(Decimal(str(u['amount'])) * Decimal('1e8'))} et = time.time() common.debug('bitcoind sync_unspent took ' + str((et - st)) + 'sec') @@ -398,6 +433,16 @@ def fetchtx(self, txid): def pushtx(self, txhex): return self.rpc(['sendrawtransaction', txhex]).strip() + def is_output_suitable(self, txout): + if not isinstance(txout, list): + txout = [txout] + + for txo in txout: + ret = self.rpc(['gettxout', txo[:64], txo[65:], 'false']) + if ret == '': + return False, 'tx ' + txo + ' not found' + return True, 'success' + #class for regtest chain access #running on local daemon. Only #to be instantiated after network is up diff --git a/taker.py b/taker.py index 4ee5f847..40c8429d 100644 --- a/taker.py +++ b/taker.py @@ -58,7 +58,6 @@ def auth_counterparty(self, nick, btc_sig, cj_pub): return True def recv_txio(self, nick, utxo_list, cj_pub, change_addr): - cj_addr = btc.pubtoaddr(cj_pub, get_addr_vbyte()) if nick not in self.nonrespondants: debug('nick(' + nick + ') not in nonrespondants ' + str(self.nonrespondants)) return @@ -66,13 +65,18 @@ def recv_txio(self, nick, utxo_list, cj_pub, change_addr): self.nonrespondants.remove(nick) order = self.db.execute('SELECT ordertype, txfee, cjfee FROM ' 'orderbook WHERE oid=? AND counterparty=?', - (self.active_orders[nick], nick)).fetchone() + (self.active_orders[nick], nick)).fetchone() + goodoutputs, errmsg = common.bc_interface.is_output_suitable(utxo_list) + if not goodoutputs: + common.debug('bad outputs: ' + errmsg) + return total_input = calc_total_input_value(self.utxos[nick]) real_cjfee = calc_cj_fee(order['ordertype'], order['cjfee'], self.cj_amount) self.outputs.append({'address': change_addr, 'value': total_input - self.cj_amount - order['txfee'] + real_cjfee}) print 'fee breakdown for %s totalin=%d cjamount=%d txfee=%d realcjfee=%d' % (nick, total_input, self.cj_amount, order['txfee'], real_cjfee) + cj_addr = btc.pubtoaddr(cj_pub, get_addr_vbyte()) self.outputs.append({'address': cj_addr, 'value': self.cj_amount}) self.cjfee_total += real_cjfee if len(self.nonrespondants) > 0: From f7eeff6ee87a8de93c2b8996368897f7c5f47b7a Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 5 Apr 2015 22:46:07 +0100 Subject: [PATCH 182/409] added code checking whether there has been any bug in the wallet code --- maker.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/maker.py b/maker.py index 2e6d8ef7..f3983f25 100644 --- a/maker.py +++ b/maker.py @@ -27,6 +27,16 @@ def __init__(self, maker, nick, oid, amount, taker_pk): if amount < order['minsize'] or amount > order['maxsize']: self.maker.send_error(nick, 'amount out of range') self.utxos, self.cj_addr, self.change_addr = maker.oid_to_order(oid, amount) + #check nothing has messed up with the wallet code, remove this code after a while + import pprint + pprint.pprint(self.utxos) + for utxo, va in self.utxos.iteritems(): + txd = btc.deserialize(common.bc_interface.fetchtx(utxo[:64])) + value = txd['outs'][int(utxo[65:])]['value'] + if value != va['value']: + debug('wrongly labeled utxo, expected value ' + str(va['value']) + ' got ' + str(value)) + sys.exit(0) + self.ordertype = order['ordertype'] self.txfee = order['txfee'] self.cjfee = order['cjfee'] From 794b941869a0ee6c48e8442ba159f774c525bcf1 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 5 Apr 2015 23:47:21 +0100 Subject: [PATCH 183/409] moved many files to lib/ directory to reduce clutter --- gui-taker.py | 11 ++++++----- {bitcoin => lib/bitcoin}/__init__.py | 0 {bitcoin => lib/bitcoin}/bci.py | 0 {bitcoin => lib/bitcoin}/composite.py | 0 {bitcoin => lib/bitcoin}/deterministic.py | 0 {bitcoin => lib/bitcoin}/main.py | 0 {bitcoin => lib/bitcoin}/ripemd.py | 0 {bitcoin => lib/bitcoin}/stealth.py | 0 {bitcoin => lib/bitcoin}/transaction.py | 0 blockchaininterface.py => lib/blockchaininterface.py | 0 common.py => lib/common.py | 2 +- enc_wrapper.py => lib/enc_wrapper.py | 0 irc.py => lib/irc.py | 0 {libnacl => lib/libnacl}/__init__.py | 0 {libnacl => lib/libnacl}/base.py | 0 {libnacl => lib/libnacl}/blake.py | 0 {libnacl => lib/libnacl}/dual.py | 0 {libnacl => lib/libnacl}/encode.py | 0 {libnacl => lib/libnacl}/public.py | 0 {libnacl => lib/libnacl}/secret.py | 0 {libnacl => lib/libnacl}/sign.py | 0 {libnacl => lib/libnacl}/utils.py | 0 {libnacl => lib/libnacl}/version.py | 0 maker.py => lib/maker.py | 0 message_channel.py => lib/message_channel.py | 0 regtesttest.py => lib/regtesttest.py | 0 taker.py => lib/taker.py | 0 patientsendpayment.py | 10 ++++++---- sendpayment.py | 7 +++++-- wallet-tool.py | 8 ++++++-- yield-generator.py | 8 +++++--- 31 files changed, 29 insertions(+), 17 deletions(-) rename {bitcoin => lib/bitcoin}/__init__.py (100%) rename {bitcoin => lib/bitcoin}/bci.py (100%) rename {bitcoin => lib/bitcoin}/composite.py (100%) rename {bitcoin => lib/bitcoin}/deterministic.py (100%) rename {bitcoin => lib/bitcoin}/main.py (100%) rename {bitcoin => lib/bitcoin}/ripemd.py (100%) rename {bitcoin => lib/bitcoin}/stealth.py (100%) rename {bitcoin => lib/bitcoin}/transaction.py (100%) rename blockchaininterface.py => lib/blockchaininterface.py (100%) rename common.py => lib/common.py (99%) rename enc_wrapper.py => lib/enc_wrapper.py (100%) rename irc.py => lib/irc.py (100%) rename {libnacl => lib/libnacl}/__init__.py (100%) rename {libnacl => lib/libnacl}/base.py (100%) rename {libnacl => lib/libnacl}/blake.py (100%) rename {libnacl => lib/libnacl}/dual.py (100%) rename {libnacl => lib/libnacl}/encode.py (100%) rename {libnacl => lib/libnacl}/public.py (100%) rename {libnacl => lib/libnacl}/secret.py (100%) rename {libnacl => lib/libnacl}/sign.py (100%) rename {libnacl => lib/libnacl}/utils.py (100%) rename {libnacl => lib/libnacl}/version.py (100%) rename maker.py => lib/maker.py (100%) rename message_channel.py => lib/message_channel.py (100%) rename regtesttest.py => lib/regtesttest.py (100%) rename taker.py => lib/taker.py (100%) diff --git a/gui-taker.py b/gui-taker.py index 28d9cb34..c511c1d0 100644 --- a/gui-taker.py +++ b/gui-taker.py @@ -1,13 +1,14 @@ +import BaseHTTPServer, SimpleHTTPServer, threading +from decimal import Decimal +import io, base64, time, sys, os +data_dir = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.join(data_dir, 'lib')) + import taker from irc import IRCMessageChannel from common import * -import BaseHTTPServer, SimpleHTTPServer, threading -from decimal import Decimal - -import io -import base64, time tableheading = ''' diff --git a/bitcoin/__init__.py b/lib/bitcoin/__init__.py similarity index 100% rename from bitcoin/__init__.py rename to lib/bitcoin/__init__.py diff --git a/bitcoin/bci.py b/lib/bitcoin/bci.py similarity index 100% rename from bitcoin/bci.py rename to lib/bitcoin/bci.py diff --git a/bitcoin/composite.py b/lib/bitcoin/composite.py similarity index 100% rename from bitcoin/composite.py rename to lib/bitcoin/composite.py diff --git a/bitcoin/deterministic.py b/lib/bitcoin/deterministic.py similarity index 100% rename from bitcoin/deterministic.py rename to lib/bitcoin/deterministic.py diff --git a/bitcoin/main.py b/lib/bitcoin/main.py similarity index 100% rename from bitcoin/main.py rename to lib/bitcoin/main.py diff --git a/bitcoin/ripemd.py b/lib/bitcoin/ripemd.py similarity index 100% rename from bitcoin/ripemd.py rename to lib/bitcoin/ripemd.py diff --git a/bitcoin/stealth.py b/lib/bitcoin/stealth.py similarity index 100% rename from bitcoin/stealth.py rename to lib/bitcoin/stealth.py diff --git a/bitcoin/transaction.py b/lib/bitcoin/transaction.py similarity index 100% rename from bitcoin/transaction.py rename to lib/bitcoin/transaction.py diff --git a/blockchaininterface.py b/lib/blockchaininterface.py similarity index 100% rename from blockchaininterface.py rename to lib/blockchaininterface.py diff --git a/common.py b/lib/common.py similarity index 99% rename from common.py rename to lib/common.py index 7c457325..2ab43046 100644 --- a/common.py +++ b/lib/common.py @@ -15,7 +15,7 @@ alert_message = None config = SafeConfigParser() -config_location = os.path.join(os.path.dirname(os.path.realpath(__file__)),'joinmarket.cfg') +config_location = 'joinmarket.cfg' required_options = {'BLOCKCHAIN':['blockchain_source', 'network', 'bitcoin_cli_cmd'], 'MESSAGING':['host','channel','port']} diff --git a/enc_wrapper.py b/lib/enc_wrapper.py similarity index 100% rename from enc_wrapper.py rename to lib/enc_wrapper.py diff --git a/irc.py b/lib/irc.py similarity index 100% rename from irc.py rename to lib/irc.py diff --git a/libnacl/__init__.py b/lib/libnacl/__init__.py similarity index 100% rename from libnacl/__init__.py rename to lib/libnacl/__init__.py diff --git a/libnacl/base.py b/lib/libnacl/base.py similarity index 100% rename from libnacl/base.py rename to lib/libnacl/base.py diff --git a/libnacl/blake.py b/lib/libnacl/blake.py similarity index 100% rename from libnacl/blake.py rename to lib/libnacl/blake.py diff --git a/libnacl/dual.py b/lib/libnacl/dual.py similarity index 100% rename from libnacl/dual.py rename to lib/libnacl/dual.py diff --git a/libnacl/encode.py b/lib/libnacl/encode.py similarity index 100% rename from libnacl/encode.py rename to lib/libnacl/encode.py diff --git a/libnacl/public.py b/lib/libnacl/public.py similarity index 100% rename from libnacl/public.py rename to lib/libnacl/public.py diff --git a/libnacl/secret.py b/lib/libnacl/secret.py similarity index 100% rename from libnacl/secret.py rename to lib/libnacl/secret.py diff --git a/libnacl/sign.py b/lib/libnacl/sign.py similarity index 100% rename from libnacl/sign.py rename to lib/libnacl/sign.py diff --git a/libnacl/utils.py b/lib/libnacl/utils.py similarity index 100% rename from libnacl/utils.py rename to lib/libnacl/utils.py diff --git a/libnacl/version.py b/lib/libnacl/version.py similarity index 100% rename from libnacl/version.py rename to lib/libnacl/version.py diff --git a/maker.py b/lib/maker.py similarity index 100% rename from maker.py rename to lib/maker.py diff --git a/message_channel.py b/lib/message_channel.py similarity index 100% rename from message_channel.py rename to lib/message_channel.py diff --git a/regtesttest.py b/lib/regtesttest.py similarity index 100% rename from regtesttest.py rename to lib/regtesttest.py diff --git a/taker.py b/lib/taker.py similarity index 100% rename from taker.py rename to lib/taker.py diff --git a/patientsendpayment.py b/patientsendpayment.py index 83eeb8f3..69b8e06f 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -1,4 +1,10 @@ +from optparse import OptionParser +from datetime import timedelta +import threading, time, binascii, os, sys +data_dir = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.join(data_dir, 'lib')) + from common import * import common import taker @@ -6,10 +12,6 @@ from irc import IRCMessageChannel import bitcoin as btc -from optparse import OptionParser -from datetime import timedelta -import threading, time, binascii, os - class TakerThread(threading.Thread): def __init__(self, tmaker): threading.Thread.__init__(self) diff --git a/sendpayment.py b/sendpayment.py index ca2655d0..7472ba5d 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -1,13 +1,16 @@ #! /usr/bin/env python +from optparse import OptionParser +import threading, pprint, sys, os +data_dir = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.join(data_dir, 'lib')) + from common import * import common import taker as takermodule from irc import IRCMessageChannel import bitcoin as btc -from optparse import OptionParser -import threading, pprint #thread which does the buy-side algorithm # chooses which coinjoins to initiate and when diff --git a/wallet-tool.py b/wallet-tool.py index 517907ba..4c7b8dae 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -1,10 +1,14 @@ +import sys, os +from optparse import OptionParser +data_dir = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.join(data_dir, 'lib')) + import bitcoin as btc from common import Wallet, get_signed_tx, load_program_config, get_addr_vbyte import common -import sys -from optparse import OptionParser + #structure for cj market wallet # m/0/ root key diff --git a/yield-generator.py b/yield-generator.py index 0e4c8a4c..c9217da7 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -1,11 +1,13 @@ #! /usr/bin/env python +import time, os, binascii, sys +import pprint +data_dir = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.join(data_dir, 'lib')) + from maker import * from irc import IRCMessageChannel import bitcoin as btc -import time -import os, binascii -import pprint import common from socket import gethostname From 9e6f954b020c45691ebeabca0400d760004fe8fe Mon Sep 17 00:00:00 2001 From: Belcher Date: Mon, 6 Apr 2015 03:23:01 +0100 Subject: [PATCH 184/409] made output txlist a bit clearer for tumbler.py, plus bugfixes --- tumbler.py | 47 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/tumbler.py b/tumbler.py index 3896f032..45c533a1 100644 --- a/tumbler.py +++ b/tumbler.py @@ -1,11 +1,14 @@ +import datetime, threading, binascii, sys, os, copy +data_dir = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.join(data_dir, 'lib')) + import taker as takermodule import common from common import * from irc import IRCMessageChannel from optparse import OptionParser -import datetime, threading, binascii import numpy as np from pprint import pprint @@ -96,7 +99,9 @@ def send_tx(self, tx, balance, sweep, i, l): destaddr = tx['dest'] if sweep: - total_value = self.wallet.get_balance_by_mixdepth()[tx['srcmixdepth']] + print 'sweeping' + all_utxos = self.taker.wallet.get_utxos_by_mixdepth()[tx['srcmixdepth']] + total_value = sum([addrval['value'] for addrval in all_utxos.values()]) orders, cjamount = choose_sweep_order(self.taker.db, total_value, self.taker.txfee, tx['makercount']) self.taker.start_cj(self.taker.wallet, cjamount, orders, all_utxos, destaddr, None, self.taker.txfee, self.finishcallback) @@ -104,7 +109,13 @@ def send_tx(self, tx, balance, sweep, i, l): amount = int(tx['amount_ratio'] * balance) changeaddr = self.taker.wallet.get_change_addr(tx['srcmixdepth']) print 'coinjoining ' + str(amount) - orders, total_cj_fee = choose_order(self.taker.db, amount, tx['makercount']) + while True: + orders, total_cj_fee = choose_order(self.taker.db, amount, tx['makercount']) + cj_fee = 1.0*total_cj_fee / tx['makercount'] / amount + if cj_fee < self.taker.maxcjfee: + break + print 'cj fee too high at ' + str(cj_fee) + ', waiting 10 seconds' + time.sleep(10) print 'chosen orders to fill ' + str(orders) + ' totalcjfee=' + str(total_cj_fee) total_amount = amount + total_cj_fee + self.taker.txfee print 'total amount spent = ' + str(total_amount) @@ -125,22 +136,28 @@ def run(self): print 'waiting for all orders to certainly arrive' time.sleep(orderwaittime) + sqlorders = self.taker.db.execute('SELECT cjfee, ordertype FROM orderbook;').fetchall() + orders = [o['cjfee'] for o in sqlorders if o['ordertype'] == 'relorder'] + orders = sorted(orders) + relorder_fee = float(orders[0]) + print 'relorder fee = ' + str(relorder_fee) maker_count = sum([tx['makercount'] for tx in self.taker.tx_list]) - relorder_fee = 0.01 print('uses ' + str(maker_count) + ' makers, at ' + str(relorder_fee*100) + '% per maker, estimated total cost ' + str(round((1 - (1 - relorder_fee)**maker_count) * 100, 3)) + '%') + time.sleep(orderwaittime) + print 'starting' self.lockcond = threading.Condition() self.balance_by_mixdepth = {} for i, tx in enumerate(self.taker.tx_list): if tx['srcmixdepth'] not in self.balance_by_mixdepth: - self.balance_by_mixdepth[tx['srcmixdepth']] = self.wallet.get_balance_by_mixdepth()[tx['srcmixdepth']] + self.balance_by_mixdepth[tx['srcmixdepth']] = self.taker.wallet.get_balance_by_mixdepth()[tx['srcmixdepth']] sweep = True for later_tx in self.taker.tx_list[i + 1:]: if later_tx['srcmixdepth'] == tx['srcmixdepth']: sweep = False - self.send_tx(tx, sweep, self.balance_by_mixdepth[tx['srcmixdepth']], + self.send_tx(tx, self.balance_by_mixdepth[tx['srcmixdepth']], sweep, i, len(self.taker.tx_list)) print 'total finished' @@ -169,7 +186,7 @@ def on_welcome(self): TumblerThread(self).start() def main(): - parser = OptionParser(usage='usage: %prog [options] [seed] [tumble-file / destaddr...]', + parser = OptionParser(usage='usage: %prog [options] [seed] [destaddr...]', description='Sends bitcoins to many different addresses using coinjoin in' ' an attempt to break the link between them. Sending to multiple ' ' addresses is highly recommended for privacy. This tumbler can' @@ -181,7 +198,7 @@ def main(): parser.add_option('-f', '--txfee', type='int', dest='txfee', default=10000, help='miner fee contribution, in satoshis, default=10000') parser.add_option('-x', '--maxcjfee', type='float', dest='maxcjfee', - default=0.02, help='maximum coinjoin fee the tumbler is willing to pay for a single coinjoin. default=0.02 (2%)') + default=0.03, help='maximum coinjoin fee the tumbler is willing to pay to a single market maker. default=0.03 (3%)') parser.add_option('-a', '--addrask', type='int', dest='addrask', default=2, help='How many more addresses to ask for in the terminal. Should ' 'be similar to --txcountparams. default=2') @@ -225,7 +242,19 @@ def main(): print str(options) tx_list = generate_tumbler_tx(destaddrs, options) - pprint(tx_list) + tx_list2 = copy.deepcopy(tx_list) + tx_dict = {} + for tx in tx_list2: + srcmixdepth = tx['srcmixdepth'] + tx.pop('srcmixdepth') + if srcmixdepth not in tx_dict: + tx_dict[srcmixdepth] = [] + tx_dict[srcmixdepth].append(tx) + dbg_tx_list = [] + for srcmixdepth, txlist in tx_dict.iteritems(): + dbg_tx_list.append({'srcmixdepth': srcmixdepth, 'tx': txlist}) + pprint(dbg_tx_list) + total_wait = sum([tx['wait'] for tx in tx_list]) print 'waits in total for ' + str(len(tx_list)) + ' blocks and ' + str(total_wait) + ' minutes' total_block_and_wait = len(tx_list)*10 + total_wait From e5d62c878f92b9551119d77e564207916c7f4ca8 Mon Sep 17 00:00:00 2001 From: Belcher Date: Mon, 6 Apr 2015 19:29:11 +0100 Subject: [PATCH 185/409] implemented encrypetd wallet files and mnemonic seeds, taken from electrum, also changed default wallettool mixdepth to 5, same as yieldgen --- lib/common.py | 23 +++++++++-- wallet-tool.py | 108 +++++++++++++++++++++++++------------------------ 2 files changed, 75 insertions(+), 56 deletions(-) diff --git a/lib/common.py b/lib/common.py index 2ab43046..c1728b81 100644 --- a/lib/common.py +++ b/lib/common.py @@ -2,7 +2,7 @@ import bitcoin as btc from decimal import Decimal from math import factorial -import sys, datetime, json, time, pprint, threading +import sys, datetime, json, time, pprint, threading, aes, getpass import numpy as np import blockchaininterface from ConfigParser import SafeConfigParser @@ -95,9 +95,10 @@ def get_addr_from_utxo(txhash, index): return btc.script_to_address(btc.deserialize(bc_interface.fetchtx(txhash))['outs'][index]['script'], get_addr_vbyte()) class Wallet(object): - def __init__(self, seed, max_mix_depth=2): + def __init__(self, seedarg, max_mix_depth=2): self.max_mix_depth = max_mix_depth - master = btc.bip32_master_key(seed) + self.seed = self.get_seed(seedarg) + master = btc.bip32_master_key(self.seed) m_0 = btc.bip32_ckd(master, 0) mixing_depth_keys = [btc.bip32_ckd(m_0, c) for c in range(max_mix_depth)] self.keys = [(btc.bip32_ckd(m, 0), btc.bip32_ckd(m, 1)) for m in mixing_depth_keys] @@ -114,6 +115,22 @@ def __init__(self, seed, max_mix_depth=2): self.addr_cache = {} self.unspent = {} + def get_seed(self, seedarg): + path = os.path.join('wallets', seedarg) + if not os.path.isfile(path): + debug('seedarg interpreted as seed') + return seedarg + debug('seedarg interpreted as wallet file name') + fd = open(path, 'r') + walletfile = fd.read() + fd.close() + walletdata = json.loads(walletfile) + password = getpass.getpass('Enter wallet decryption passphrase: ') + password_key = btc.bin_dbl_sha256(password) + decrypted_seed = aes.decryptData(password_key, walletdata['encrypted_seed'] + .decode('hex')).encode('hex') + return decrypted_seed + def get_key(self, mixing_depth, forchange, i): return btc.bip32_extract_key(btc.bip32_ckd(self.keys[mixing_depth][forchange], i)) diff --git a/wallet-tool.py b/wallet-tool.py index 4c7b8dae..90d72797 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -1,5 +1,6 @@ import sys, os +import getpass, aes, json, datetime from optparse import OptionParser data_dir = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(data_dir, 'lib')) @@ -7,8 +8,7 @@ import bitcoin as btc from common import Wallet, get_signed_tx, load_program_config, get_addr_vbyte import common - - +import old_mnemonic #structure for cj market wallet # m/0/ root key @@ -20,36 +20,38 @@ # m/0/n/1/k kth change address, for mixing depth n -parser = OptionParser(usage='usage: %prog [options] [seed] [method]', - description='Does useful little lasts involving your bip32 wallet. The' - + ' method is one of the following: display- shows all addresses and balances.' - + ' summary- shows a summary of mixing depth balances.' - + ' combine- combines all utxos into one output for each mixing level. Used for' - + ' testing and is detrimental to privacy.' - + ' reset - send all utxos back to first receiving address at zeroth mixing level.' - + ' Also only for testing and destroys privacy.') +parser = OptionParser(usage='usage: %prog [options] [seed / wallet file] [method]', + description='Does useful little tasks involving your bip32 wallet. The' + + ' method is one of the following: display- shows addresses and balances.' + + ' displayall - shows ALL addresses and balances.' + + ' summary - shows a summary of mixing depth balances.' + + ' generate - generates a new wallet.' + + ' recover - recovers a wallet from the 12 word recovery seed.' + + ' showseed - shows the wallet recovery seed and hex seed.') parser.add_option('-p', '--privkey', action='store_true', dest='showprivkey', help='print private key along with address, default false') parser.add_option('-m', '--maxmixdepth', action='store', type='int', dest='maxmixdepth', - default=2, help='maximum mixing depth to look for, default=2') + default=5, help='maximum mixing depth to look for, default=5') parser.add_option('-g', '--gap-limit', action='store', dest='gaplimit', help='gap limit for wallet, default=6', default=6) (options, args) = parser.parse_args() +noseed_methods = ['generate', 'recover'] +methods = ['display', 'displayall', 'summary'] + noseed_methods + if len(args) < 1: - parser.error('Needs a seed') + parser.error('Needs a seed, wallet file or method') sys.exit(0) -seed = args[0] load_program_config() - -method = ('display' if len(args) == 1 else args[1].lower()) - - -#seed = '256 bits of randomness' - -wallet = Wallet(seed, options.maxmixdepth) -common.bc_interface.sync_wallet(wallet, options.gaplimit) +if args[0] in noseed_methods: + method = args[0] +else: + seed = args[0] + method = ('display' if len(args) == 1 else args[1].lower()) + wallet = Wallet(seed, options.maxmixdepth) + if method != 'showseed': + common.bc_interface.sync_wallet(wallet, options.gaplimit) if method == 'display' or method == 'displayall': total_balance = 0 @@ -87,35 +89,35 @@ print 'for mixdepth=%d balance=%.8fbtc' % (m, balance_depth/1e8) total_balance += balance_depth print 'total balance = %.8fbtc' % (total_balance/1e8) - -elif method == 'combine': - ins = [] - outs = [] - for m in range(wallet.max_mix_depth): - for forchange in [0, 1]: - balance = 0 - for k in range(wallet.index[m][forchange]): - addr = wallet.get_addr(m, forchange, k) - for utxo, addrvalue in wallet.unspent.iteritems(): - if addr != addrvalue['address']: - continue - ins.append({'output': utxo}) - balance += addrvalue['value'] - - if balance > 0: - destaddr = wallet.get_addr(m, forchange, wallet.index[m][forchange]) - outs.append({'address': destaddr, 'value': balance}) - print get_signed_tx(wallet, ins, outs) - -elif method == 'reset': - ins = [] - outs = [] - balance = 0 - for utxo, addrvalue in wallet.unspent.iteritems(): - ins.append({'output': utxo}) - balance += addrvalue['value'] - destaddr = wallet.get_addr(0,0,0) - outs.append({'address': destaddr, 'value': balance}) - print get_signed_tx(wallet, ins, outs) - - +elif method == 'generate' or method == 'recover': + if method == 'generate': + seed = btc.sha256(os.urandom(64))[:32] + words = old_mnemonic.mn_encode(seed) + print 'Write down this wallet recovery seed\n' + ' '.join(words) + '\n' + elif method == 'recover': + words = raw_input('Input 12 word recovery seed: ') + words = words.split(' ') + seed = old_mnemonic.mn_decode(words) + print seed + password = getpass.getpass('Enter wallet encryption passphrase: ') + password2 = getpass.getpass('Reenter wallet encryption passphrase: ') + if password != password2: + print 'ERROR. Passwords did not match' + sys.exit(0) + password_key = btc.bin_dbl_sha256(password) + encrypted_seed = aes.encryptData(password_key, seed.decode('hex')) + timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S") + walletfile = json.dumps({'creator': 'joinmarket project', 'creation_time': timestamp, + 'encrypted_seed': encrypted_seed.encode('hex')}) + walletname = raw_input('Input wallet file name (default: wallet.json): ') + if len(walletname) == 0: + walletname = 'wallet.json' + fd = open(os.path.join('wallets', walletname), 'w') + fd.write(walletfile) + fd.close() + print 'saved to ' + walletname +elif method == 'showseed': + hexseed = wallet.seed + print 'hexseed = ' + hexseed + words = old_mnemonic.mn_encode(hexseed) + print 'Wallet recovery seed\n' + ' '.join(words) + '\n' From f30209684d80d877d14992c57a7e82cb711b4613 Mon Sep 17 00:00:00 2001 From: Belcher Date: Mon, 6 Apr 2015 19:37:30 +0100 Subject: [PATCH 186/409] removed all seeds, keys and other private information from the crash dumps --- patientsendpayment.py | 2 +- sendpayment.py | 2 +- tumbler.py | 2 +- yield-generator.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/patientsendpayment.py b/patientsendpayment.py index 69b8e06f..0d726c90 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -159,7 +159,7 @@ def main(): except: debug('CRASHING, DUMPING EVERYTHING') debug('wallet seed = ' + seed) - debug_dump_object(wallet, ['addr_cache']) + debug_dump_object(wallet, ['addr_cache', 'keys', 'seed']) debug_dump_object(taker) import traceback traceback.print_exc() diff --git a/sendpayment.py b/sendpayment.py index 7472ba5d..f4952e36 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -106,7 +106,7 @@ def main(): except: debug('CRASHING, DUMPING EVERYTHING') debug('wallet seed = ' + seed) - debug_dump_object(wallet, ['addr_cache']) + debug_dump_object(wallet, ['addr_cache', 'keys', 'seed']) debug_dump_object(taker) import traceback debug(traceback.format_exc()) diff --git a/tumbler.py b/tumbler.py index 45c533a1..e9f1a02c 100644 --- a/tumbler.py +++ b/tumbler.py @@ -290,7 +290,7 @@ def main(): except: debug('CRASHING, DUMPING EVERYTHING') debug('wallet seed = ' + seed) - debug_dump_object(wallet, ['addr_cache']) + debug_dump_object(wallet, ['addr_cache', 'keys', 'seed']) debug_dump_object(tumbler) import traceback debug(traceback.format_exc()) diff --git a/yield-generator.py b/yield-generator.py index c9217da7..75858d7e 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -100,7 +100,7 @@ def main(): except: debug('CRASHING, DUMPING EVERYTHING') debug('wallet seed = ' + seed) - debug_dump_object(wallet, ['addr_cache', 'keys']) + debug_dump_object(wallet, ['addr_cache', 'keys', 'seed']) debug_dump_object(maker) debug_dump_object(irc) import traceback From acef5d77e7e4c6d7f0b09c9bfd7b61a212b9e350 Mon Sep 17 00:00:00 2001 From: Belcher Date: Mon, 6 Apr 2015 19:42:44 +0100 Subject: [PATCH 187/409] added wallets directory --- wallets/.gitignore | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 wallets/.gitignore diff --git a/wallets/.gitignore b/wallets/.gitignore new file mode 100644 index 00000000..86d0cb27 --- /dev/null +++ b/wallets/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore \ No newline at end of file From e01854c51ed57bf5aa816c5be69a601f3430f5d1 Mon Sep 17 00:00:00 2001 From: Belcher Date: Mon, 6 Apr 2015 22:02:32 +0100 Subject: [PATCH 188/409] updated to latest pybitcointools --- lib/bitcoin/__init__.py | 15 +- lib/bitcoin/bci.py | 204 +++++++++--- lib/bitcoin/blocks.py | 50 +++ lib/bitcoin/composite.py | 110 ++++++- lib/bitcoin/deterministic.py | 153 +++++---- lib/bitcoin/main.py | 621 +++++++++++++++++++++-------------- lib/bitcoin/py2specials.py | 94 ++++++ lib/bitcoin/py3specials.py | 119 +++++++ lib/bitcoin/ripemd.py | 27 +- lib/bitcoin/stealth.py | 56 ++-- lib/bitcoin/transaction.py | 418 ++++++++++++++--------- lib/blockchaininterface.py | 2 +- 12 files changed, 1316 insertions(+), 553 deletions(-) create mode 100644 lib/bitcoin/blocks.py create mode 100644 lib/bitcoin/py2specials.py create mode 100644 lib/bitcoin/py3specials.py diff --git a/lib/bitcoin/__init__.py b/lib/bitcoin/__init__.py index cedc2c47..a772073f 100644 --- a/lib/bitcoin/__init__.py +++ b/lib/bitcoin/__init__.py @@ -1,6 +1,9 @@ -from main import * -from transaction import * -from deterministic import * -from bci import * -from composite import * -from stealth import * +from bitcoin.py2specials import * +from bitcoin.py3specials import * +from bitcoin.main import * +from bitcoin.transaction import * +from bitcoin.deterministic import * +from bitcoin.bci import * +from bitcoin.composite import * +from bitcoin.stealth import * +from bitcoin.blocks import * diff --git a/lib/bitcoin/bci.py b/lib/bitcoin/bci.py index 45b6719b..a9b8d15f 100644 --- a/lib/bitcoin/bci.py +++ b/lib/bitcoin/bci.py @@ -1,31 +1,21 @@ #!/usr/bin/python -import urllib2 -import json -import re +import json, re import random import sys - -#i changed this 8/2014 to include this proxy thing -# that also required changing everything to http requests since the -# proxy just works for http in urllib2 -__proxy_handler = None - -def bci_set_proxy(handler): - global __proxy_handler - __proxy_handler = handler +try: + from urllib.request import build_opener +except: + from urllib2 import build_opener # Makes a request to a given URL (first arg) and optional params (second arg) def make_request(*args): - if __proxy_handler != None: - opener = urllib2.build_opener(__proxy_handler) - else: - opener = urllib2.build_opener() + opener = build_opener() opener.addheaders = [('User-agent', 'Mozilla/5.0'+str(random.randrange(1000000)))] try: return opener.open(*args).read().strip() - except Exception, e: + except Exception as e: try: p = e.read().strip() except: @@ -33,21 +23,31 @@ def make_request(*args): raise Exception(p) +def parse_addr_args(*args): + # Valid input formats: blockr_unspent([addr1, addr2,addr3]) + # blockr_unspent(addr1, addr2, addr3) + # blockr_unspent([addr1, addr2, addr3], network) + # blockr_unspent(addr1, addr2, addr3, network) + # Where network is 'btc' or 'testnet' + network = 'btc' + addr_args = args + if len(args) >= 1 and args[-1] in ('testnet', 'btc'): + network = args[-1] + addr_args = args[:-1] + if len(addr_args) == 1 and isinstance(addr_args, list): + addr_args = addr_args[0] + + return network, addr_args + + # Gets the unspent outputs of one or more addresses -def unspent(*args): - # Valid input formats: unspent([addr1, addr2,addr3]) - # unspent(addr1, addr2, addr3) - if len(args) == 0: - return [] - elif isinstance(args[0], list): - addrs = args[0] - else: - addrs = args +def bci_unspent(*args): + network, addrs = parse_addr_args(*args) u = [] for a in addrs: try: data = make_request('http://blockchain.info/unspent?address='+a) - except Exception, e: + except Exception as e: if str(e) == 'No free outputs to spend': continue else: @@ -71,11 +71,7 @@ def blockr_unspent(*args): # blockr_unspent([addr1, addr2, addr3], network) # blockr_unspent(addr1, addr2, addr3, network) # Where network is 'btc' or 'testnet' - network = 'btc' - addr_args = args - if len(args) >= 1 and args[-1] in ('testnet', 'btc'): - network = args[-1] - addr_args = args[:-1] + network, addr_args = parse_addr_args(*args) if network == 'testnet': blockr_url = 'http://tbtc.blockr.io/api/v1/address/unspent/' @@ -105,6 +101,41 @@ def blockr_unspent(*args): return o +def helloblock_unspent(*args): + network, addrs = parse_addr_args(*args) + if network == 'testnet': + url = 'http://testnet.helloblock.io/v1/addresses/%s/unspents?limit=500&offset=%s' + elif network == 'btc': + url = 'http://mainnet.helloblock.io/v1/addresses/%s/unspents?limit=500&offset=%s' + o = [] + for addr in addrs: + for offset in xrange(0, 10**9, 500): + res = make_request(url % (addr, offset)) + data = json.loads(res)["data"] + if not len(data["unspents"]): + break + elif offset: + sys.stderr.write("Getting more unspents: %d\n" % offset) + for dat in data["unspents"]: + o.append({ + "output": dat["txHash"]+':'+str(dat["index"]), + "value": dat["value"], + }) + return o + + +unspent_getters = { + 'bci': bci_unspent, + 'blockr': blockr_unspent, + 'helloblock': helloblock_unspent +} + + +def unspent(*args, **kwargs): + f = unspent_getters.get(kwargs.get('source', ''), bci_unspent) + return f(*args) + + # Gets the transaction output history of a given set of addresses, # including whether or not they have been spent def history(*args): @@ -154,8 +185,8 @@ def history(*args): return [outs[k] for k in outs] -# Pushes a transaction to the network using https://blockchain.info/pushtx -def pushtx(tx): +# Pushes a transaction to the network using http://blockchain.info/pushtx +def bci_pushtx(tx): if not re.match('^[0-9a-fA-F]*$', tx): tx = tx.encode('hex') return make_request('http://blockchain.info/pushtx', 'tx='+tx) @@ -194,6 +225,17 @@ def helloblock_pushtx(tx): return make_request('http://mainnet.helloblock.io/v1/transactions', 'rawTxHex='+tx) +pushtx_getters = { + 'bci': bci_pushtx, + 'blockr': blockr_pushtx, + 'helloblock': helloblock_pushtx +} + + +def pushtx(*args, **kwargs): + f = pushtx_getters.get(kwargs.get('source', ''), bci_pushtx) + return f(*args) + def last_block_height(): data = make_request('http://blockchain.info/latestblock') @@ -223,11 +265,54 @@ def blockr_fetchtx(txhash, network='btc'): return jsondata['data']['tx']['hex'] -def fetchtx(txhash): - try: - return bci_fetchtx(txhash) - except: - return blockr_fetchtx(txhash) +def helloblock_fetchtx(txhash, network='btc'): + if not re.match('^[0-9a-fA-F]*$', txhash): + txhash = txhash.encode('hex') + if network == 'testnet': + url = 'http://testnet.helloblock.io/v1/transactions/' + elif network == 'btc': + url = 'http://mainnet.helloblock.io/v1/transactions/' + else: + raise Exception( + 'Unsupported network {0} for helloblock_fetchtx'.format(network)) + data = json.loads(make_request(url + txhash))["data"]["transaction"] + o = { + "locktime": data["locktime"], + "version": data["version"], + "ins": [], + "outs": [] + } + for inp in data["inputs"]: + o["ins"].append({ + "script": inp["scriptSig"], + "outpoint": { + "index": inp["prevTxoutIndex"], + "hash": inp["prevTxHash"], + }, + "sequence": 4294967295 + }) + for outp in data["outputs"]: + o["outs"].append({ + "value": outp["value"], + "script": outp["scriptPubKey"] + }) + from bitcoin.transaction import serialize + from bitcoin.transaction import txhash as TXHASH + tx = serialize(o) + assert TXHASH(tx) == txhash + return tx + + +fetchtx_getters = { + 'bci': bci_fetchtx, + 'blockr': blockr_fetchtx, + 'helloblock': helloblock_fetchtx +} + + +def fetchtx(*args, **kwargs): + f = fetchtx_getters.get(kwargs.get('source', ''), bci_fetchtx) + return f(*args) def firstbits(address): @@ -236,3 +321,44 @@ def firstbits(address): else: return make_request( 'http://blockchain.info/q/resolvefirstbits/'+address) + + +def get_block_at_height(height): + j = json.loads(make_request("http://blockchain.info/block-height/" + + str(height)+"?format=json")) + for b in j['blocks']: + if b['main_chain'] is True: + return b + raise Exception("Block at this height not found") + + +def _get_block(inp): + if len(str(inp)) < 64: + return get_block_at_height(inp) + else: + return json.loads(make_request( + 'http://blockchain.info/rawblock/'+inp)) + + +def get_block_header_data(inp): + j = _get_block(inp) + return { + 'version': j['ver'], + 'hash': j['hash'], + 'prevhash': j['prev_block'], + 'timestamp': j['time'], + 'merkle_root': j['mrkl_root'], + 'bits': j['bits'], + 'nonce': j['nonce'], + } + + +def get_txs_in_block(inp): + j = _get_block(inp) + hashes = [t['hash'] for t in j['tx']] + return hashes + + +def get_block_height(txhash): + j = json.loads(make_request('http://blockchain.info/rawtx/'+txhash)) + return j['block_height'] diff --git a/lib/bitcoin/blocks.py b/lib/bitcoin/blocks.py new file mode 100644 index 00000000..e60dcfdd --- /dev/null +++ b/lib/bitcoin/blocks.py @@ -0,0 +1,50 @@ +from bitcoin.main import * + + +def serialize_header(inp): + o = encode(inp['version'], 256, 4)[::-1] + \ + inp['prevhash'].decode('hex')[::-1] + \ + inp['merkle_root'].decode('hex')[::-1] + \ + encode(inp['timestamp'], 256, 4)[::-1] + \ + encode(inp['bits'], 256, 4)[::-1] + \ + encode(inp['nonce'], 256, 4)[::-1] + h = bin_sha256(bin_sha256(o))[::-1].encode('hex') + assert h == inp['hash'], (sha256(o), inp['hash']) + return o.encode('hex') + + +def deserialize_header(inp): + inp = inp.decode('hex') + return { + "version": decode(inp[:4][::-1], 256), + "prevhash": inp[4:36][::-1].encode('hex'), + "merkle_root": inp[36:68][::-1].encode('hex'), + "timestamp": decode(inp[68:72][::-1], 256), + "bits": decode(inp[72:76][::-1], 256), + "nonce": decode(inp[76:80][::-1], 256), + "hash": bin_sha256(bin_sha256(inp))[::-1].encode('hex') + } + + +def mk_merkle_proof(header, hashes, index): + nodes = [h.decode('hex')[::-1] for h in hashes] + if len(nodes) % 2 and len(nodes) > 2: + nodes.append(nodes[-1]) + layers = [nodes] + while len(nodes) > 1: + newnodes = [] + for i in range(0, len(nodes) - 1, 2): + newnodes.append(bin_sha256(bin_sha256(nodes[i] + nodes[i+1]))) + if len(newnodes) % 2 and len(newnodes) > 2: + newnodes.append(newnodes[-1]) + nodes = newnodes + layers.append(nodes) + # Sanity check, make sure merkle root is valid + assert nodes[0][::-1].encode('hex') == header['merkle_root'] + merkle_siblings = \ + [layers[i][(index >> i) ^ 1] for i in range(len(layers)-1)] + return { + "hash": hashes[index], + "siblings": [x[::-1].encode('hex') for x in merkle_siblings], + "header": header + } diff --git a/lib/bitcoin/composite.py b/lib/bitcoin/composite.py index dc234b8b..cbceea93 100644 --- a/lib/bitcoin/composite.py +++ b/lib/bitcoin/composite.py @@ -1,19 +1,54 @@ -from main import * -from transaction import * -from bci import * -from deterministic import * +from bitcoin.main import * +from bitcoin.transaction import * +from bitcoin.bci import * +from bitcoin.deterministic import * +from bitcoin.blocks import * # Takes privkey, address, value (satoshis), fee (satoshis) -def send(frm, to, value, fee=1000): - u = unspent(privtoaddr(frm)) - u2 = select(u, value+fee) - argz = u2 + [to+':'+str(value), privtoaddr(to), fee] - tx = mksend(argz) - tx2 = signall(tx, privtoaddr(to)) - pushtx(tx2) +def send(frm, to, value, fee=10000): + return sendmultitx(frm, to + ":" + str(value), fee) +# Takes privkey, "address1:value1,address2:value2" (satoshis), fee (satoshis) +def sendmultitx(frm, tovalues, fee=10000, **kwargs): + outs = [] + outvalue = 0 + tv = tovalues.split(",") + for a in tv: + outs.append(a) + outvalue += int(a.split(":")[1]) + + u = unspent(privtoaddr(frm), **kwargs) + u2 = select(u, int(outvalue)+int(fee)) + argz = u2 + outs + [frm, fee] + tx = mksend(*argz) + tx2 = signall(tx, frm) + return pushtx(tx2, **kwargs) + + +# Takes address, address, value (satoshis), fee(satoshis) +def preparetx(frm, to, value, fee=10000, **kwargs): + tovalues = to + ":" + str(value) + return preparemultitx(frm, tovalues, fee, **kwargs) + + +# Takes address, address:value, address:value ... (satoshis), fee(satoshis) +def preparemultitx(frm, *args, **kwargs): + tv, fee = args[:-1], int(args[-1]) + outs = [] + outvalue = 0 + for a in tv: + outs.append(a) + outvalue += int(a.split(":")[1]) + + u = unspent(frm, **kwargs) + u2 = select(u, int(outvalue)+int(fee)) + argz = u2 + outs + [frm, fee] + return mksend(*argz) + + +# BIP32 hierarchical deterministic multisig script def bip32_hdm_script(*args): if len(args) == 3: keys, req, path = args @@ -28,13 +63,66 @@ def bip32_hdm_script(*args): return mk_multisig_script(pubs, req) +# BIP32 hierarchical deterministic multisig address def bip32_hdm_addr(*args): return scriptaddr(bip32_hdm_script(*args)) +# Setup a coinvault transaction def setup_coinvault_tx(tx, script): txobj = deserialize(tx) N = deserialize_script(script)[-2] for inp in txobj["ins"]: inp["script"] = serialize_script([None] * (N+1) + [script]) return serialize(txobj) + + +# Sign a coinvault transaction +def sign_coinvault_tx(tx, priv): + pub = privtopub(priv) + txobj = deserialize(tx) + subscript = deserialize_script(txobj['ins'][0]['script']) + oscript = deserialize_script(subscript[-1]) + k, pubs = oscript[0], oscript[1:-2] + for j in range(len(txobj['ins'])): + scr = deserialize_script(txobj['ins'][j]['script']) + for i, p in enumerate(pubs): + if p == pub: + scr[i+1] = multisign(tx, j, subscript[-1], priv) + if len(filter(lambda x: x, scr[1:-1])) >= k: + scr = [None] + filter(lambda x: x, scr[1:-1])[:k] + [scr[-1]] + txobj['ins'][j]['script'] = serialize_script(scr) + return serialize(txobj) + + +# Inspects a transaction +def inspect(tx, **kwargs): + d = deserialize(tx) + isum = 0 + ins = {} + for _in in d['ins']: + h = _in['outpoint']['hash'] + i = _in['outpoint']['index'] + prevout = deserialize(fetchtx(h, **kwargs))['outs'][i] + isum += prevout['value'] + a = script_to_address(prevout['script']) + ins[a] = ins.get(a, 0) + prevout['value'] + outs = [] + osum = 0 + for _out in d['outs']: + outs.append({'address': script_to_address(_out['script']), + 'value': _out['value']}) + osum += _out['value'] + return { + 'fee': isum - osum, + 'outs': outs, + 'ins': ins + } + + +def merkle_prove(txhash): + blocknum = str(get_block_height(txhash)) + header = get_block_header_data(blocknum) + hashes = get_txs_in_block(blocknum) + i = hashes.index(txhash) + return mk_merkle_proof(header, hashes, i) diff --git a/lib/bitcoin/deterministic.py b/lib/bitcoin/deterministic.py index ac463d5d..200dc4a4 100644 --- a/lib/bitcoin/deterministic.py +++ b/lib/bitcoin/deterministic.py @@ -1,137 +1,174 @@ -from main import * -import hmac, hashlib +from bitcoin.main import * +import hmac +import hashlib +from binascii import hexlify +# Electrum wallets -### Electrum wallets -def electrum_stretch(seed): return slowsha(seed) +def electrum_stretch(seed): + return slowsha(seed) # Accepts seed or stretched seed, returns master public key + + def electrum_mpk(seed): - if len(seed) == 32: seed = electrum_stretch(seed) + if len(seed) == 32: + seed = electrum_stretch(seed) return privkey_to_pubkey(seed)[2:] # Accepts (seed or stretched seed), index and secondary index # (conventionally 0 for ordinary addresses, 1 for change) , returns privkey -def electrum_privkey(seed,n,for_change=0): - if len(seed) == 32: seed = electrum_stretch(seed) + + +def electrum_privkey(seed, n, for_change=0): + if len(seed) == 32: + seed = electrum_stretch(seed) mpk = electrum_mpk(seed) - offset = dbl_sha256(str(n)+':'+str(for_change)+':'+mpk.decode('hex')) + offset = dbl_sha256(from_int_representation_to_bytes(n)+b':'+from_int_representation_to_bytes(for_change)+b':'+binascii.unhexlify(mpk)) return add_privkeys(seed, offset) -# Accepts (seed or stretched seed or master public key), index and secondary index +# Accepts (seed or stretched seed or master pubkey), index and secondary index # (conventionally 0 for ordinary addresses, 1 for change) , returns pubkey -def electrum_pubkey(masterkey,n,for_change=0): - if len(masterkey) == 32: mpk = electrum_mpk(electrum_stretch(masterkey)) - elif len(masterkey) == 64: mpk = electrum_mpk(masterkey) - else: mpk = masterkey - bin_mpk = encode_pubkey(mpk,'bin_electrum') - offset = bin_dbl_sha256(str(n)+':'+str(for_change)+':'+bin_mpk) - return add_pubkeys('04'+mpk,privtopub(offset)) + + +def electrum_pubkey(masterkey, n, for_change=0): + if len(masterkey) == 32: + mpk = electrum_mpk(electrum_stretch(masterkey)) + elif len(masterkey) == 64: + mpk = electrum_mpk(masterkey) + else: + mpk = masterkey + bin_mpk = encode_pubkey(mpk, 'bin_electrum') + offset = bin_dbl_sha256(from_int_representation_to_bytes(n)+b':'+from_int_representation_to_bytes(for_change)+b':'+bin_mpk) + return add_pubkeys('04'+mpk, privtopub(offset)) # seed/stretched seed/pubkey -> address (convenience method) -def electrum_address(masterkey,n,for_change=0,version=0): - return pubkey_to_address(electrum_pubkey(masterkey,n,for_change),version) + + +def electrum_address(masterkey, n, for_change=0, version=0): + return pubkey_to_address(electrum_pubkey(masterkey, n, for_change), version) # Given a master public key, a private key from that wallet and its index, # cracks the secret exponent which can be used to generate all other private # keys in the wallet -def crack_electrum_wallet(mpk,pk,n,for_change=0): - bin_mpk = encode_pubkey(mpk,'bin_electrum') + + +def crack_electrum_wallet(mpk, pk, n, for_change=0): + bin_mpk = encode_pubkey(mpk, 'bin_electrum') offset = dbl_sha256(str(n)+':'+str(for_change)+':'+bin_mpk) return subtract_privkeys(pk, offset) # Below code ASSUMES binary inputs and compressed pubkeys -PRIVATE = '\x04\x88\xAD\xE4' -PUBLIC = '\x04\x88\xB2\x1E' +MAINNET_PRIVATE = b'\x04\x88\xAD\xE4' +MAINNET_PUBLIC = b'\x04\x88\xB2\x1E' +TESTNET_PRIVATE = b'\x04\x35\x83\x94' +TESTNET_PUBLIC = b'\x04\x35\x87\xCF' +PRIVATE = [MAINNET_PRIVATE, TESTNET_PRIVATE] +PUBLIC = [MAINNET_PUBLIC, TESTNET_PUBLIC] # BIP32 child key derivation + + def raw_bip32_ckd(rawtuple, i): vbytes, depth, fingerprint, oldi, chaincode, key = rawtuple i = int(i) - if vbytes == PRIVATE: + if vbytes in PRIVATE: priv = key pub = privtopub(key) else: pub = key if i >= 2**31: - if vbytes == PUBLIC: + if vbytes in PUBLIC: raise Exception("Can't do private derivation on public key!") - I = hmac.new(chaincode,'\x00'+priv[:32]+encode(i,256,4),hashlib.sha512).digest() + I = hmac.new(chaincode, b'\x00'+priv[:32]+encode(i, 256, 4), hashlib.sha512).digest() else: - I = hmac.new(chaincode,pub+encode(i,256,4),hashlib.sha512).digest() + I = hmac.new(chaincode, pub+encode(i, 256, 4), hashlib.sha512).digest() - if vbytes == PRIVATE: - newkey = add_privkeys(I[:32]+'\x01',priv) + if vbytes in PRIVATE: + newkey = add_privkeys(I[:32]+B'\x01', priv) fingerprint = bin_hash160(privtopub(key))[:4] - if vbytes == PUBLIC: - newkey = add_pubkeys(compress(privtopub(I[:32])),key) + if vbytes in PUBLIC: + newkey = add_pubkeys(compress(privtopub(I[:32])), key) fingerprint = bin_hash160(key)[:4] return (vbytes, depth + 1, fingerprint, i, I[32:], newkey) + def bip32_serialize(rawtuple): vbytes, depth, fingerprint, i, chaincode, key = rawtuple - depth = chr(depth % 256) - i = encode(i,256,4) - chaincode = encode(hash_to_int(chaincode),256,32) - keydata = '\x00'+key[:-1] if vbytes == PRIVATE else key - bindata = vbytes + depth + fingerprint + i + chaincode + keydata - return changebase(bindata+bin_dbl_sha256(bindata)[:4],256,58) + i = encode(i, 256, 4) + chaincode = encode(hash_to_int(chaincode), 256, 32) + keydata = b'\x00'+key[:-1] if vbytes in PRIVATE else key + bindata = vbytes + from_int_to_byte(depth % 256) + fingerprint + i + chaincode + keydata + return changebase(bindata+bin_dbl_sha256(bindata)[:4], 256, 58) + def bip32_deserialize(data): - dbin = changebase(data,58,256) + dbin = changebase(data, 58, 256) if bin_dbl_sha256(dbin[:-4])[:4] != dbin[-4:]: raise Exception("Invalid checksum") vbytes = dbin[0:4] - depth = ord(dbin[4]) + depth = from_byte_to_int(dbin[4]) fingerprint = dbin[5:9] - i = decode(dbin[9:13],256) + i = decode(dbin[9:13], 256) chaincode = dbin[13:45] - key = dbin[46:78]+'\x01' if vbytes == PRIVATE else dbin[45:78] + key = dbin[46:78]+b'\x01' if vbytes in PRIVATE else dbin[45:78] return (vbytes, depth, fingerprint, i, chaincode, key) + def raw_bip32_privtopub(rawtuple): vbytes, depth, fingerprint, i, chaincode, key = rawtuple - return (PUBLIC, depth, fingerprint, i, chaincode, privtopub(key)) + newvbytes = MAINNET_PUBLIC if vbytes == MAINNET_PRIVATE else TESTNET_PUBLIC + return (newvbytes, depth, fingerprint, i, chaincode, privtopub(key)) + def bip32_privtopub(data): return bip32_serialize(raw_bip32_privtopub(bip32_deserialize(data))) -def bip32_ckd(data,i): - return bip32_serialize(raw_bip32_ckd(bip32_deserialize(data),i)) -def bip32_master_key(seed): - I = hmac.new("Bitcoin seed",seed,hashlib.sha512).digest() - return bip32_serialize((PRIVATE, 0, '\x00'*4, 0, I[32:], I[:32]+'\x01')) +def bip32_ckd(data, i): + return bip32_serialize(raw_bip32_ckd(bip32_deserialize(data), i)) + + +def bip32_master_key(seed, vbytes=MAINNET_PRIVATE): + I = hmac.new(from_string_to_bytes("Bitcoin seed"), seed, hashlib.sha512).digest() + return bip32_serialize((vbytes, 0, b'\x00'*4, 0, I[32:], I[:32]+b'\x01')) + def bip32_bin_extract_key(data): return bip32_deserialize(data)[-1] + def bip32_extract_key(data): - return bip32_deserialize(data)[-1].encode('hex') + return safe_hexlify(bip32_deserialize(data)[-1]) # Exploits the same vulnerability as above in Electrum wallets -# Takes a BIP32 pubkey and one of the child privkeys of its corresponding privkey -# and returns the BIP32 privkey associated with that pubkey -def raw_crack_bip32_privkey(parent_pub,priv): +# Takes a BIP32 pubkey and one of the child privkeys of its corresponding +# privkey and returns the BIP32 privkey associated with that pubkey + + +def raw_crack_bip32_privkey(parent_pub, priv): vbytes, depth, fingerprint, i, chaincode, key = priv pvbytes, pdepth, pfingerprint, pi, pchaincode, pkey = parent_pub i = int(i) - if i >= 2**31: raise Exception("Can't crack private derivation!") + if i >= 2**31: + raise Exception("Can't crack private derivation!") + + I = hmac.new(pchaincode, pkey+encode(i, 256, 4), hashlib.sha512).digest() - I = hmac.new(pchaincode,pkey+encode(i,256,4),hashlib.sha512).digest() + pprivkey = subtract_privkeys(key, I[:32]+b'\x01') - pprivkey = subtract_privkeys(key,I[:32]+'\x01') + newvbytes = MAINNET_PRIVATE if vbytes == MAINNET_PUBLIC else TESTNET_PRIVATE + return (newvbytes, pdepth, pfingerprint, pi, pchaincode, pprivkey) - return (PRIVATE, pdepth, pfingerprint, pi, pchaincode, pprivkey) -def crack_bip32_privkey(parent_pub,priv): +def crack_bip32_privkey(parent_pub, priv): dsppub = bip32_deserialize(parent_pub) dspriv = bip32_deserialize(priv) - return bip32_serialize(raw_crack_bip32_privkey(dsppub,dspriv)) + return bip32_serialize(raw_crack_bip32_privkey(dsppub, dspriv)) def coinvault_pub_to_bip32(*args): @@ -140,7 +177,7 @@ def coinvault_pub_to_bip32(*args): vals = map(int, args[34:]) I1 = ''.join(map(chr, vals[:33])) I2 = ''.join(map(chr, vals[35:67])) - return bip32_serialize((PUBLIC, 0, '\x00'*4, 0, I2, I1)) + return bip32_serialize((MAINNET_PUBLIC, 0, b'\x00'*4, 0, I2, I1)) def coinvault_priv_to_bip32(*args): @@ -149,7 +186,7 @@ def coinvault_priv_to_bip32(*args): vals = map(int, args[34:]) I2 = ''.join(map(chr, vals[35:67])) I3 = ''.join(map(chr, vals[72:104])) - return bip32_serialize((PRIVATE, 0, '\x00'*4, 0, I2, I3+'\x01')) + return bip32_serialize((MAINNET_PRIVATE, 0, b'\x00'*4, 0, I2, I3+b'\x01')) def bip32_descend(*args): diff --git a/lib/bitcoin/main.py b/lib/bitcoin/main.py index c5358950..daa66bd9 100644 --- a/lib/bitcoin/main.py +++ b/lib/bitcoin/main.py @@ -1,16 +1,26 @@ #!/usr/bin/python -import hashlib, re, sys, os, base64, time, random, hmac -import ripemd - -### Elliptic curve parameters (secp256k1) - -P = 2**256-2**32-2**9-2**8-2**7-2**6-2**4-1 +from .py2specials import * +from .py3specials import * +import binascii +import hashlib +import re +import sys +import os +import base64 +import time +import random +import hmac +from bitcoin.ripemd import * + +# Elliptic curve parameters (secp256k1) + +P = 2**256 - 2**32 - 977 N = 115792089237316195423570985008687907852837564279074904382605163141518161494337 A = 0 B = 7 Gx = 55066263022277343669578718895168534326250603453777594175500187360389116729240 Gy = 32670510020758816978083085130507043184471273380659243275938904335757337482424 -G = (Gx,Gy) +G = (Gx, Gy) def change_curve(p, n, a, b, gx, gy): @@ -18,155 +28,205 @@ def change_curve(p, n, a, b, gx, gy): P, N, A, B, Gx, Gy = p, n, a, b, gx, gy G = (Gx, Gy) + def getG(): return G -### Extended Euclidean Algorithm +# Extended Euclidean Algorithm + -def inv(a,n): - lm, hm = 1,0 - low, high = a%n,n +def inv(a, n): + lm, hm = 1, 0 + low, high = a % n, n while low > 1: - r = high/low + r = high//low nm, new = hm-lm*r, high-low*r lm, low, hm, high = nm, new, lm, low return lm % n -### Base switching - -def get_code_string(base): - if base == 2: return '01' - elif base == 10: return '0123456789' - elif base == 16: return '0123456789abcdef' - elif base == 32: return 'abcdefghijklmnopqrstuvwxyz234567' - elif base == 58: return '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' - elif base == 256: return ''.join([chr(x) for x in range(256)]) - else: raise ValueError("Invalid base!") - -def lpad(msg,symbol,length): - if len(msg) >= length: return msg - return symbol * (length - len(msg)) + msg - -def encode(val,base,minlen=0): - base, minlen = int(base), int(minlen) - code_string = get_code_string(base) - result = "" - while val > 0: - result = code_string[val % base] + result - val /= base - return lpad(result,code_string[0],minlen) - -def decode(string,base): - base = int(base) - code_string = get_code_string(base) - result = 0 - if base == 16: string = string.lower() - while len(string) > 0: - result *= base - result += code_string.find(string[0]) - string = string[1:] - return result - -def changebase(string,frm,to,minlen=0): - if frm == to: return lpad(string,get_code_string(frm)[0],minlen) - return encode(decode(string,frm),to,minlen) - -### JSON access (for pybtctool convenience) - -def access(obj,prop): - if isinstance(obj,dict): - if prop in obj: return obj[prop] - elif '.' in prop: return obj[float(prop)] - else: return obj[int(prop)] + + +# JSON access (for pybtctool convenience) + + +def access(obj, prop): + if isinstance(obj, dict): + if prop in obj: + return obj[prop] + elif '.' in prop: + return obj[float(prop)] + else: + return obj[int(prop)] else: return obj[int(prop)] -def multiaccess(obj,prop): - return [access(o,prop) for o in obj] -def slice(obj,start=0,end=2**200): +def multiaccess(obj, prop): + return [access(o, prop) for o in obj] + + +def slice(obj, start=0, end=2**200): return obj[int(start):int(end)] + def count(obj): return len(obj) _sum = sum + + def sum(obj): return _sum(obj) - -### Elliptic Curve functions - -def isinf(p): return p[0] == 0 and p[1] == 0 - -def base10_add(a,b): - if isinf(a): return b[0],b[1] - if isinf(b): return a[0],a[1] - if a[0] == b[0]: - if a[1] == b[1]: return base10_double((a[0],a[1])) - else: return (0,0) - m = ((b[1]-a[1]) * inv(b[0]-a[0],P)) % P - x = (m*m-a[0]-b[0]) % P - y = (m*(a[0]-x)-a[1]) % P - return (x,y) - -def base10_double(a): - if isinf(a): return (0,0) - m = ((3*a[0]*a[0]+A)*inv(2*a[1],P)) % P - x = (m*m-2*a[0]) % P - y = (m*(a[0]-x)-a[1]) % P - return (x,y) - -def base10_multiply(a,n): - if isinf(a) or n == 0: return (0,0) - if n == 1: return a - if n < 0 or n >= N: return base10_multiply(a,n%N) - if (n%2) == 0: return base10_double(base10_multiply(a,n/2)) - if (n%2) == 1: return base10_add(base10_double(base10_multiply(a,n/2)),a) + + +# Elliptic curve Jordan form functions +# P = (m, n, p, q) where m/n = x, p/q = y + +def isinf(p): + return p[0] == 0 and p[1] == 0 + + +def jordan_isinf(p): + return p[0][0] == 0 and p[1][0] == 0 + + +def mulcoords(c1, c2): + return (c1[0] * c2[0] % P, c1[1] * c2[1] % P) + + +def mul_by_const(c, v): + return (c[0] * v % P, c[1]) + + +def addcoords(c1, c2): + return ((c1[0] * c2[1] + c2[0] * c1[1]) % P, c1[1] * c2[1] % P) + + +def subcoords(c1, c2): + return ((c1[0] * c2[1] - c2[0] * c1[1]) % P, c1[1] * c2[1] % P) + + +def invcoords(c): + return (c[1], c[0]) + + +def jordan_add(a, b): + if jordan_isinf(a): + return b + if jordan_isinf(b): + return a + + if (a[0][0] * b[0][1] - b[0][0] * a[0][1]) % P == 0: + if (a[1][0] * b[1][1] - b[1][0] * a[1][1]) % P == 0: + return jordan_double(a) + else: + return ((0, 1), (0, 1)) + xdiff = subcoords(b[0], a[0]) + ydiff = subcoords(b[1], a[1]) + m = mulcoords(ydiff, invcoords(xdiff)) + x = subcoords(subcoords(mulcoords(m, m), a[0]), b[0]) + y = subcoords(mulcoords(m, subcoords(a[0], x)), a[1]) + return (x, y) + + +def jordan_double(a): + if jordan_isinf(a): + return ((0, 1), (0, 1)) + num = addcoords(mul_by_const(mulcoords(a[0], a[0]), 3), (A, 1)) + den = mul_by_const(a[1], 2) + m = mulcoords(num, invcoords(den)) + x = subcoords(mulcoords(m, m), mul_by_const(a[0], 2)) + y = subcoords(mulcoords(m, subcoords(a[0], x)), a[1]) + return (x, y) + + +def jordan_multiply(a, n): + if jordan_isinf(a) or n == 0: + return ((0, 0), (0, 0)) + if n == 1: + return a + if n < 0 or n >= N: + return jordan_multiply(a, n % N) + if (n % 2) == 0: + return jordan_double(jordan_multiply(a, n//2)) + if (n % 2) == 1: + return jordan_add(jordan_double(jordan_multiply(a, n//2)), a) + + +def to_jordan(p): + return ((p[0], 1), (p[1], 1)) + + +def from_jordan(p): + return (p[0][0] * inv(p[0][1], P) % P, p[1][0] * inv(p[1][1], P) % P) + return (p[0][0] * inv(p[0][1], P) % P, p[1][0] * inv(p[1][1], P) % P) + + +def fast_multiply(a, n): + return from_jordan(jordan_multiply(to_jordan(a), n)) + + +def fast_add(a, b): + return from_jordan(jordan_add(to_jordan(a), to_jordan(b))) # Functions for handling pubkey and privkey formats + def get_pubkey_format(pub): - if isinstance(pub,(tuple,list)): return 'decimal' - elif len(pub) == 65 and pub[0] == '\x04': return 'bin' + if is_python2: + two = '\x02' + three = '\x03' + four = '\x04' + else: + two = 2 + three = 3 + four = 4 + + if isinstance(pub, (tuple, list)): return 'decimal' + elif len(pub) == 65 and pub[0] == four: return 'bin' elif len(pub) == 130 and pub[0:2] == '04': return 'hex' - elif len(pub) == 33 and pub[0] in ['\x02','\x03']: return 'bin_compressed' - elif len(pub) == 66 and pub[0:2] in ['02','03']: return 'hex_compressed' + elif len(pub) == 33 and pub[0] in [two, three]: return 'bin_compressed' + elif len(pub) == 66 and pub[0:2] in ['02', '03']: return 'hex_compressed' elif len(pub) == 64: return 'bin_electrum' elif len(pub) == 128: return 'hex_electrum' else: raise Exception("Pubkey not in recognized format") -def encode_pubkey(pub,formt): - if not isinstance(pub,(tuple,list)): + +def encode_pubkey(pub, formt): + if not isinstance(pub, (tuple, list)): pub = decode_pubkey(pub) if formt == 'decimal': return pub - elif formt == 'bin': return '\x04' + encode(pub[0],256,32) + encode(pub[1],256,32) - elif formt == 'bin_compressed': return chr(2+(pub[1]%2)) + encode(pub[0],256,32) - elif formt == 'hex': return '04' + encode(pub[0],16,64) + encode(pub[1],16,64) - elif formt == 'hex_compressed': return '0'+str(2+(pub[1]%2)) + encode(pub[0],16,64) - elif formt == 'bin_electrum': return encode(pub[0],256,32) + encode(pub[1],256,32) - elif formt == 'hex_electrum': return encode(pub[0],16,64) + encode(pub[1],16,64) + elif formt == 'bin': return b'\x04' + encode(pub[0], 256, 32) + encode(pub[1], 256, 32) + elif formt == 'bin_compressed': + return from_int_to_byte(2+(pub[1] % 2)) + encode(pub[0], 256, 32) + elif formt == 'hex': return '04' + encode(pub[0], 16, 64) + encode(pub[1], 16, 64) + elif formt == 'hex_compressed': + return '0'+str(2+(pub[1] % 2)) + encode(pub[0], 16, 64) + elif formt == 'bin_electrum': return encode(pub[0], 256, 32) + encode(pub[1], 256, 32) + elif formt == 'hex_electrum': return encode(pub[0], 16, 64) + encode(pub[1], 16, 64) else: raise Exception("Invalid format!") -def decode_pubkey(pub,formt=None): + +def decode_pubkey(pub, formt=None): if not formt: formt = get_pubkey_format(pub) if formt == 'decimal': return pub - elif formt == 'bin': return (decode(pub[1:33],256),decode(pub[33:65],256)) + elif formt == 'bin': return (decode(pub[1:33], 256), decode(pub[33:65], 256)) elif formt == 'bin_compressed': - x = decode(pub[1:33],256) - beta = pow(x*x*x+A*x+B,(P+1)/4,P) - y = (P-beta) if ((beta + ord(pub[0])) % 2) else beta - return (x,y) - elif formt == 'hex': return (decode(pub[2:66],16),decode(pub[66:130],16)) + x = decode(pub[1:33], 256) + beta = pow(int(x*x*x+A*x+B), int((P+1)//4), int(P)) + y = (P-beta) if ((beta + from_byte_to_int(pub[0])) % 2) else beta + return (x, y) + elif formt == 'hex': return (decode(pub[2:66], 16), decode(pub[66:130], 16)) elif formt == 'hex_compressed': - return decode_pubkey(pub.decode('hex'),'bin_compressed') + return decode_pubkey(safe_from_hex(pub), 'bin_compressed') elif formt == 'bin_electrum': - return (decode(pub[:32],256),decode(pub[32:64],256)) + return (decode(pub[:32], 256), decode(pub[32:64], 256)) elif formt == 'hex_electrum': - return (decode(pub[:64],16),decode(pub[64:128],16)) + return (decode(pub[:64], 16), decode(pub[64:128], 16)) else: raise Exception("Invalid format!") def get_privkey_format(priv): - if isinstance(priv,(int,long)): return 'decimal' + if isinstance(priv, int_types): return 'decimal' elif len(priv) == 32: return 'bin' elif len(priv) == 33: return 'bin_compressed' elif len(priv) == 64: return 'hex' @@ -177,255 +237,314 @@ def get_privkey_format(priv): elif len(bin_p) == 33: return 'wif_compressed' else: raise Exception("WIF does not represent privkey") -def encode_privkey(priv,formt,vbyte=0): - if not isinstance(priv,(int,long)): - return encode_privkey(decode_privkey(priv),formt,vbyte) +def encode_privkey(priv, formt, vbyte=0): + if not isinstance(priv, int_types): + return encode_privkey(decode_privkey(priv), formt, vbyte) if formt == 'decimal': return priv - elif formt == 'bin': return encode(priv,256,32) - elif formt == 'bin_compressed': return encode(priv,256,32)+'\x01' - elif formt == 'hex': return encode(priv,16,64) - elif formt == 'hex_compressed': return encode(priv,16,64)+'01' + elif formt == 'bin': return encode(priv, 256, 32) + elif formt == 'bin_compressed': return encode(priv, 256, 32)+b'\x01' + elif formt == 'hex': return encode(priv, 16, 64) + elif formt == 'hex_compressed': return encode(priv, 16, 64)+'01' elif formt == 'wif': - return bin_to_b58check(encode(priv,256,32),128+int(vbyte)) + return bin_to_b58check(encode(priv, 256, 32), 128+int(vbyte)) elif formt == 'wif_compressed': - return bin_to_b58check(encode(priv,256,32)+'\x01',128+int(vbyte)) + return bin_to_b58check(encode(priv, 256, 32)+b'\x01', 128+int(vbyte)) else: raise Exception("Invalid format!") def decode_privkey(priv,formt=None): if not formt: formt = get_privkey_format(priv) if formt == 'decimal': return priv - elif formt == 'bin': return decode(priv,256) - elif formt == 'bin_compressed': return decode(priv[:32],256) - elif formt == 'hex': return decode(priv,16) - elif formt == 'hex_compressed': return decode(priv[:64],16) - else: - bin_p = b58check_to_bin(priv) - if len(bin_p) == 32: return decode(bin_p,256) - elif len(bin_p) == 33: return decode(bin_p[:32],256) - else: raise Exception("WIF does not represent privkey") + elif formt == 'bin': return decode(priv, 256) + elif formt == 'bin_compressed': return decode(priv[:32], 256) + elif formt == 'hex': return decode(priv, 16) + elif formt == 'hex_compressed': return decode(priv[:64], 16) + elif formt == 'wif': return decode(b58check_to_bin(priv),256) + elif formt == 'wif_compressed': + return decode(b58check_to_bin(priv)[:32],256) + else: raise Exception("WIF does not represent privkey") + +def add_pubkeys(p1, p2): + f1, f2 = get_pubkey_format(p1), get_pubkey_format(p2) + return encode_pubkey(fast_add(decode_pubkey(p1, f1), decode_pubkey(p2, f2)), f1) + +def add_privkeys(p1, p2): + f1, f2 = get_privkey_format(p1), get_privkey_format(p2) + return encode_privkey((decode_privkey(p1, f1) + decode_privkey(p2, f2)) % N, f1) -def add_pubkeys(p1,p2): - f1,f2 = get_pubkey_format(p1), get_pubkey_format(p2) - return encode_pubkey(base10_add(decode_pubkey(p1,f1),decode_pubkey(p2,f2)),f1) -def add_privkeys(p1,p2): - f1,f2 = get_privkey_format(p1), get_privkey_format(p2) - return encode_privkey((decode_privkey(p1,f1) + decode_privkey(p2,f2)) % N,f1) +def multiply(pubkey, privkey): + f1, f2 = get_pubkey_format(pubkey), get_privkey_format(privkey) + pubkey, privkey = decode_pubkey(pubkey, f1), decode_privkey(privkey, f2) + # http://safecurves.cr.yp.to/twist.html + if not isinf(pubkey) and (pubkey[0]**3+B-pubkey[1]*pubkey[1]) % P != 0: + raise Exception("Point not on curve") + return encode_pubkey(fast_multiply(pubkey, privkey), f1) -def multiply(pubkey,privkey): - f1,f2 = get_pubkey_format(pubkey), get_privkey_format(privkey) - pubkey, privkey = decode_pubkey(pubkey,f1), decode_privkey(privkey,f2) - # http://safecurves.cr.yp.to/twist.html - if not isinf(pubkey) and (pubkey[0]**3+B-pubkey[1]*pubkey[1]) % P != 0: - raise Exception("Point not on curve") - return encode_pubkey(base10_multiply(pubkey,privkey),f1) -def divide(pubkey,privkey): - factor = inv(decode_privkey(privkey),N) - return multiply(pubkey,factor) +def divide(pubkey, privkey): + factor = inv(decode_privkey(privkey), N) + return multiply(pubkey, factor) + def compress(pubkey): f = get_pubkey_format(pubkey) if 'compressed' in f: return pubkey - elif f == 'bin': return encode_pubkey(decode_pubkey(pubkey,f),'bin_compressed') + elif f == 'bin': return encode_pubkey(decode_pubkey(pubkey, f), 'bin_compressed') elif f == 'hex' or f == 'decimal': - return encode_pubkey(decode_pubkey(pubkey,f),'hex_compressed') + return encode_pubkey(decode_pubkey(pubkey, f), 'hex_compressed') + def decompress(pubkey): f = get_pubkey_format(pubkey) if 'compressed' not in f: return pubkey - elif f == 'bin_compressed': return encode_pubkey(decode_pubkey(pubkey,f),'bin') + elif f == 'bin_compressed': return encode_pubkey(decode_pubkey(pubkey, f), 'bin') elif f == 'hex_compressed' or f == 'decimal': - return encode_pubkey(decode_pubkey(pubkey,f),'hex') + return encode_pubkey(decode_pubkey(pubkey, f), 'hex') + def privkey_to_pubkey(privkey): f = get_privkey_format(privkey) - privkey = decode_privkey(privkey,f) - if privkey == 0 or privkey >= N: + privkey = decode_privkey(privkey, f) + if privkey >= N: raise Exception("Invalid privkey") - if f in ['bin','bin_compressed','hex','hex_compressed','decimal']: - return encode_pubkey(base10_multiply(G,privkey),f) + if f in ['bin', 'bin_compressed', 'hex', 'hex_compressed', 'decimal']: + return encode_pubkey(fast_multiply(G, privkey), f) else: - return encode_pubkey(base10_multiply(G,privkey),f.replace('wif','hex')) + return encode_pubkey(fast_multiply(G, privkey), f.replace('wif', 'hex')) privtopub = privkey_to_pubkey -def privkey_to_address(priv,magicbyte=0): - return pubkey_to_address(privkey_to_pubkey(priv),magicbyte) + +def privkey_to_address(priv, magicbyte=0): + return pubkey_to_address(privkey_to_pubkey(priv), magicbyte) privtoaddr = privkey_to_address -def neg_pubkey(pubkey): + +def neg_pubkey(pubkey): f = get_pubkey_format(pubkey) - pubkey = decode_pubkey(pubkey,f) - return encode_pubkey((pubkey[0],(P-pubkey[1]) % P),f) + pubkey = decode_pubkey(pubkey, f) + return encode_pubkey((pubkey[0], (P-pubkey[1]) % P), f) + def neg_privkey(privkey): f = get_privkey_format(privkey) - privkey = decode_privkey(privkey,f) - return encode_privkey((N - privkey) % N,f) + privkey = decode_privkey(privkey, f) + return encode_privkey((N - privkey) % N, f) def subtract_pubkeys(p1, p2): - f1,f2 = get_pubkey_format(p1), get_pubkey_format(p2) - k2 = decode_pubkey(p2,f2) - return encode_pubkey(base10_add(decode_pubkey(p1,f1),(k2[0],(P - k2[1]) % P)),f1) + f1, f2 = get_pubkey_format(p1), get_pubkey_format(p2) + k2 = decode_pubkey(p2, f2) + return encode_pubkey(fast_add(decode_pubkey(p1, f1), (k2[0], (P - k2[1]) % P)), f1) + def subtract_privkeys(p1, p2): - f1,f2 = get_privkey_format(p1), get_privkey_format(p2) - k2 = decode_privkey(p2,f2) - return encode_privkey((decode_privkey(p1,f1) - k2) % N,f1) + f1, f2 = get_privkey_format(p1), get_privkey_format(p2) + k2 = decode_privkey(p2, f2) + return encode_privkey((decode_privkey(p1, f1) - k2) % N, f1) + +# Hashes -### Hashes def bin_hash160(string): - intermed = hashlib.sha256(string).digest() - digest = '' - try: - digest = hashlib.new('ripemd160',intermed).digest() - except: - digest = ripemd.RIPEMD160(intermed).digest() - return digest + intermed = hashlib.sha256(string).digest() + digest = '' + try: + digest = hashlib.new('ripemd160', intermed).digest() + except: + digest = RIPEMD160(intermed).digest() + return digest + + def hash160(string): - return bin_hash160(string).encode('hex') + return safe_hexlify(bin_hash160(string)) + def bin_sha256(string): - return hashlib.sha256(string).digest() + binary_data = string if isinstance(string, bytes) else bytes(string, 'utf-8') + return hashlib.sha256(binary_data).digest() + def sha256(string): - return bin_sha256(string).encode('hex') + return bytes_to_hex_string(bin_sha256(string)) + + +def bin_ripemd160(string): + try: + digest = hashlib.new('ripemd160', string).digest() + except: + digest = RIPEMD160(string).digest() + return digest + + +def ripemd160(string): + return safe_hexlify(bin_ripemd160(string)) + + +def bin_dbl_sha256(s): + bytes_to_hash = from_string_to_bytes(s) + return hashlib.sha256(hashlib.sha256(bytes_to_hash).digest()).digest() + -def bin_dbl_sha256(string): - return hashlib.sha256(hashlib.sha256(string).digest()).digest() def dbl_sha256(string): - return bin_dbl_sha256(string).encode('hex') + return safe_hexlify(bin_dbl_sha256(string)) + def bin_slowsha(string): + string = from_string_to_bytes(string) orig_input = string for i in range(100000): string = hashlib.sha256(string + orig_input).digest() return string + + def slowsha(string): - return bin_slowsha(string).encode('hex') + return safe_hexlify(bin_slowsha(string)) + def hash_to_int(x): - if len(x) in [40,64]: return decode(x,16) - else: return decode(x,256) + if len(x) in [40, 64]: + return decode(x, 16) + return decode(x, 256) + def num_to_var_int(x): x = int(x) - if x < 253: return chr(x) - elif x < 65536: return chr(253) + encode(x,256,2)[::-1] - elif x < 4294967296: return chr(254) + encode(x,256,4)[::-1] - else: return chr(255) + encode(x,256,8)[::-1] + if x < 253: return from_int_to_byte(x) + elif x < 65536: return from_int_to_byte(253)+encode(x, 256, 2)[::-1] + elif x < 4294967296: return from_int_to_byte(254) + encode(x, 256, 4)[::-1] + else: return from_int_to_byte(255) + encode(x, 256, 8)[::-1] + # WTF, Electrum? def electrum_sig_hash(message): - padded = "\x18Bitcoin Signed Message:\n" + num_to_var_int( len(message) ) + message + padded = b"\x18Bitcoin Signed Message:\n" + num_to_var_int(len(message)) + from_string_to_bytes(message) return bin_dbl_sha256(padded) + def random_key(): # Gotta be secure after that java.SecureRandom fiasco... - entropy = os.urandom(32)+str(random.randrange(2**256))+str(int(time.time())**7) + entropy = random_string(32) \ + + str(random.randrange(2**256)) \ + + str(int(time.time() * 1000000)) return sha256(entropy) + def random_electrum_seed(): - entropy = os.urandom(32)+str(random.randrange(2**256))+str(int(time.time())**7) + entropy = os.urandom(32) \ + + str(random.randrange(2**256)) \ + + str(int(time.time() * 1000000)) return sha256(entropy)[:32] -### Encodings - -def bin_to_b58check(inp,magicbyte=0): - inp_fmtd = chr(int(magicbyte)) + inp - leadingzbytes = len(re.match('^\x00*',inp_fmtd).group(0)) - checksum = bin_dbl_sha256(inp_fmtd)[:4] - return '1' * leadingzbytes + changebase(inp_fmtd+checksum,256,58) +# Encodings def b58check_to_bin(inp): - leadingzbytes = len(re.match('^1*',inp).group(0)) - data = '\x00' * leadingzbytes + changebase(inp,58,256) + leadingzbytes = len(re.match('^1*', inp).group(0)) + data = b'\x00' * leadingzbytes + changebase(inp, 58, 256) assert bin_dbl_sha256(data[:-4])[:4] == data[-4:] return data[1:-4] + def get_version_byte(inp): - leadingzbytes = len(re.match('^1*',inp).group(0)) - data = '\x00' * leadingzbytes + changebase(inp,58,256) + leadingzbytes = len(re.match('^1*', inp).group(0)) + data = b'\x00' * leadingzbytes + changebase(inp, 58, 256) assert bin_dbl_sha256(data[:-4])[:4] == data[-4:] return ord(data[0]) -def hex_to_b58check(inp,magicbyte=0): - return bin_to_b58check(inp.decode('hex'),magicbyte) -def b58check_to_hex(inp): return b58check_to_bin(inp).encode('hex') +def hex_to_b58check(inp, magicbyte=0): + return bin_to_b58check(binascii.unhexlify(inp), magicbyte) -def pubkey_to_address(pubkey,magicbyte=0): - if isinstance(pubkey,(list,tuple)): - pubkey = encode_pubkey(pubkey,'bin') - if len(pubkey) in [66,130]: - return bin_to_b58check(bin_hash160(pubkey.decode('hex')),magicbyte) - return bin_to_b58check(bin_hash160(pubkey),magicbyte) + +def b58check_to_hex(inp): + return safe_hexlify(b58check_to_bin(inp)) + + +def pubkey_to_address(pubkey, magicbyte=0): + if isinstance(pubkey, (list, tuple)): + pubkey = encode_pubkey(pubkey, 'bin') + if len(pubkey) in [66, 130]: + return bin_to_b58check( + bin_hash160(binascii.unhexlify(pubkey)), magicbyte) + return bin_to_b58check(bin_hash160(pubkey), magicbyte) pubtoaddr = pubkey_to_address -### EDCSA +# EDCSA + + +def encode_sig(v, r, s): + vb, rb, sb = from_int_to_byte(v), encode(r, 256), encode(s, 256) + + result = base64.b64encode(vb+b'\x00'*(32-len(rb))+rb+b'\x00'*(32-len(sb))+sb) + return result if is_python2 else str(result, 'utf-8') -def encode_sig(v,r,s): - vb, rb, sb = chr(v), encode(r,256), encode(s,256) - return base64.b64encode(vb+'\x00'*(32-len(rb))+rb+'\x00'*(32-len(sb))+sb) def decode_sig(sig): bytez = base64.b64decode(sig) - return ord(bytez[0]), decode(bytez[1:33],256), decode(bytez[33:],256) + return from_byte_to_int(bytez[0]), decode(bytez[1:33], 256), decode(bytez[33:], 256) # https://tools.ietf.org/html/rfc6979#section-3.2 -def deterministic_generate_k(msghash,priv): - v = '\x01' * 32 - k = '\x00' * 32 - priv = encode_privkey(priv,'bin') - msghash = encode(hash_to_int(msghash),256,32) - k = hmac.new(k, v+'\x00'+priv+msghash, hashlib.sha256).digest() + + +def deterministic_generate_k(msghash, priv): + v = b'\x01' * 32 + k = b'\x00' * 32 + priv = encode_privkey(priv, 'bin') + msghash = encode(hash_to_int(msghash), 256, 32) + k = hmac.new(k, v+b'\x00'+priv+msghash, hashlib.sha256).digest() v = hmac.new(k, v, hashlib.sha256).digest() - k = hmac.new(k, v+'\x01'+priv+msghash, hashlib.sha256).digest() + k = hmac.new(k, v+b'\x01'+priv+msghash, hashlib.sha256).digest() v = hmac.new(k, v, hashlib.sha256).digest() - return decode(hmac.new(k, v, hashlib.sha256).digest(),256) + return decode(hmac.new(k, v, hashlib.sha256).digest(), 256) + -def ecdsa_raw_sign(msghash,priv): +def ecdsa_raw_sign(msghash, priv): z = hash_to_int(msghash) - k = deterministic_generate_k(msghash,priv) + k = deterministic_generate_k(msghash, priv) - r,y = base10_multiply(G,k) - s = inv(k,N) * (z + r*decode_privkey(priv)) % N + r, y = fast_multiply(G, k) + s = inv(k, N) * (z + r*decode_privkey(priv)) % N - return 27+(y%2),r,s + return 27+(y % 2), r, s -def ecdsa_sign(msg,priv): - return encode_sig(*ecdsa_raw_sign(electrum_sig_hash(msg),priv)) -def ecdsa_raw_verify(msghash,vrs,pub): - v,r,s = vrs +def ecdsa_sign(msg, priv): + return encode_sig(*ecdsa_raw_sign(electrum_sig_hash(msg), priv)) - w = inv(s,N) + +def ecdsa_raw_verify(msghash, vrs, pub): + v, r, s = vrs + + w = inv(s, N) z = hash_to_int(msghash) - + u1, u2 = z*w % N, r*w % N - x,y = base10_add(base10_multiply(G,u1), base10_multiply(decode_pubkey(pub),u2)) + x, y = fast_add(fast_multiply(G, u1), fast_multiply(decode_pubkey(pub), u2)) return r == x -def ecdsa_verify(msg,sig,pub): - return ecdsa_raw_verify(electrum_sig_hash(msg),decode_sig(sig),pub) -def ecdsa_raw_recover(msghash,vrs): - v,r,s = vrs +def ecdsa_verify(msg, sig, pub): + return ecdsa_raw_verify(electrum_sig_hash(msg), decode_sig(sig), pub) - x = r - beta = pow(x*x*x+A*x+B,(P+1)/4,P) - y = beta if v%2 ^ beta%2 else (P - beta) - z = hash_to_int(msghash) - Qr = base10_add(neg_pubkey(base10_multiply(G,z)),base10_multiply((x,y),s)) - Q = base10_multiply(Qr,inv(r,N)) +def ecdsa_raw_recover(msghash, vrs): + v, r, s = vrs - if ecdsa_raw_verify(msghash,vrs,Q): return Q + x = r + beta = pow(x*x*x+A*x+B, (P+1)//4, P) + y = beta if v % 2 ^ beta % 2 else (P - beta) + z = hash_to_int(msghash) + Gz = jordan_multiply(((Gx, 1), (Gy, 1)), (N - z) % N) + XY = jordan_multiply(((x, 1), (y, 1)), s) + Qr = jordan_add(Gz, XY) + Q = jordan_multiply(Qr, inv(r, N)) + Q = from_jordan(Q) + + if ecdsa_raw_verify(msghash, vrs, Q): + return Q return False -def ecdsa_recover(msg,sig): - return encode_pubkey(ecdsa_raw_recover(electrum_sig_hash(msg),decode_sig(sig)),'hex') + +def ecdsa_recover(msg, sig): + return encode_pubkey(ecdsa_raw_recover(electrum_sig_hash(msg), decode_sig(sig)), 'hex') diff --git a/lib/bitcoin/py2specials.py b/lib/bitcoin/py2specials.py new file mode 100644 index 00000000..4e2e42bb --- /dev/null +++ b/lib/bitcoin/py2specials.py @@ -0,0 +1,94 @@ +import sys, re +import binascii +import os +import hashlib + + +if sys.version_info.major == 2: + string_types = (str, unicode) + string_or_bytes_types = string_types + int_types = (int, float, long) + + # Base switching + code_strings = { + 2: '01', + 10: '0123456789', + 16: '0123456789abcdef', + 32: 'abcdefghijklmnopqrstuvwxyz234567', + 58: '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz', + 256: ''.join([chr(x) for x in range(256)]) + } + + def bin_dbl_sha256(s): + bytes_to_hash = from_string_to_bytes(s) + return hashlib.sha256(hashlib.sha256(bytes_to_hash).digest()).digest() + + def lpad(msg, symbol, length): + if len(msg) >= length: + return msg + return symbol * (length - len(msg)) + msg + + def get_code_string(base): + if base in code_strings: + return code_strings[base] + else: + raise ValueError("Invalid base!") + + def changebase(string, frm, to, minlen=0): + if frm == to: + return lpad(string, get_code_string(frm)[0], minlen) + return encode(decode(string, frm), to, minlen) + + def bin_to_b58check(inp, magicbyte=0): + inp_fmtd = chr(int(magicbyte)) + inp + leadingzbytes = len(re.match('^\x00*', inp_fmtd).group(0)) + checksum = bin_dbl_sha256(inp_fmtd)[:4] + return '1' * leadingzbytes + changebase(inp_fmtd+checksum, 256, 58) + + def bytes_to_hex_string(b): + return b.encode('hex') + + def safe_from_hex(s): + return s.decode('hex') + + def from_int_representation_to_bytes(a): + return str(a) + + def from_int_to_byte(a): + return chr(a) + + def from_byte_to_int(a): + return ord(a) + + def from_bytes_to_string(s): + return s + + def from_string_to_bytes(a): + return a + + def safe_hexlify(a): + return binascii.hexlify(a) + + def encode(val, base, minlen=0): + base, minlen = int(base), int(minlen) + code_string = get_code_string(base) + result = "" + while val > 0: + result = code_string[val % base] + result + val //= base + return code_string[0] * max(minlen - len(result), 0) + result + + def decode(string, base): + base = int(base) + code_string = get_code_string(base) + result = 0 + if base == 16: + string = string.lower() + while len(string) > 0: + result *= base + result += code_string.find(string[0]) + string = string[1:] + return result + + def random_string(x): + return os.urandom(x) diff --git a/lib/bitcoin/py3specials.py b/lib/bitcoin/py3specials.py new file mode 100644 index 00000000..be234722 --- /dev/null +++ b/lib/bitcoin/py3specials.py @@ -0,0 +1,119 @@ +import sys, os +import binascii +import hashlib + + +if sys.version_info.major == 3: + string_types = (str) + string_or_bytes_types = (str, bytes) + int_types = (int, float) + # Base switching + code_strings = { + 2: '01', + 10: '0123456789', + 16: '0123456789abcdef', + 32: 'abcdefghijklmnopqrstuvwxyz234567', + 58: '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz', + 256: ''.join([chr(x) for x in range(256)]) + } + + def bin_dbl_sha256(s): + bytes_to_hash = from_string_to_bytes(s) + return hashlib.sha256(hashlib.sha256(bytes_to_hash).digest()).digest() + + def lpad(msg, symbol, length): + if len(msg) >= length: + return msg + return symbol * (length - len(msg)) + msg + + def get_code_string(base): + if base in code_strings: + return code_strings[base] + else: + raise ValueError("Invalid base!") + + def changebase(string, frm, to, minlen=0): + if frm == to: + return lpad(string, get_code_string(frm)[0], minlen) + return encode(decode(string, frm), to, minlen) + + def bin_to_b58check(inp, magicbyte=0): + inp_fmtd = from_int_to_byte(int(magicbyte))+inp + + leadingzbytes = 0 + for x in inp_fmtd: + if x != 0: + break + leadingzbytes += 1 + + checksum = bin_dbl_sha256(inp_fmtd)[:4] + return '1' * leadingzbytes + changebase(inp_fmtd+checksum, 256, 58) + + def bytes_to_hex_string(b): + if isinstance(b, str): + return b + + return ''.join('{:02x}'.format(y) for y in b) + + def safe_from_hex(s): + return bytes.fromhex(s) + + def from_int_representation_to_bytes(a): + return bytes(str(a), 'utf-8') + + def from_int_to_byte(a): + return bytes([a]) + + def from_byte_to_int(a): + return a + + def from_string_to_bytes(a): + return a if isinstance(a, bytes) else bytes(a, 'utf-8') + + def safe_hexlify(a): + return str(binascii.hexlify(a), 'utf-8') + + def encode(val, base, minlen=0): + base, minlen = int(base), int(minlen) + code_string = get_code_string(base) + result_bytes = bytes() + while val > 0: + curcode = code_string[val % base] + result_bytes = bytes([ord(curcode)]) + result_bytes + val //= base + + pad_size = minlen - len(result_bytes) + + padding_element = b'\x00' if base == 256 else b'1' \ + if base == 58 else b'0' + if (pad_size > 0): + result_bytes = padding_element*pad_size + result_bytes + + result_string = ''.join([chr(y) for y in result_bytes]) + result = result_bytes if base == 256 else result_string + + return result + + def decode(string, base): + if base == 256 and isinstance(string, str): + string = bytes(bytearray.fromhex(string)) + base = int(base) + code_string = get_code_string(base) + result = 0 + if base == 256: + def extract(d, cs): + return d + else: + def extract(d, cs): + return cs.find(d if isinstance(d, str) else chr(d)) + + if base == 16: + string = string.lower() + while len(string) > 0: + result *= base + result += extract(string[0], code_string) + string = string[1:] + return result + + def random_string(x): + return str(os.urandom(x)) diff --git a/lib/bitcoin/ripemd.py b/lib/bitcoin/ripemd.py index a9d652c5..4b0c6045 100644 --- a/lib/bitcoin/ripemd.py +++ b/lib/bitcoin/ripemd.py @@ -43,10 +43,18 @@ except ImportError: pass +import sys + +is_python2 = sys.version_info.major == 2 #block_size = 1 digest_size = 20 digestsize = 20 +try: + range = xrange +except: + pass + class RIPEMD160: """Return a new RIPEMD160 object. An optional string argument may be provided; if present, this string will be automatically @@ -77,7 +85,10 @@ def hexdigest(self): dig = self.digest() hex_digest = '' for d in dig: - hex_digest += '%02x' % ord(d) + if (is_python2): + hex_digest += '%02x' % ord(d) + else: + hex_digest += '%02x' % d return hex_digest def copy(self): @@ -155,7 +166,10 @@ def R(a, b, c, d, e, Fj, Kj, sj, rj, X): def RMD160Transform(state, block): #uint32 state[5], uchar block[64] x = [0]*16 if sys.byteorder == 'little': - x = struct.unpack('<16L', ''.join([chr(x) for x in block[0:64]])) + if is_python2: + x = struct.unpack('<16L', ''.join([chr(x) for x in block[0:64]])) + else: + x = struct.unpack('<16L', bytes(block[0:64])) else: raise "Error!!" a = state[0] @@ -362,13 +376,14 @@ def RMD160Update(ctx, inp, inplen): if type(inp) == str: inp = [ord(i)&0xff for i in inp] - have = (ctx.count / 8) % 64 + have = int((ctx.count // 8) % 64) + inplen = int(inplen) need = 64 - have ctx.count += 8 * inplen off = 0 if inplen >= need: if have: - for i in xrange(need): + for i in range(need): ctx.buffer[have+i] = inp[i] RMD160Transform(ctx.state, ctx.buffer) off = need @@ -378,12 +393,12 @@ def RMD160Update(ctx, inp, inplen): off += 64 if off < inplen: # memcpy(ctx->buffer + have, input+off, len-off); - for i in xrange(inplen - off): + for i in range(inplen - off): ctx.buffer[have+i] = inp[off+i] def RMD160Final(ctx): size = struct.pack("= 2**255: + b1 = '00' + b1 + if s >= 2**255: + b2 = '00' + b2 + left = '02'+encode(len(b1)//2, 16, 2)+b1 + right = '02'+encode(len(b2)//2, 16, 2)+b2 + return '30'+encode(len(left+right)//2, 16, 2)+left+right -def der_encode_sig(v,r,s): - b1, b2 = encode(r,256).encode('hex'), encode(s,256).encode('hex') - if r >= 2**255: b1 = '00' + b1 - if s >= 2**255: b2 = '00' + b2 - left = '02'+encode(len(b1)/2,16,2)+b1 - right = '02'+encode(len(b2)/2,16,2)+b2 - return '30'+encode(len(left+right)/2,16,2)+left+right def der_decode_sig(sig): - leftlen = decode(sig[6:8],16)*2 + leftlen = decode(sig[6:8], 16)*2 left = sig[8:8+leftlen] - rightlen = decode(sig[10+leftlen:12+leftlen],16)*2 + rightlen = decode(sig[10+leftlen:12+leftlen], 16)*2 right = sig[12+leftlen:12+leftlen+rightlen] - return (None,decode(left,16),decode(right,16)) + return (None, decode(left, 16), decode(right, 16)) + + +def txhash(tx, hashcode=None): + if isinstance(tx, str) and re.match('^[0-9a-fA-F]*$', tx): + tx = changebase(tx, 16, 256) + if hashcode: + return dbl_sha256(from_string_to_bytes(tx) + encode(int(hashcode), 256, 4)[::-1]) + else: + return safe_hexlify(bin_dbl_sha256(tx)[::-1]) -def txhash(tx,hashcode=None): - if re.match('^[0-9a-fA-F]*$',tx): - tx = changebase(tx,16,256) - if hashcode: return dbl_sha256(tx + encode(int(hashcode),256,4)[::-1]) - else: return bin_dbl_sha256(tx)[::-1].encode('hex') -def bin_txhash(tx,hashcode=None): - return txhash(tx,hashcode).decode('hex') +def bin_txhash(tx, hashcode=None): + return binascii.unhexlify(txhash(tx, hashcode)) -def ecdsa_tx_sign(tx,priv,hashcode=SIGHASH_ALL): - rawsig = ecdsa_raw_sign(bin_txhash(tx,hashcode),priv) - return der_encode_sig(*rawsig)+encode(hashcode,16,2) -def ecdsa_tx_verify(tx,sig,pub,hashcode=SIGHASH_ALL): - return ecdsa_raw_verify(bin_txhash(tx,hashcode),der_decode_sig(sig),pub) +def ecdsa_tx_sign(tx, priv, hashcode=SIGHASH_ALL): + rawsig = ecdsa_raw_sign(bin_txhash(tx, hashcode), priv) + return der_encode_sig(*rawsig)+encode(hashcode, 16, 2) -def ecdsa_tx_recover(tx,sig,hashcode=SIGHASH_ALL): - z = bin_txhash(tx,hashcode) - _,r,s = der_decode_sig(sig) - left = ecdsa_raw_recover(z,(0,r,s)) - right = ecdsa_raw_recover(z,(1,r,s)) - return (encode_pubkey(left,'hex'), encode_pubkey(right,'hex')) -### Scripts +def ecdsa_tx_verify(tx, sig, pub, hashcode=SIGHASH_ALL): + return ecdsa_raw_verify(bin_txhash(tx, hashcode), der_decode_sig(sig), pub) -def mk_pubkey_script(addr): # Keep the auxiliary functions around for altcoins' sake + +def ecdsa_tx_recover(tx, sig, hashcode=SIGHASH_ALL): + z = bin_txhash(tx, hashcode) + _, r, s = der_decode_sig(sig) + left = ecdsa_raw_recover(z, (0, r, s)) + right = ecdsa_raw_recover(z, (1, r, s)) + return (encode_pubkey(left, 'hex'), encode_pubkey(right, 'hex')) + +# Scripts + + +def mk_pubkey_script(addr): + # Keep the auxiliary functions around for altcoins' sake return '76a914' + b58check_to_hex(addr) + '88ac' + def mk_scripthash_script(addr): return 'a914' + b58check_to_hex(addr) + '87' # Address representation to output script + + def address_to_script(addr): - if addr[0] == '3' or addr[0] == '2': return mk_scripthash_script(addr) - else: return mk_pubkey_script(addr) + if addr[0] == '3' or addr[0] == '2': + return mk_scripthash_script(addr) + else: + return mk_pubkey_script(addr) # Output script to address representation -def script_to_address(script,vbyte=0): - if re.match('^[0-9a-fA-F]*$',script): - script = script.decode('hex') - if script[:3] == '\x76\xa9\x14' and script[-2:] == '\x88\xac' and len(script) == 25: - return bin_to_b58check(script[3:-2],vbyte) # pubkey hash addresses + + +def script_to_address(script, vbyte=0): + if re.match('^[0-9a-fA-F]*$', script): + script = binascii.unhexlify(script) + if script[:3] == b'\x76\xa9\x14' and script[-2:] == b'\x88\xac' and len(script) == 25: + return bin_to_b58check(script[3:-2], vbyte) # pubkey hash addresses else: - if vbyte == 111: + if vbyte in [111, 196]: # Testnet - scripthash_byte = 192 + scripthash_byte = 196 else: scripthash_byte = 5 - - return bin_to_b58check(script[2:-1],scripthash_byte) # BIP0016 scripthash addresses + # BIP0016 scripthash addresses + return bin_to_b58check(script[2:-1], scripthash_byte) def p2sh_scriptaddr(script, magicbyte=5): if re.match('^[0-9a-fA-F]*$', script): - script = script.decode('hex') + script = binascii.unhexlify(script) return hex_to_b58check(hash160(script), magicbyte) scriptaddr = p2sh_scriptaddr def deserialize_script(script): - if re.match('^[0-9a-fA-F]*$',script): - return json_changebase(deserialize_script(script.decode('hex')),lambda x:x.encode('hex')) + if isinstance(script, str) and re.match('^[0-9a-fA-F]*$', script): + return json_changebase(deserialize_script(binascii.unhexlify(script)), + lambda x: safe_hexlify(x)) out, pos = [], 0 while pos < len(script): - code = ord(script[pos]) + code = from_byte_to_int(script[pos]) if code == 0: out.append(None) pos += 1 @@ -207,8 +257,8 @@ def deserialize_script(script): out.append(script[pos+1:pos+1+code]) pos += 1 + code elif code <= 78: - szsz = pow(2,code - 76) - sz = decode(script[pos + szsz : pos : -1],256) + szsz = pow(2, code - 76) + sz = decode(script[pos+szsz: pos:-1], 256) out.append(script[pos + 1 + szsz:pos + 1 + szsz + sz]) pos += 1 + szsz + sz elif code <= 96: @@ -219,112 +269,156 @@ def deserialize_script(script): pos += 1 return out + def serialize_script_unit(unit): - if isinstance(unit,int): - if unit < 16: return chr(unit + 80) - else: return chr(unit) + if isinstance(unit, int): + if unit < 16: + return from_int_to_byte(unit + 80) + else: + return bytes([unit]) elif unit is None: - return '\x00' + return b'\x00' else: - if len(unit) <= 75: return chr(len(unit))+unit - elif len(unit) < 256: return chr(76)+chr(len(unit))+unit - elif len(unit) < 65536: return chr(77)+encode(len(unit),256,2)[::-1]+unit - else: return chr(78)+encode(len(unit),256,4)[::-1]+unit - -def serialize_script(script): - if json_is_base(script,16): - return serialize_script(json_changebase(script,lambda x:x.decode('hex'))).encode('hex') - return ''.join(map(serialize_script_unit,script)) + if len(unit) <= 75: + return from_int_to_byte(len(unit))+unit + elif len(unit) < 256: + return from_int_to_byte(76)+from_int_to_byte(len(unit))+unit + elif len(unit) < 65536: + return from_int_to_byte(77)+encode(len(unit), 256, 2)[::-1]+unit + else: + return from_int_to_byte(78)+encode(len(unit), 256, 4)[::-1]+unit + + +if is_python2: + def serialize_script(script): + if json_is_base(script, 16): + return binascii.hexlify(serialize_script(json_changebase(script, + lambda x: binascii.unhexlify(x)))) + return ''.join(map(serialize_script_unit, script)) +else: + def serialize_script(script): + if json_is_base(script, 16): + return safe_hexlify(serialize_script(json_changebase(script, + lambda x: binascii.unhexlify(x)))) + + result = bytes() + for b in map(serialize_script_unit, script): + result += b if isinstance(b, bytes) else bytes(b, 'utf-8') + return result def mk_multisig_script(*args): # [pubs],k or pub1,pub2...pub[n],k if isinstance(args[0], list): pubs, k = args[0], int(args[1]) else: - pubs = filter(lambda x: len(str(x)) >= 32, args) - k = args[pubs] - return serialize_script([k]+pubs+[len(pubs), 174]) + pubs = list(filter(lambda x: len(str(x)) >= 32, args)) + k = int(args[len(pubs)]) + return serialize_script([k]+pubs+[len(pubs)]) + 'ae' + +# Signing and verifying -### Signing and verifying -def verify_tx_input(tx,i,script,sig,pub): - if re.match('^[0-9a-fA-F]*$',tx): tx = tx.decode('hex') - if re.match('^[0-9a-fA-F]*$',script): script = script.decode('hex') - if not re.match('^[0-9a-fA-F]*$',sig): sig = sig.encode('hex') - hashcode = decode(sig[-2:],16) - modtx = signature_form(tx,int(i),script,hashcode) - return ecdsa_tx_verify(modtx,sig,pub,hashcode) +def verify_tx_input(tx, i, script, sig, pub): + if re.match('^[0-9a-fA-F]*$', tx): + tx = binascii.unhexlify(tx) + if re.match('^[0-9a-fA-F]*$', script): + script = binascii.unhexlify(script) + if not re.match('^[0-9a-fA-F]*$', sig): + sig = safe_hexlify(sig) + hashcode = decode(sig[-2:], 16) + modtx = signature_form(tx, int(i), script, hashcode) + return ecdsa_tx_verify(modtx, sig, pub, hashcode) + -def sign(tx,i,priv): +def sign(tx, i, priv, hashcode=SIGHASH_ALL): i = int(i) - if not re.match('^[0-9a-fA-F]*$',tx): - return sign(tx.encode('hex'),i,priv).decode('hex') - if len(priv) <= 33: priv = priv.encode('hex') + if (not is_python2 and isinstance(re, bytes)) or not re.match('^[0-9a-fA-F]*$', tx): + return binascii.unhexlify(sign(safe_hexlify(tx), i, priv)) + if len(priv) <= 33: + priv = safe_hexlify(priv) pub = privkey_to_pubkey(priv) address = pubkey_to_address(pub) - signing_tx = signature_form(tx,i,mk_pubkey_script(address)) - sig = ecdsa_tx_sign(signing_tx,priv) + signing_tx = signature_form(tx, i, mk_pubkey_script(address), hashcode) + sig = ecdsa_tx_sign(signing_tx, priv, hashcode) txobj = deserialize(tx) - txobj["ins"][i]["script"] = serialize_script([sig,pub]) + txobj["ins"][i]["script"] = serialize_script([sig, pub]) return serialize(txobj) -def signall(tx,priv): - for i in range(len(deserialize(tx)["ins"])): - tx = sign(tx,i,priv) + +def signall(tx, priv): + # if priv is a dictionary, assume format is + # { 'txinhash:txinidx' : privkey } + if isinstance(priv, dict): + for e, i in enumerate(deserialize(tx)["ins"]): + k = priv["%s:%d" % (i["outpoint"]["hash"], i["outpoint"]["index"])] + tx = sign(tx, e, k) + else: + for i in range(len(deserialize(tx)["ins"])): + tx = sign(tx, i, priv) return tx -def multisign(tx,i,script,pk,hashcode = SIGHASH_ALL): - if re.match('^[0-9a-fA-F]*$',tx): tx = tx.decode('hex') - if re.match('^[0-9a-fA-F]*$',script): script = script.decode('hex') - modtx = signature_form(tx,i,script,hashcode) - return ecdsa_tx_sign(modtx,pk,hashcode) -def apply_multisignatures(*args): # tx,i,script,sigs OR tx,i,script,sig1,sig2...,sig[n] +def multisign(tx, i, script, pk, hashcode=SIGHASH_ALL): + if re.match('^[0-9a-fA-F]*$', tx): + tx = binascii.unhexlify(tx) + if re.match('^[0-9a-fA-F]*$', script): + script = binascii.unhexlify(script) + modtx = signature_form(tx, i, script, hashcode) + return ecdsa_tx_sign(modtx, pk, hashcode) + + +def apply_multisignatures(*args): + # tx,i,script,sigs OR tx,i,script,sig1,sig2...,sig[n] tx, i, script = args[0], int(args[1]), args[2] - sigs = args[3] if isinstance(args[3],list) else list(args[3:]) + sigs = args[3] if isinstance(args[3], list) else list(args[3:]) - if re.match('^[0-9a-fA-F]*$',script): script = script.decode('hex') - sigs = [x.decode('hex') if x[:2] == '30' else x for x in sigs] - if re.match('^[0-9a-fA-F]*$',tx): - return apply_multisignatures(tx.decode('hex'),i,script,sigs).encode('hex') + if isinstance(script, str) and re.match('^[0-9a-fA-F]*$', script): + script = binascii.unhexlify(script) + sigs = [binascii.unhexlify(x) if x[:2] == '30' else x for x in sigs] + if isinstance(tx, str) and re.match('^[0-9a-fA-F]*$', tx): + return safe_hexlify(apply_multisignatures(binascii.unhexlify(tx), i, script, sigs)) txobj = deserialize(tx) txobj["ins"][i]["script"] = serialize_script([None]+sigs+[script]) return serialize(txobj) + def is_inp(arg): return len(arg) > 64 or "output" in arg or "outpoint" in arg -def mktx(*args): # [in0, in1...],[out0, out1...] or in0, in1 ... out0 out1 ... + +def mktx(*args): + # [in0, in1...],[out0, out1...] or in0, in1 ... out0 out1 ... ins, outs = [], [] for arg in args: - if isinstance(arg,list): + if isinstance(arg, list): for a in arg: (ins if is_inp(a) else outs).append(a) else: (ins if is_inp(arg) else outs).append(arg) - txobj = { "locktime" : 0, "version" : 1,"ins" : [], "outs" : [] } + txobj = {"locktime": 0, "version": 1, "ins": [], "outs": []} for i in ins: - if isinstance(i,dict) and "outpoint" in i: + if isinstance(i, dict) and "outpoint" in i: txobj["ins"].append(i) else: - if isinstance(i,dict) and "output" in i: i = i["output"] + if isinstance(i, dict) and "output" in i: + i = i["output"] txobj["ins"].append({ - "outpoint" : { "hash": i[:64], "index": int(i[65:]) }, + "outpoint": {"hash": i[:64], "index": int(i[65:])}, "script": "", "sequence": 4294967295 }) for o in outs: - if isinstance(o,str): + if isinstance(o, string_or_bytes_types): addr = o[:o.find(':')] val = int(o[o.find(':')+1:]) o = {} - if re.match('^[0-9a-fA-F]*$',addr): + if re.match('^[0-9a-fA-F]*$', addr): o["script"] = addr else: o["address"] = addr o["value"] = val - + outobj = {} if "address" in o: outobj["script"] = address_to_script(o["address"]) @@ -334,47 +428,55 @@ def mktx(*args): # [in0, in1...],[out0, out1...] or in0, in1 ... out0 out1 ... raise Exception("Could not find 'address' or 'script' in output.") outobj["value"] = o["value"] txobj["outs"].append(outobj) - + return serialize(txobj) -def select(unspent,value): + +def select(unspent, value): value = int(value) high = [u for u in unspent if u["value"] >= value] - high.sort(key=lambda u:u["value"]) + high.sort(key=lambda u: u["value"]) low = [u for u in unspent if u["value"] < value] - low.sort(key=lambda u:-u["value"]) - if len(high): return [high[0]] + low.sort(key=lambda u: -u["value"]) + if len(high): + return [high[0]] i, tv = 0, 0 while tv < value and i < len(low): tv += low[i]["value"] i += 1 - if tv < value: raise Exception("Not enough funds") + if tv < value: + raise Exception("Not enough funds") return low[:i] # Only takes inputs of the form { "output": blah, "value": foo } + + def mksend(*args): argz, change, fee = args[:-2], args[-2], int(args[-1]) ins, outs = [], [] for arg in argz: - if isinstance(arg,list): - for a in arg: (ins if is_inp(a) else outs).append(a) + if isinstance(arg, list): + for a in arg: + (ins if is_inp(a) else outs).append(a) else: (ins if is_inp(arg) else outs).append(arg) isum = sum([i["value"] for i in ins]) osum, outputs2 = 0, [] for o in outs: - if isinstance(o,str): o2 = { - "address": o[:o.find(':')], - "value": int(o[o.find(':')+1:]) - } - else: o2 = o + if isinstance(o, string_types): + o2 = { + "address": o[:o.find(':')], + "value": int(o[o.find(':')+1:]) + } + else: + o2 = o outputs2.append(o2) osum += o2["value"] if isum < osum+fee: raise Exception("Not enough money") elif isum > osum+fee+5430: - outputs2 += [{"address": change, "value": isum-osum-fee }] + outputs2 += [{"address": change, "value": isum-osum-fee}] - return mktx(ins,outputs2) + return mktx(ins, outputs2) diff --git a/lib/blockchaininterface.py b/lib/blockchaininterface.py index 250a7266..2fce930a 100644 --- a/lib/blockchaininterface.py +++ b/lib/blockchaininterface.py @@ -231,7 +231,7 @@ def run(self): NotifyThread(self.blockr_domain, txd, unconfirmfun, confirmfun).start() def fetchtx(self, txid): - return btc.blockr_fetchtx(txid, self.network) + return str(btc.blockr_fetchtx(txid, self.network)) def pushtx(self, txhex): data = json.loads(btc.blockr_pushtx(txhex, self.network)) From e6b76df357a69f37bc6559f25d86fe1a0d31026f Mon Sep 17 00:00:00 2001 From: Belcher Date: Mon, 6 Apr 2015 22:07:49 +0100 Subject: [PATCH 189/409] added old_mnemonic.py --- lib/old_mnemonic.py | 1691 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1691 insertions(+) create mode 100644 lib/old_mnemonic.py diff --git a/lib/old_mnemonic.py b/lib/old_mnemonic.py new file mode 100644 index 00000000..3a5449b8 --- /dev/null +++ b/lib/old_mnemonic.py @@ -0,0 +1,1691 @@ + +#!/usr/bin/env python +# +# Electrum - lightweight Bitcoin client +# Copyright (C) 2011 thomasv@gitorious +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + + +# list of words from http://en.wiktionary.org/wiki/Wiktionary:Frequency_lists/Contemporary_poetry + +words = [ +"like", +"just", +"love", +"know", +"never", +"want", +"time", +"out", +"there", +"make", +"look", +"eye", +"down", +"only", +"think", +"heart", +"back", +"then", +"into", +"about", +"more", +"away", +"still", +"them", +"take", +"thing", +"even", +"through", +"long", +"always", +"world", +"too", +"friend", +"tell", +"try", +"hand", +"thought", +"over", +"here", +"other", +"need", +"smile", +"again", +"much", +"cry", +"been", +"night", +"ever", +"little", +"said", +"end", +"some", +"those", +"around", +"mind", +"people", +"girl", +"leave", +"dream", +"left", +"turn", +"myself", +"give", +"nothing", +"really", +"off", +"before", +"something", +"find", +"walk", +"wish", +"good", +"once", +"place", +"ask", +"stop", +"keep", +"watch", +"seem", +"everything", +"wait", +"got", +"yet", +"made", +"remember", +"start", +"alone", +"run", +"hope", +"maybe", +"believe", +"body", +"hate", +"after", +"close", +"talk", +"stand", +"own", +"each", +"hurt", +"help", +"home", +"god", +"soul", +"new", +"many", +"two", +"inside", +"should", +"true", +"first", +"fear", +"mean", +"better", +"play", +"another", +"gone", +"change", +"use", +"wonder", +"someone", +"hair", +"cold", +"open", +"best", +"any", +"behind", +"happen", +"water", +"dark", +"laugh", +"stay", +"forever", +"name", +"work", +"show", +"sky", +"break", +"came", +"deep", +"door", +"put", +"black", +"together", +"upon", +"happy", +"such", +"great", +"white", +"matter", +"fill", +"past", +"please", +"burn", +"cause", +"enough", +"touch", +"moment", +"soon", +"voice", +"scream", +"anything", +"stare", +"sound", +"red", +"everyone", +"hide", +"kiss", +"truth", +"death", +"beautiful", +"mine", +"blood", +"broken", +"very", +"pass", +"next", +"forget", +"tree", +"wrong", +"air", +"mother", +"understand", +"lip", +"hit", +"wall", +"memory", +"sleep", +"free", +"high", +"realize", +"school", +"might", +"skin", +"sweet", +"perfect", +"blue", +"kill", +"breath", +"dance", +"against", +"fly", +"between", +"grow", +"strong", +"under", +"listen", +"bring", +"sometimes", +"speak", +"pull", +"person", +"become", +"family", +"begin", +"ground", +"real", +"small", +"father", +"sure", +"feet", +"rest", +"young", +"finally", +"land", +"across", +"today", +"different", +"guy", +"line", +"fire", +"reason", +"reach", +"second", +"slowly", +"write", +"eat", +"smell", +"mouth", +"step", +"learn", +"three", +"floor", +"promise", +"breathe", +"darkness", +"push", +"earth", +"guess", +"save", +"song", +"above", +"along", +"both", +"color", +"house", +"almost", +"sorry", +"anymore", +"brother", +"okay", +"dear", +"game", +"fade", +"already", +"apart", +"warm", +"beauty", +"heard", +"notice", +"question", +"shine", +"began", +"piece", +"whole", +"shadow", +"secret", +"street", +"within", +"finger", +"point", +"morning", +"whisper", +"child", +"moon", +"green", +"story", +"glass", +"kid", +"silence", +"since", +"soft", +"yourself", +"empty", +"shall", +"angel", +"answer", +"baby", +"bright", +"dad", +"path", +"worry", +"hour", +"drop", +"follow", +"power", +"war", +"half", +"flow", +"heaven", +"act", +"chance", +"fact", +"least", +"tired", +"children", +"near", +"quite", +"afraid", +"rise", +"sea", +"taste", +"window", +"cover", +"nice", +"trust", +"lot", +"sad", +"cool", +"force", +"peace", +"return", +"blind", +"easy", +"ready", +"roll", +"rose", +"drive", +"held", +"music", +"beneath", +"hang", +"mom", +"paint", +"emotion", +"quiet", +"clear", +"cloud", +"few", +"pretty", +"bird", +"outside", +"paper", +"picture", +"front", +"rock", +"simple", +"anyone", +"meant", +"reality", +"road", +"sense", +"waste", +"bit", +"leaf", +"thank", +"happiness", +"meet", +"men", +"smoke", +"truly", +"decide", +"self", +"age", +"book", +"form", +"alive", +"carry", +"escape", +"damn", +"instead", +"able", +"ice", +"minute", +"throw", +"catch", +"leg", +"ring", +"course", +"goodbye", +"lead", +"poem", +"sick", +"corner", +"desire", +"known", +"problem", +"remind", +"shoulder", +"suppose", +"toward", +"wave", +"drink", +"jump", +"woman", +"pretend", +"sister", +"week", +"human", +"joy", +"crack", +"grey", +"pray", +"surprise", +"dry", +"knee", +"less", +"search", +"bleed", +"caught", +"clean", +"embrace", +"future", +"king", +"son", +"sorrow", +"chest", +"hug", +"remain", +"sat", +"worth", +"blow", +"daddy", +"final", +"parent", +"tight", +"also", +"create", +"lonely", +"safe", +"cross", +"dress", +"evil", +"silent", +"bone", +"fate", +"perhaps", +"anger", +"class", +"scar", +"snow", +"tiny", +"tonight", +"continue", +"control", +"dog", +"edge", +"mirror", +"month", +"suddenly", +"comfort", +"given", +"loud", +"quickly", +"gaze", +"plan", +"rush", +"stone", +"town", +"battle", +"ignore", +"spirit", +"stood", +"stupid", +"yours", +"brown", +"build", +"dust", +"hey", +"kept", +"pay", +"phone", +"twist", +"although", +"ball", +"beyond", +"hidden", +"nose", +"taken", +"fail", +"float", +"pure", +"somehow", +"wash", +"wrap", +"angry", +"cheek", +"creature", +"forgotten", +"heat", +"rip", +"single", +"space", +"special", +"weak", +"whatever", +"yell", +"anyway", +"blame", +"job", +"choose", +"country", +"curse", +"drift", +"echo", +"figure", +"grew", +"laughter", +"neck", +"suffer", +"worse", +"yeah", +"disappear", +"foot", +"forward", +"knife", +"mess", +"somewhere", +"stomach", +"storm", +"beg", +"idea", +"lift", +"offer", +"breeze", +"field", +"five", +"often", +"simply", +"stuck", +"win", +"allow", +"confuse", +"enjoy", +"except", +"flower", +"seek", +"strength", +"calm", +"grin", +"gun", +"heavy", +"hill", +"large", +"ocean", +"shoe", +"sigh", +"straight", +"summer", +"tongue", +"accept", +"crazy", +"everyday", +"exist", +"grass", +"mistake", +"sent", +"shut", +"surround", +"table", +"ache", +"brain", +"destroy", +"heal", +"nature", +"shout", +"sign", +"stain", +"choice", +"doubt", +"glance", +"glow", +"mountain", +"queen", +"stranger", +"throat", +"tomorrow", +"city", +"either", +"fish", +"flame", +"rather", +"shape", +"spin", +"spread", +"ash", +"distance", +"finish", +"image", +"imagine", +"important", +"nobody", +"shatter", +"warmth", +"became", +"feed", +"flesh", +"funny", +"lust", +"shirt", +"trouble", +"yellow", +"attention", +"bare", +"bite", +"money", +"protect", +"amaze", +"appear", +"born", +"choke", +"completely", +"daughter", +"fresh", +"friendship", +"gentle", +"probably", +"six", +"deserve", +"expect", +"grab", +"middle", +"nightmare", +"river", +"thousand", +"weight", +"worst", +"wound", +"barely", +"bottle", +"cream", +"regret", +"relationship", +"stick", +"test", +"crush", +"endless", +"fault", +"itself", +"rule", +"spill", +"art", +"circle", +"join", +"kick", +"mask", +"master", +"passion", +"quick", +"raise", +"smooth", +"unless", +"wander", +"actually", +"broke", +"chair", +"deal", +"favorite", +"gift", +"note", +"number", +"sweat", +"box", +"chill", +"clothes", +"lady", +"mark", +"park", +"poor", +"sadness", +"tie", +"animal", +"belong", +"brush", +"consume", +"dawn", +"forest", +"innocent", +"pen", +"pride", +"stream", +"thick", +"clay", +"complete", +"count", +"draw", +"faith", +"press", +"silver", +"struggle", +"surface", +"taught", +"teach", +"wet", +"bless", +"chase", +"climb", +"enter", +"letter", +"melt", +"metal", +"movie", +"stretch", +"swing", +"vision", +"wife", +"beside", +"crash", +"forgot", +"guide", +"haunt", +"joke", +"knock", +"plant", +"pour", +"prove", +"reveal", +"steal", +"stuff", +"trip", +"wood", +"wrist", +"bother", +"bottom", +"crawl", +"crowd", +"fix", +"forgive", +"frown", +"grace", +"loose", +"lucky", +"party", +"release", +"surely", +"survive", +"teacher", +"gently", +"grip", +"speed", +"suicide", +"travel", +"treat", +"vein", +"written", +"cage", +"chain", +"conversation", +"date", +"enemy", +"however", +"interest", +"million", +"page", +"pink", +"proud", +"sway", +"themselves", +"winter", +"church", +"cruel", +"cup", +"demon", +"experience", +"freedom", +"pair", +"pop", +"purpose", +"respect", +"shoot", +"softly", +"state", +"strange", +"bar", +"birth", +"curl", +"dirt", +"excuse", +"lord", +"lovely", +"monster", +"order", +"pack", +"pants", +"pool", +"scene", +"seven", +"shame", +"slide", +"ugly", +"among", +"blade", +"blonde", +"closet", +"creek", +"deny", +"drug", +"eternity", +"gain", +"grade", +"handle", +"key", +"linger", +"pale", +"prepare", +"swallow", +"swim", +"tremble", +"wheel", +"won", +"cast", +"cigarette", +"claim", +"college", +"direction", +"dirty", +"gather", +"ghost", +"hundred", +"loss", +"lung", +"orange", +"present", +"swear", +"swirl", +"twice", +"wild", +"bitter", +"blanket", +"doctor", +"everywhere", +"flash", +"grown", +"knowledge", +"numb", +"pressure", +"radio", +"repeat", +"ruin", +"spend", +"unknown", +"buy", +"clock", +"devil", +"early", +"false", +"fantasy", +"pound", +"precious", +"refuse", +"sheet", +"teeth", +"welcome", +"add", +"ahead", +"block", +"bury", +"caress", +"content", +"depth", +"despite", +"distant", +"marry", +"purple", +"threw", +"whenever", +"bomb", +"dull", +"easily", +"grasp", +"hospital", +"innocence", +"normal", +"receive", +"reply", +"rhyme", +"shade", +"someday", +"sword", +"toe", +"visit", +"asleep", +"bought", +"center", +"consider", +"flat", +"hero", +"history", +"ink", +"insane", +"muscle", +"mystery", +"pocket", +"reflection", +"shove", +"silently", +"smart", +"soldier", +"spot", +"stress", +"train", +"type", +"view", +"whether", +"bus", +"energy", +"explain", +"holy", +"hunger", +"inch", +"magic", +"mix", +"noise", +"nowhere", +"prayer", +"presence", +"shock", +"snap", +"spider", +"study", +"thunder", +"trail", +"admit", +"agree", +"bag", +"bang", +"bound", +"butterfly", +"cute", +"exactly", +"explode", +"familiar", +"fold", +"further", +"pierce", +"reflect", +"scent", +"selfish", +"sharp", +"sink", +"spring", +"stumble", +"universe", +"weep", +"women", +"wonderful", +"action", +"ancient", +"attempt", +"avoid", +"birthday", +"branch", +"chocolate", +"core", +"depress", +"drunk", +"especially", +"focus", +"fruit", +"honest", +"match", +"palm", +"perfectly", +"pillow", +"pity", +"poison", +"roar", +"shift", +"slightly", +"thump", +"truck", +"tune", +"twenty", +"unable", +"wipe", +"wrote", +"coat", +"constant", +"dinner", +"drove", +"egg", +"eternal", +"flight", +"flood", +"frame", +"freak", +"gasp", +"glad", +"hollow", +"motion", +"peer", +"plastic", +"root", +"screen", +"season", +"sting", +"strike", +"team", +"unlike", +"victim", +"volume", +"warn", +"weird", +"attack", +"await", +"awake", +"built", +"charm", +"crave", +"despair", +"fought", +"grant", +"grief", +"horse", +"limit", +"message", +"ripple", +"sanity", +"scatter", +"serve", +"split", +"string", +"trick", +"annoy", +"blur", +"boat", +"brave", +"clearly", +"cling", +"connect", +"fist", +"forth", +"imagination", +"iron", +"jock", +"judge", +"lesson", +"milk", +"misery", +"nail", +"naked", +"ourselves", +"poet", +"possible", +"princess", +"sail", +"size", +"snake", +"society", +"stroke", +"torture", +"toss", +"trace", +"wise", +"bloom", +"bullet", +"cell", +"check", +"cost", +"darling", +"during", +"footstep", +"fragile", +"hallway", +"hardly", +"horizon", +"invisible", +"journey", +"midnight", +"mud", +"nod", +"pause", +"relax", +"shiver", +"sudden", +"value", +"youth", +"abuse", +"admire", +"blink", +"breast", +"bruise", +"constantly", +"couple", +"creep", +"curve", +"difference", +"dumb", +"emptiness", +"gotta", +"honor", +"plain", +"planet", +"recall", +"rub", +"ship", +"slam", +"soar", +"somebody", +"tightly", +"weather", +"adore", +"approach", +"bond", +"bread", +"burst", +"candle", +"coffee", +"cousin", +"crime", +"desert", +"flutter", +"frozen", +"grand", +"heel", +"hello", +"language", +"level", +"movement", +"pleasure", +"powerful", +"random", +"rhythm", +"settle", +"silly", +"slap", +"sort", +"spoken", +"steel", +"threaten", +"tumble", +"upset", +"aside", +"awkward", +"bee", +"blank", +"board", +"button", +"card", +"carefully", +"complain", +"crap", +"deeply", +"discover", +"drag", +"dread", +"effort", +"entire", +"fairy", +"giant", +"gotten", +"greet", +"illusion", +"jeans", +"leap", +"liquid", +"march", +"mend", +"nervous", +"nine", +"replace", +"rope", +"spine", +"stole", +"terror", +"accident", +"apple", +"balance", +"boom", +"childhood", +"collect", +"demand", +"depression", +"eventually", +"faint", +"glare", +"goal", +"group", +"honey", +"kitchen", +"laid", +"limb", +"machine", +"mere", +"mold", +"murder", +"nerve", +"painful", +"poetry", +"prince", +"rabbit", +"shelter", +"shore", +"shower", +"soothe", +"stair", +"steady", +"sunlight", +"tangle", +"tease", +"treasure", +"uncle", +"begun", +"bliss", +"canvas", +"cheer", +"claw", +"clutch", +"commit", +"crimson", +"crystal", +"delight", +"doll", +"existence", +"express", +"fog", +"football", +"gay", +"goose", +"guard", +"hatred", +"illuminate", +"mass", +"math", +"mourn", +"rich", +"rough", +"skip", +"stir", +"student", +"style", +"support", +"thorn", +"tough", +"yard", +"yearn", +"yesterday", +"advice", +"appreciate", +"autumn", +"bank", +"beam", +"bowl", +"capture", +"carve", +"collapse", +"confusion", +"creation", +"dove", +"feather", +"girlfriend", +"glory", +"government", +"harsh", +"hop", +"inner", +"loser", +"moonlight", +"neighbor", +"neither", +"peach", +"pig", +"praise", +"screw", +"shield", +"shimmer", +"sneak", +"stab", +"subject", +"throughout", +"thrown", +"tower", +"twirl", +"wow", +"army", +"arrive", +"bathroom", +"bump", +"cease", +"cookie", +"couch", +"courage", +"dim", +"guilt", +"howl", +"hum", +"husband", +"insult", +"led", +"lunch", +"mock", +"mostly", +"natural", +"nearly", +"needle", +"nerd", +"peaceful", +"perfection", +"pile", +"price", +"remove", +"roam", +"sanctuary", +"serious", +"shiny", +"shook", +"sob", +"stolen", +"tap", +"vain", +"void", +"warrior", +"wrinkle", +"affection", +"apologize", +"blossom", +"bounce", +"bridge", +"cheap", +"crumble", +"decision", +"descend", +"desperately", +"dig", +"dot", +"flip", +"frighten", +"heartbeat", +"huge", +"lazy", +"lick", +"odd", +"opinion", +"process", +"puzzle", +"quietly", +"retreat", +"score", +"sentence", +"separate", +"situation", +"skill", +"soak", +"square", +"stray", +"taint", +"task", +"tide", +"underneath", +"veil", +"whistle", +"anywhere", +"bedroom", +"bid", +"bloody", +"burden", +"careful", +"compare", +"concern", +"curtain", +"decay", +"defeat", +"describe", +"double", +"dreamer", +"driver", +"dwell", +"evening", +"flare", +"flicker", +"grandma", +"guitar", +"harm", +"horrible", +"hungry", +"indeed", +"lace", +"melody", +"monkey", +"nation", +"object", +"obviously", +"rainbow", +"salt", +"scratch", +"shown", +"shy", +"stage", +"stun", +"third", +"tickle", +"useless", +"weakness", +"worship", +"worthless", +"afternoon", +"beard", +"boyfriend", +"bubble", +"busy", +"certain", +"chin", +"concrete", +"desk", +"diamond", +"doom", +"drawn", +"due", +"felicity", +"freeze", +"frost", +"garden", +"glide", +"harmony", +"hopefully", +"hunt", +"jealous", +"lightning", +"mama", +"mercy", +"peel", +"physical", +"position", +"pulse", +"punch", +"quit", +"rant", +"respond", +"salty", +"sane", +"satisfy", +"savior", +"sheep", +"slept", +"social", +"sport", +"tuck", +"utter", +"valley", +"wolf", +"aim", +"alas", +"alter", +"arrow", +"awaken", +"beaten", +"belief", +"brand", +"ceiling", +"cheese", +"clue", +"confidence", +"connection", +"daily", +"disguise", +"eager", +"erase", +"essence", +"everytime", +"expression", +"fan", +"flag", +"flirt", +"foul", +"fur", +"giggle", +"glorious", +"ignorance", +"law", +"lifeless", +"measure", +"mighty", +"muse", +"north", +"opposite", +"paradise", +"patience", +"patient", +"pencil", +"petal", +"plate", +"ponder", +"possibly", +"practice", +"slice", +"spell", +"stock", +"strife", +"strip", +"suffocate", +"suit", +"tender", +"tool", +"trade", +"velvet", +"verse", +"waist", +"witch", +"aunt", +"bench", +"bold", +"cap", +"certainly", +"click", +"companion", +"creator", +"dart", +"delicate", +"determine", +"dish", +"dragon", +"drama", +"drum", +"dude", +"everybody", +"feast", +"forehead", +"former", +"fright", +"fully", +"gas", +"hook", +"hurl", +"invite", +"juice", +"manage", +"moral", +"possess", +"raw", +"rebel", +"royal", +"scale", +"scary", +"several", +"slight", +"stubborn", +"swell", +"talent", +"tea", +"terrible", +"thread", +"torment", +"trickle", +"usually", +"vast", +"violence", +"weave", +"acid", +"agony", +"ashamed", +"awe", +"belly", +"blend", +"blush", +"character", +"cheat", +"common", +"company", +"coward", +"creak", +"danger", +"deadly", +"defense", +"define", +"depend", +"desperate", +"destination", +"dew", +"duck", +"dusty", +"embarrass", +"engine", +"example", +"explore", +"foe", +"freely", +"frustrate", +"generation", +"glove", +"guilty", +"health", +"hurry", +"idiot", +"impossible", +"inhale", +"jaw", +"kingdom", +"mention", +"mist", +"moan", +"mumble", +"mutter", +"observe", +"ode", +"pathetic", +"pattern", +"pie", +"prefer", +"puff", +"rape", +"rare", +"revenge", +"rude", +"scrape", +"spiral", +"squeeze", +"strain", +"sunset", +"suspend", +"sympathy", +"thigh", +"throne", +"total", +"unseen", +"weapon", +"weary" +] + + + +n = 1626 + +# Note about US patent no 5892470: Here each word does not represent a given digit. +# Instead, the digit represented by a word is variable, it depends on the previous word. + +def mn_encode( message ): + assert len(message) % 8 == 0 + out = [] + for i in range(len(message)/8): + word = message[8*i:8*i+8] + x = int(word, 16) + w1 = (x%n) + w2 = ((x/n) + w1)%n + w3 = ((x/n/n) + w2)%n + out += [ words[w1], words[w2], words[w3] ] + return out + +def mn_decode( wlist ): + out = '' + for i in range(len(wlist)/3): + word1, word2, word3 = wlist[3*i:3*i+3] + w1 = words.index(word1) + w2 = (words.index(word2))%n + w3 = (words.index(word3))%n + x = w1 +n*((w2-w1)%n) +n*n*((w3-w2)%n) + out += '%08x'%x + return out + + +if __name__ == '__main__': + import sys + if len( sys.argv ) == 1: + print 'I need arguments: a hex string to encode, or a list of words to decode' + elif len( sys.argv ) == 2: + print ' '.join(mn_encode(sys.argv[1])) + else: + print mn_decode(sys.argv[1:]) From 78daab005a4ab0bbcb2b84247a437aae63858cb2 Mon Sep 17 00:00:00 2001 From: Belcher Date: Mon, 6 Apr 2015 23:01:41 +0100 Subject: [PATCH 190/409] created validate_address() and used it to check user-inputted addresses, also removed get_signed_tx() --- lib/common.py | 17 +++++++++-------- patientsendpayment.py | 4 ++++ sendpayment.py | 4 ++++ tumbler.py | 5 +++++ wallet-tool.py | 2 +- 5 files changed, 23 insertions(+), 9 deletions(-) diff --git a/lib/common.py b/lib/common.py index c1728b81..a59bf4a9 100644 --- a/lib/common.py +++ b/lib/common.py @@ -60,20 +60,21 @@ def get_network(): else: raise Exception("Only testnet is currently implemented") -#TODO change this name into get_addr_ver() or something def get_addr_vbyte(): if get_network() == 'testnet': return 0x6f else: return 0x00 -def get_signed_tx(wallet, ins, outs): - tx = btc.mktx(ins, outs) - for index, utxo in enumerate(ins): - addr = wallet.unspent[utxo['output']]['address'] - tx = btc.sign(tx, index, wallet.get_key_from_addr(addr)) - return tx - +def validate_address(addr): + try: + ver = btc.get_version_byte(addr) + except AssertionError: + return False, 'Checksum wrong. Typo in address?' + if ver != get_addr_vbyte(): + return False, 'Wrong address version. Testnet/mainnet confused?' + return True, 'address validated' + def debug_dump_object(obj, skip_fields=[]): debug('Class debug dump, name:' + obj.__class__.__name__) for k, v in obj.__dict__.iteritems(): diff --git a/patientsendpayment.py b/patientsendpayment.py index 0d726c90..0d64939d 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -135,6 +135,10 @@ def main(): destaddr = args[2] load_program_config() + addr_valid, errormsg = validate_address(destaddr) + if not addr_valid: + print 'ERROR: Address invalid. ' + errormsg + return waittime = timedelta(hours=options.waittime).total_seconds() print 'Running patient sender of a payment' diff --git a/sendpayment.py b/sendpayment.py index f4952e36..b6d83897 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -89,6 +89,10 @@ def main(): destaddr = args[2] load_program_config() + addr_valid, errormsg = validate_address(destaddr) + if not addr_valid: + print 'ERROR: Address invalid. ' + errormsg + return import binascii, os common.nickname = 'payer-' +binascii.hexlify(os.urandom(4)) diff --git a/tumbler.py b/tumbler.py index e9f1a02c..4f093053 100644 --- a/tumbler.py +++ b/tumbler.py @@ -230,6 +230,11 @@ def main(): destaddrs = args[1:] common.load_program_config() + for addr in destaddrs: + addr_valid, errormsg = validate_address(addr) + if not addr_valid: + print 'ERROR: Address ' + addr + ' invalid. ' + errormsg + return if len(destaddrs) + options.addrask <= 1: print '='*50 diff --git a/wallet-tool.py b/wallet-tool.py index 90d72797..fff08547 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -6,7 +6,7 @@ sys.path.insert(0, os.path.join(data_dir, 'lib')) import bitcoin as btc -from common import Wallet, get_signed_tx, load_program_config, get_addr_vbyte +from common import Wallet, load_program_config, get_addr_vbyte import common import old_mnemonic From b95fcf3a65b0ac7771fb19c0c629e1dc6ca5a946 Mon Sep 17 00:00:00 2001 From: Belcher Date: Mon, 6 Apr 2015 23:18:11 +0100 Subject: [PATCH 191/409] slight edit to usage text --- patientsendpayment.py | 2 +- sendpayment.py | 2 +- tumbler.py | 2 +- yield-generator.py | 1 - 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/patientsendpayment.py b/patientsendpayment.py index 0d64939d..c794e29a 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -110,7 +110,7 @@ def on_tx_confirmed(self, cjorder, confirmations, txid, balance): def main(): - parser = OptionParser(usage='usage: %prog [options] [seed] [amount] [destaddr]', + parser = OptionParser(usage='usage: %prog [options] [wallet file / seed] [amount] [destaddr]', description='Sends a payment from your wallet to an given address' + ' using coinjoin. First acts as a maker, announcing an order and ' + 'waiting for someone to fill it. After a set period of time, gives' + diff --git a/sendpayment.py b/sendpayment.py index b6d83897..85a92bac 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -67,7 +67,7 @@ def on_welcome(self): PaymentThread(self).start() def main(): - parser = OptionParser(usage='usage: %prog [options] [seed] [amount] [destaddr]', + parser = OptionParser(usage='usage: %prog [options] [wallet file / seed] [amount] [destaddr]', description='Sends a single payment from the zero mixing depth of your ' + 'wallet to an given address using coinjoin and then switches off. ' + 'Setting amount to zero will do a sweep, where the entire mix depth is emptied') diff --git a/tumbler.py b/tumbler.py index 4f093053..a43ebff2 100644 --- a/tumbler.py +++ b/tumbler.py @@ -186,7 +186,7 @@ def on_welcome(self): TumblerThread(self).start() def main(): - parser = OptionParser(usage='usage: %prog [options] [seed] [destaddr...]', + parser = OptionParser(usage='usage: %prog [options] [wallet file / seed] [destaddr...]', description='Sends bitcoins to many different addresses using coinjoin in' ' an attempt to break the link between them. Sending to multiple ' ' addresses is highly recommended for privacy. This tumbler can' diff --git a/yield-generator.py b/yield-generator.py index 75858d7e..c697408d 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -99,7 +99,6 @@ def main(): irc.run() except: debug('CRASHING, DUMPING EVERYTHING') - debug('wallet seed = ' + seed) debug_dump_object(wallet, ['addr_cache', 'keys', 'seed']) debug_dump_object(maker) debug_dump_object(irc) From 9185e6120c4b4fa7b591b9f93d3bdc85178d3248 Mon Sep 17 00:00:00 2001 From: Belcher Date: Tue, 7 Apr 2015 01:38:53 +0100 Subject: [PATCH 192/409] fixed some bug with encoding --- lib/blockchaininterface.py | 29 ++++++++++++++++------------- lib/common.py | 5 ++++- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/lib/blockchaininterface.py b/lib/blockchaininterface.py index 2fce930a..f6890792 100644 --- a/lib/blockchaininterface.py +++ b/lib/blockchaininterface.py @@ -181,16 +181,18 @@ def run(self): common.debug('sharedtxid = ' + str(shared_txid)) if len(shared_txid) == 0: continue + time.sleep(2) blockr_url = 'http://' + self.blockr_domain + '.blockr.io/api/v1/tx/raw/' data = json.loads(btc.make_request(blockr_url + ','.join(shared_txid)))['data'] if not isinstance(data, list): data = [data] for txinfo in data: - outs = set([(sv['script'], sv['value']) for sv in btc.deserialize(txinfo['tx']['hex'])['outs']]) + txhex = str(txinfo['tx']['hex']) + outs = set([(sv['script'], sv['value']) for sv in btc.deserialize(txhex)['outs']]) common.debug('unconfirm query outs = ' + str(outs)) if outs == self.tx_output_set: unconfirmed_txid = txinfo['tx']['txid'] - unconfirmed_txhex = txinfo['tx']['hex'] + unconfirmed_txhex = str(txinfo['tx']['hex']) break self.unconfirmfun(btc.deserialize(unconfirmed_txhex), unconfirmed_txid) @@ -220,11 +222,12 @@ def run(self): if not isinstance(data, list): data = [data] for txinfo in data: - outs = set([(sv['script'], sv['value']) for sv in btc.deserialize(txinfo['tx']['hex'])['outs']]) + txhex = str(txinfo['tx']['hex']) + outs = set([(sv['script'], sv['value']) for sv in btc.deserialize(txhex)['outs']]) common.debug('confirm query outs = ' + str(outs)) if outs == self.tx_output_set: confirmed_txid = txinfo['tx']['txid'] - confirmed_txhex = txinfo['tx']['hex'] + confirmed_txhex = str(txinfo['tx']['hex']) break self.confirmfun(btc.deserialize(confirmed_txhex), confirmed_txid, 1) @@ -275,8 +278,7 @@ def __init__(self, request, client_address, base_server): def do_HEAD(self): pages = ('/walletnotify?', '/alertnotify?') - if not self.path.startswith(pages): - return + if self.path.startswith('/walletnotify?'): txid = self.path[len(pages[0]):] txd = btc.deserialize(self.btcinterface.fetchtx(txid)) @@ -288,19 +290,20 @@ def do_HEAD(self): unconfirmfun = ucfun confirmfun = cfun break - if not unconfirmfun: + if unconfirmfun == None: common.debug('txid=' + txid + ' not being listened for') - return - txdata = json.loads(self.btcinterface.rpc(['gettxout', txid, '0', 'true'])) - if txdata['confirmations'] == 0: - unconfirmfun(txd, txid) else: - confirmfun(txd, txid, txdata['confirmations']) - self.btcinterface.txnotify_fun.remove((tx_out, unconfirmfun, confirmfun)) + txdata = json.loads(self.btcinterface.rpc(['gettxout', txid, '0', 'true'])) + if txdata['confirmations'] == 0: + unconfirmfun(txd, txid) + else: + confirmfun(txd, txid, txdata['confirmations']) + self.btcinterface.txnotify_fun.remove((tx_out, unconfirmfun, confirmfun)) elif self.path.startswith('/alertnotify?'): common.alert_message = self.path[len(pages[1]):] common.debug('Got an alert!\nMessage=' + common.alert_message) + self.send_response(200) #self.send_header('Connection', 'close') self.end_headers() diff --git a/lib/common.py b/lib/common.py index a59bf4a9..61afd8e8 100644 --- a/lib/common.py +++ b/lib/common.py @@ -267,10 +267,13 @@ def weighted_order_choose(orders, n, feekey): return orders[chosen_order_index] def choose_order(db, cj_amount, n): - sqlorders = db.execute('SELECT * FROM orderbook;').fetchall() orders = [(o['counterparty'], o['oid'], calc_cj_fee(o['ordertype'], o['cjfee'], cj_amount)) for o in sqlorders if cj_amount >= o['minsize'] and cj_amount <= o['maxsize']] + counterparties = set([o[0] for o in orders]) + if n > len(counterparties): + debug('ERROR not enough liquidity in the orderbook') + return None, 0 #TODO handle not enough liquidity better, maybe an Exception orders = sorted(orders, key=lambda k: k[2]) debug('considered orders = ' + str(orders)) total_cj_fee = 0 From f8b70a6ec10a208089d74f262f626c0f0c6d7210 Mon Sep 17 00:00:00 2001 From: Belcher Date: Tue, 7 Apr 2015 22:49:47 +0100 Subject: [PATCH 193/409] added a debug line --- lib/common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/common.py b/lib/common.py index 61afd8e8..e1bb1860 100644 --- a/lib/common.py +++ b/lib/common.py @@ -263,6 +263,7 @@ def weighted_order_choose(orders, n, feekey): fee = np.array([feekey(o) for o in orders]) weight = np.exp(-(1.0*fee - minfee) / phi) weight /= sum(weight) + debug('randomly choosing orders with weighting\n' + pprint.pformat(zip(orders, weight))) chosen_order_index = np.random.choice(len(orders), p=weight) return orders[chosen_order_index] From 2d0a22935967e9c83ada63105ba60e918f67cac0 Mon Sep 17 00:00:00 2001 From: Belcher Date: Wed, 8 Apr 2015 21:49:47 +0100 Subject: [PATCH 194/409] fixed some more bugs, kinda --- lib/blockchaininterface.py | 6 ++++-- lib/maker.py | 15 ++++++++------- lib/taker.py | 6 +++--- tumbler.py | 22 +++++++++++++++++----- 4 files changed, 32 insertions(+), 17 deletions(-) diff --git a/lib/blockchaininterface.py b/lib/blockchaininterface.py index f6890792..3df66dab 100644 --- a/lib/blockchaininterface.py +++ b/lib/blockchaininterface.py @@ -181,7 +181,7 @@ def run(self): common.debug('sharedtxid = ' + str(shared_txid)) if len(shared_txid) == 0: continue - time.sleep(2) + time.sleep(2) #here for some race condition bullshit with blockr.io blockr_url = 'http://' + self.blockr_domain + '.blockr.io/api/v1/tx/raw/' data = json.loads(btc.make_request(blockr_url + ','.join(shared_txid)))['data'] if not isinstance(data, list): @@ -296,9 +296,11 @@ def do_HEAD(self): txdata = json.loads(self.btcinterface.rpc(['gettxout', txid, '0', 'true'])) if txdata['confirmations'] == 0: unconfirmfun(txd, txid) + common.debug('ran unconfirmfun') else: confirmfun(txd, txid, txdata['confirmations']) self.btcinterface.txnotify_fun.remove((tx_out, unconfirmfun, confirmfun)) + common.debug('ran confirmfun') elif self.path.startswith('/alertnotify?'): common.alert_message = self.path[len(pages[1]):] @@ -443,7 +445,7 @@ def is_output_suitable(self, txout): for txo in txout: ret = self.rpc(['gettxout', txo[:64], txo[65:], 'false']) if ret == '': - return False, 'tx ' + txo + ' not found' + return False, 'tx ' + txo + ' not found. Unconfirmed or already spent.' return True, 'success' #class for regtest chain access diff --git a/lib/maker.py b/lib/maker.py index f3983f25..3040d381 100644 --- a/lib/maker.py +++ b/lib/maker.py @@ -22,10 +22,10 @@ def __init__(self, maker, nick, oid, amount, taker_pk): order_s = [o for o in maker.orderlist if o['oid'] == oid] if len(order_s) == 0: - self.maker.send_error(nick, 'oid not found') + self.maker.msgchan.send_error(nick, 'oid not found') order = order_s[0] if amount < order['minsize'] or amount > order['maxsize']: - self.maker.send_error(nick, 'amount out of range') + self.maker.msgchan.send_error(nick, 'amount out of range') self.utxos, self.cj_addr, self.change_addr = maker.oid_to_order(oid, amount) #check nothing has messed up with the wallet code, remove this code after a while import pprint @@ -66,12 +66,12 @@ def recv_tx(self, nick, txhex): try: self.tx = btc.deserialize(txhex) except IndexError as e: - self.maker.send_error(nick, 'malformed txhex. ' + repr(e)) + self.maker.msgchan.send_error(nick, 'malformed txhex. ' + repr(e)) debug('obtained tx\n' + pprint.pformat(self.tx)) goodtx, errmsg = self.verify_unsigned_tx(self.tx) if not goodtx: debug('not a good tx, reason=' + errmsg) - self.maker.send_error(nick, errmsg) + self.maker.msgchan.send_error(nick, errmsg) #TODO: the above 3 errors should be encrypted, but it's a bit messy. debug('goodtx') sigs = [] @@ -145,7 +145,8 @@ def verify_unsigned_tx(self, txd): if outs['value'] != expected_change_value: return False, 'wrong change, i expect ' + str(expected_change_value) if times_seen_cj_addr != 1 or times_seen_change_addr != 1: - return False, 'cj or change addr not in tx outputs exactly once' + return False, ('cj or change addr not in tx outputs once, #cjaddr=' + + str(times_seen_cj_addr) + ', #chaddr=' + str(times_seen_change_addr)) return True, None class CJMakerOrderError(StandardError): @@ -184,14 +185,14 @@ def on_order_fill(self, nick, oid, amount, taker_pubkey): def on_seen_auth(self, nick, pubkey, sig): if nick not in self.active_orders or self.active_orders[nick] == None: - self.send_error(nick, 'No open order from this nick') + self.msgchan.send_error(nick, 'No open order from this nick') self.active_orders[nick].auth_counterparty(nick, pubkey, sig) #TODO if auth_counterparty returns false, remove this order from active_orders # and send an error def on_seen_tx(self, nick, txhex): if nick not in self.active_orders or self.active_orders[nick] == None: - self.send_error(nick, 'No open order from this nick') + self.msgchan.send_error(nick, 'No open order from this nick') self.wallet_unspent_lock.acquire() try: self.active_orders[nick].recv_tx(nick, txhex) diff --git a/lib/taker.py b/lib/taker.py index 40c8429d..d72ce15b 100644 --- a/lib/taker.py +++ b/lib/taker.py @@ -62,14 +62,14 @@ def recv_txio(self, nick, utxo_list, cj_pub, change_addr): debug('nick(' + nick + ') not in nonrespondants ' + str(self.nonrespondants)) return self.utxos[nick] = utxo_list - self.nonrespondants.remove(nick) order = self.db.execute('SELECT ordertype, txfee, cjfee FROM ' 'orderbook WHERE oid=? AND counterparty=?', (self.active_orders[nick], nick)).fetchone() goodoutputs, errmsg = common.bc_interface.is_output_suitable(utxo_list) if not goodoutputs: - common.debug('bad outputs: ' + errmsg) - return + common.debug('ERROR bad outputs: ' + errmsg) + raise RuntimeError('killing taker, TODO handle this error') + self.nonrespondants.remove(nick) total_input = calc_total_input_value(self.utxos[nick]) real_cjfee = calc_cj_fee(order['ordertype'], order['cjfee'], self.cj_amount) self.outputs.append({'address': change_addr, 'value': diff --git a/tumbler.py b/tumbler.py index a43ebff2..19ab1306 100644 --- a/tumbler.py +++ b/tumbler.py @@ -94,7 +94,12 @@ def send_tx(self, tx, balance, sweep, i, l): if tx['dest'] == 'internal': destaddr = self.taker.wallet.get_receive_addr(tx['srcmixdepth'] + 1) elif tx['dest'] == 'addrask': - destaddr = raw_input('insert new address: ') + while True: + destaddr = raw_input('insert new address: ') + addr_valid, errormsg = validate_address(destaddr) + if addr_valid: + break + print 'Address ' + addr + ' invalid. ' + errormsg + ' try again' else: destaddr = tx['dest'] @@ -107,6 +112,9 @@ def send_tx(self, tx, balance, sweep, i, l): None, self.taker.txfee, self.finishcallback) else: amount = int(tx['amount_ratio'] * balance) + if amount < self.taker.mincjamount: + print 'cj amount too low, bringing up' + amount = self.taker.mincjamount changeaddr = self.taker.wallet.get_change_addr(tx['srcmixdepth']) print 'coinjoining ' + str(amount) while True: @@ -174,12 +182,13 @@ def run(self): class Tumbler(takermodule.Taker): - def __init__(self, msgchan, wallet, tx_list, txfee, maxcjfee): + def __init__(self, msgchan, wallet, tx_list, txfee, maxcjfee, mincjamount): takermodule.Taker.__init__(self, msgchan) self.wallet = wallet self.tx_list = tx_list self.maxcjfee = maxcjfee self.txfee = txfee + self.mincjamount = mincjamount def on_welcome(self): takermodule.Taker.on_welcome(self) @@ -218,10 +227,12 @@ def main(): help='Average the number of minutes to wait between transactions. Randomly chosen ' ' following an exponential distribution, which describes the time between uncorrelated' ' events. default=5') - parser.add_option('-w', '--wait-time', action='store', type='float', dest='waittime', help='wait time in seconds to allow orders to arrive, default=5', default=5) + parser.add_option('-s', '--mincjamount', type='float', dest='mincjamount', default=0.0001, + help='minimum coinjoin amount in transaction') (options, args) = parser.parse_args() + #TODO somehow implement a lower limit if len(args) < 2: parser.error('Needs a seed and destination addresses') @@ -246,6 +257,8 @@ def main(): print 'destaddrs=' + str(destaddrs) print str(options) tx_list = generate_tumbler_tx(destaddrs, options) + if not tx_list: + return tx_list2 = copy.deepcopy(tx_list) tx_dict = {} @@ -288,13 +301,12 @@ def main(): common.nickname = 'tumbler-'+binascii.hexlify(os.urandom(4)) irc = IRCMessageChannel(common.nickname) - tumbler = Tumbler(irc, wallet, tx_list, options.txfee, options.maxcjfee) + tumbler = Tumbler(irc, wallet, tx_list, options.txfee, options.maxcjfee, options.mincjamount) try: debug('connecting to irc') irc.run() except: debug('CRASHING, DUMPING EVERYTHING') - debug('wallet seed = ' + seed) debug_dump_object(wallet, ['addr_cache', 'keys', 'seed']) debug_dump_object(tumbler) import traceback From 11ec2492cbb94833eb983b01f85be2fbe53b37f0 Mon Sep 17 00:00:00 2001 From: Belcher Date: Thu, 9 Apr 2015 01:30:24 +0100 Subject: [PATCH 195/409] choose_order() checks for liquidity, tumbler options.mixdepthsrc now works, tumbler progress printing moved --- lib/blockchaininterface.py | 1 + lib/common.py | 3 ++ tumbler.py | 66 +++++++++++++++++++++++++------------- 3 files changed, 48 insertions(+), 22 deletions(-) diff --git a/lib/blockchaininterface.py b/lib/blockchaininterface.py index 3df66dab..12ba7dc7 100644 --- a/lib/blockchaininterface.py +++ b/lib/blockchaininterface.py @@ -106,6 +106,7 @@ def sync_addresses(self, wallet, gaplimit=6): wallet.index[mix_depth][forchange] = wallet.addr_cache[last_used_addr][2] + 1 def sync_unspent(self, wallet): + time.sleep(10) #finds utxos in the wallet st = time.time() rate_limit_time = 10*60 #dont refresh unspent dict more often than 10 minutes diff --git a/lib/common.py b/lib/common.py index e1bb1860..5ede60b3 100644 --- a/lib/common.py +++ b/lib/common.py @@ -366,6 +366,9 @@ def is_amount_in_range(ordercombo, cjamount): debug('considered order combinations') debug(pprint.pformat(dbgprint)) + if len(ordercombos) == 0: + debug('ERROR not enough liquidity in the orderbook') + return None, 0 #TODO handle not enough liquidity better, maybe an Exception ordercombo = weighted_order_choose(ordercombos, n, lambda k: k[1][1]) orders = dict([(o['counterparty'], o['oid']) for o in ordercombo[0]]) cjamount = ordercombo[1][0] diff --git a/tumbler.py b/tumbler.py index 19ab1306..b3f37656 100644 --- a/tumbler.py +++ b/tumbler.py @@ -47,24 +47,27 @@ def generate_tumbler_tx(destaddrs, options): total_dest_addr = len(destaddrs) + options.addrask external_dest_addrs = destaddrs + ['addrask']*options.addrask - if total_dest_addr >= options.mixdepthcount: + if total_dest_addr > options.mixdepthcount: print 'not enough mixing depths to pay to all destination addresses' return None - for i, srcmix in enumerate(range(options.mixdepthcount - total_dest_addr, options.mixdepthcount)): + for mix_offset in range(total_dest_addr): + srcmix = options.mixdepthsrc + options.mixdepthcount - mix_offset - 1 for tx in reversed(tx_list): if tx['srcmixdepth'] == srcmix: - tx['dest'] = external_dest_addrs[i] + tx['dest'] = external_dest_addrs[mix_offset] break - if total_dest_addr - i != 1: - continue - tx_list_remove = [] - for tx in tx_list: - if tx['srcmixdepth'] == srcmix: - if tx['dest'] == 'internal': - tx_list_remove.append(tx) - else: - tx['amount_ratio'] = 1.0 - [tx_list.remove(t) for t in tx_list_remove] + if mix_offset == 0: + #setting last mixdepth to send all to dest + tx_list_remove = [] + for tx in tx_list: + if tx['srcmixdepth'] == srcmix: + if tx['dest'] == 'internal': + print 'removing tx = ' + str(tx) + tx_list_remove.append(tx) + else: + print 'setting amount to 1' + tx['amount_ratio'] = 1.0 + [tx_list.remove(t) for t in tx_list_remove] return tx_list #thread which does the buy-side algorithm @@ -76,7 +79,7 @@ def __init__(self, taker): self.taker = taker def unconfirm_callback(self, txd, txid): - pass + print 'that was %d tx out of %d' % (self.current_tx+1, len(self.taker.tx_list)) def confirm_callback(self, txd, txid, confirmations): self.taker.wallet.add_new_utxos(txd, txid) @@ -89,7 +92,7 @@ def finishcallback(self, coinjointx): self.unconfirm_callback, self.confirm_callback) self.taker.wallet.remove_old_utxos(coinjointx.latest_tx) - def send_tx(self, tx, balance, sweep, i, l): + def send_tx(self, tx, balance, sweep): destaddr = None if tx['dest'] == 'internal': destaddr = self.taker.wallet.get_receive_addr(tx['srcmixdepth'] + 1) @@ -108,6 +111,20 @@ def send_tx(self, tx, balance, sweep, i, l): all_utxos = self.taker.wallet.get_utxos_by_mixdepth()[tx['srcmixdepth']] total_value = sum([addrval['value'] for addrval in all_utxos.values()]) orders, cjamount = choose_sweep_order(self.taker.db, total_value, self.taker.txfee, tx['makercount']) + while True: + #orders, total_cj_fee = choose_order(self.taker.db, amount, tx['makercount']) + orders, cjamount = choose_sweep_order(self.taker.db, total_value, self.taker.txfee, tx['makercount']) + cj_fee = 1.0*(cjamount - total_value) / tx['makercount'] / cjamount + print 'average fee = ' + str(cj_fee) + if cj_fee > self.taker.maxcjfee: + print 'cj fee too high at ' + str(cj_fee) + ', waiting 10 seconds' + time.sleep(10) + continue + if orders == None: + print 'waiting for liquidity' + time.sleep(10) + continue + break self.taker.start_cj(self.taker.wallet, cjamount, orders, all_utxos, destaddr, None, self.taker.txfee, self.finishcallback) else: @@ -120,10 +137,16 @@ def send_tx(self, tx, balance, sweep, i, l): while True: orders, total_cj_fee = choose_order(self.taker.db, amount, tx['makercount']) cj_fee = 1.0*total_cj_fee / tx['makercount'] / amount - if cj_fee < self.taker.maxcjfee: - break - print 'cj fee too high at ' + str(cj_fee) + ', waiting 10 seconds' - time.sleep(10) + print 'average fee = ' + str(cj_fee) + if cj_fee > self.taker.maxcjfee: + print 'cj fee too high at ' + str(cj_fee) + ', waiting 10 seconds' + time.sleep(10) + continue + if orders == None: + print 'waiting for liquidity' + time.sleep(10) + continue + break print 'chosen orders to fill ' + str(orders) + ' totalcjfee=' + str(total_cj_fee) total_amount = amount + total_cj_fee + self.taker.txfee print 'total amount spent = ' + str(total_amount) @@ -132,7 +155,6 @@ def send_tx(self, tx, balance, sweep, i, l): self.taker.start_cj(self.taker.wallet, amount, orders, utxos, destaddr, changeaddr, self.taker.txfee, self.finishcallback) - print 'that was %d tx out of %d' % (i, l) self.lockcond.acquire() self.lockcond.wait() self.lockcond.release() @@ -165,8 +187,8 @@ def run(self): for later_tx in self.taker.tx_list[i + 1:]: if later_tx['srcmixdepth'] == tx['srcmixdepth']: sweep = False - self.send_tx(tx, self.balance_by_mixdepth[tx['srcmixdepth']], sweep, - i, len(self.taker.tx_list)) + self.current_tx = i + self.send_tx(tx, self.balance_by_mixdepth[tx['srcmixdepth']], sweep) print 'total finished' self.taker.msgchan.shutdown() From fa1c53fcb851fb38cb2d9e3378f4d75208869c26 Mon Sep 17 00:00:00 2001 From: Belcher Date: Thu, 9 Apr 2015 23:50:31 +0100 Subject: [PATCH 196/409] fixed spent utxo bug by maintaining a list of spent utxos, also fixed other bugs, seems to all work now --- lib/blockchaininterface.py | 4 +++- lib/common.py | 5 ++++- tumbler.py | 8 ++++---- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/lib/blockchaininterface.py b/lib/blockchaininterface.py index 12ba7dc7..1121ca1e 100644 --- a/lib/blockchaininterface.py +++ b/lib/blockchaininterface.py @@ -106,7 +106,6 @@ def sync_addresses(self, wallet, gaplimit=6): wallet.index[mix_depth][forchange] = wallet.addr_cache[last_used_addr][2] + 1 def sync_unspent(self, wallet): - time.sleep(10) #finds utxos in the wallet st = time.time() rate_limit_time = 10*60 #dont refresh unspent dict more often than 10 minutes @@ -139,6 +138,9 @@ def sync_unspent(self, wallet): for u in dat['unspent']: wallet.unspent[u['tx']+':'+str(u['n'])] = {'address': dat['address'], 'value': int(u['amount'].replace('.', ''))} + for u in wallet.spent_utxos: + wallet.unspent.pop(u, None) + self.last_sync_unspent = time.time() common.debug('blockr sync_unspent took ' + str((self.last_sync_unspent - st)) + 'sec') diff --git a/lib/common.py b/lib/common.py index 5ede60b3..7aa5d8e3 100644 --- a/lib/common.py +++ b/lib/common.py @@ -115,6 +115,7 @@ def __init__(self, seedarg, max_mix_depth=2): self.addr_cache = {} self.unspent = {} + self.spent_utxos = [] def get_seed(self, seedarg): path = os.path.join('wallets', seedarg) @@ -166,6 +167,7 @@ def remove_old_utxos(self, tx): removed_utxos[utxo] = self.unspent[utxo] del self.unspent[utxo] debug('removed utxos, wallet now is \n' + pprint.pformat(self.get_utxos_by_mixdepth())) + self.spent_utxos += removed_utxos.keys() return removed_utxos def add_new_utxos(self, tx, txid): @@ -273,7 +275,8 @@ def choose_order(db, cj_amount, n): for o in sqlorders if cj_amount >= o['minsize'] and cj_amount <= o['maxsize']] counterparties = set([o[0] for o in orders]) if n > len(counterparties): - debug('ERROR not enough liquidity in the orderbook') + debug('ERROR not enough liquidity in the orderbook n=%d counterparties=%d' + % (n, len(counterparties))) return None, 0 #TODO handle not enough liquidity better, maybe an Exception orders = sorted(orders, key=lambda k: k[2]) debug('considered orders = ' + str(orders)) diff --git a/tumbler.py b/tumbler.py index b3f37656..dd8b5b7d 100644 --- a/tumbler.py +++ b/tumbler.py @@ -114,16 +114,16 @@ def send_tx(self, tx, balance, sweep): while True: #orders, total_cj_fee = choose_order(self.taker.db, amount, tx['makercount']) orders, cjamount = choose_sweep_order(self.taker.db, total_value, self.taker.txfee, tx['makercount']) + if orders == None: + print 'waiting for liquidity' + time.sleep(10) + continue cj_fee = 1.0*(cjamount - total_value) / tx['makercount'] / cjamount print 'average fee = ' + str(cj_fee) if cj_fee > self.taker.maxcjfee: print 'cj fee too high at ' + str(cj_fee) + ', waiting 10 seconds' time.sleep(10) continue - if orders == None: - print 'waiting for liquidity' - time.sleep(10) - continue break self.taker.start_cj(self.taker.wallet, cjamount, orders, all_utxos, destaddr, None, self.taker.txfee, self.finishcallback) From d5b9c4f917f83a8b850bc599558403a72f5f95bb Mon Sep 17 00:00:00 2001 From: Belcher Date: Tue, 14 Apr 2015 21:04:59 +0100 Subject: [PATCH 197/409] coded up query_utxo_set() which will replace fetchtx() and others --- lib/blockchaininterface.py | 48 ++++++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/lib/blockchaininterface.py b/lib/blockchaininterface.py index 1121ca1e..e96dfe88 100644 --- a/lib/blockchaininterface.py +++ b/lib/blockchaininterface.py @@ -22,11 +22,6 @@ def get_blockchain_interface_instance(config): raise ValueError("Invalid blockchain source") return bc_interface - -#download_wallet_history() find_unspent_addresses() #finding where to put index and my utxos -#add address notify() -#fetchtx() needs to accept a list of addresses too -#pushtx() class BlockchainInterface(object): __metaclass__ = abc.ABCMeta def __init__(self): @@ -70,6 +65,14 @@ def is_output_suitable(self, txout): ''' pass + @abc.abstractmethod + def query_utxo_set(self, txouts): + ''' + takes a utxo or a list of utxos + returns None if they are spend or unconfirmed + otherwise returns value in satoshis, address and output script + ''' + class BlockrInterface(BlockchainInterface): def __init__(self, testnet = False): super(BlockrInterface, self).__init__() @@ -272,6 +275,26 @@ def is_output_suitable(self, txout): return False, 'some utxos already spent' return True, 'success' + def query_utxo_set(self, txout): + if not isinstance(txout, list): + txout = [txout] + txids = [h[:64] for h in txout] + txids_dupremoved = list(set(txids)) + blockr_url = 'http://' + self.blockr_domain + '.blockr.io/api/v1/tx/info/' + data = json.loads(btc.make_request(blockr_url + ','.join(txids_dupremoved)))['data'] + if not isinstance(data, list): + data = [data] + result = [] + for txo in txout: + txdata = [d for d in data if d['tx'] == txo[:64]][0] + vout = [v for v in txdata['vouts'] if v['n'] == int(txo[65:])][0] + if vout['is_spent'] == 1: + result.append(None) + else: + result.append({'value': int(Decimal(vout['amount'])*Decimal('1e8')), + 'address': vout['address'], 'script': vout['extras']['script']}) + return result if len(result) > 1 else result[0] + class NotifyRequestHeader(SimpleHTTPServer.SimpleHTTPRequestHandler): def __init__(self, request, client_address, base_server): @@ -451,6 +474,21 @@ def is_output_suitable(self, txout): return False, 'tx ' + txo + ' not found. Unconfirmed or already spent.' return True, 'success' + def query_utxo_set(self, txout): + if not isinstance(txout, list): + txout = [txout] + result = [] + for txo in txout: + ret = self.rpc(['gettxout', txo[:64], txo[65:], 'false']) + if ret == '': + result.append(None) + else: + data = json.loads(ret) + result.append({'value': int(Decimal(str(data['value']))*Decimal('1e8')), + 'address': data['scriptPubKey']['addresses'][0], 'script': data['scriptPubKey']['hex']}) + return result if len(result) > 1 else result[0] + + #class for regtest chain access #running on local daemon. Only #to be instantiated after network is up From 03f967efd7ee2e12a3d2955ea307efb85f5ea06b Mon Sep 17 00:00:00 2001 From: Belcher Date: Wed, 15 Apr 2015 00:01:13 +0100 Subject: [PATCH 198/409] removed fetchtx() and is_output_suitable() in favour of query_utxo_set() which does not require -txindex=1 --- lib/blockchaininterface.py | 62 +++----------------------------------- lib/common.py | 15 --------- lib/maker.py | 25 +++++++++------ lib/taker.py | 24 ++++++++------- 4 files changed, 32 insertions(+), 94 deletions(-) diff --git a/lib/blockchaininterface.py b/lib/blockchaininterface.py index e96dfe88..f5707c0e 100644 --- a/lib/blockchaininterface.py +++ b/lib/blockchaininterface.py @@ -46,25 +46,11 @@ def add_tx_notify(self, txd, unconfirmfun, confirmfun): '''Invokes unconfirmfun and confirmfun when tx is seen on the network''' pass - @abc.abstractmethod - def fetchtx(self, txid): - '''Returns a txhash of a given txid, or list of txids''' - pass - @abc.abstractmethod def pushtx(self, txhex): '''pushes tx to the network, returns txhash''' pass - @abc.abstractmethod - def is_output_suitable(self, txout): - ''' - checks whether the txid:vout output is suitable to be used, - must be already mined into a block, and unspent - accepts list of txid:vout too - ''' - pass - @abc.abstractmethod def query_utxo_set(self, txouts): ''' @@ -72,6 +58,7 @@ def query_utxo_set(self, txouts): returns None if they are spend or unconfirmed otherwise returns value in satoshis, address and output script ''' + #address and output script contain the same information btw class BlockrInterface(BlockchainInterface): def __init__(self, testnet = False): @@ -239,9 +226,6 @@ def run(self): NotifyThread(self.blockr_domain, txd, unconfirmfun, confirmfun).start() - def fetchtx(self, txid): - return str(btc.blockr_fetchtx(txid, self.network)) - def pushtx(self, txhex): data = json.loads(btc.blockr_pushtx(txhex, self.network)) if data['status'] != 'success': @@ -250,31 +234,6 @@ def pushtx(self, txhex): return None return data['data'] - def is_output_suitable(self, txout): - if not isinstance(txout, list): - txout = [txout] - txids = [h[:64] for h in txout] - blockr_url = 'http://' + self.blockr_domain + '.blockr.io/api/v1/tx/info/' - data = json.loads(btc.make_request(blockr_url + ','.join(txids)))['data'] - if not isinstance(data, list): - data = [data] - addrs = [] - for tx in data: - if tx['is_unconfirmed']: - return False, 'tx ' + tx['tx'] + ' unconfirmed' - for outs in tx['vouts']: - addrs.append(outs['address']) - common.debug('addrs ' + pprint.pformat(addrs)) - utxos = btc.blockr_unspent(addrs, self.network) - utxos = [u['output'] for u in utxos] - common.debug('unspents = ' + pprint.pformat(utxos)) - common.debug('txouts = ' + pprint.pformat(txout)) - txoutset = set(txout) - utxoset = set(utxos) - if not utxoset.issuperset(txoutset): - return False, 'some utxos already spent' - return True, 'success' - def query_utxo_set(self, txout): if not isinstance(txout, list): txout = [txout] @@ -293,7 +252,7 @@ def query_utxo_set(self, txout): else: result.append({'value': int(Decimal(vout['amount'])*Decimal('1e8')), 'address': vout['address'], 'script': vout['extras']['script']}) - return result if len(result) > 1 else result[0] + return result class NotifyRequestHeader(SimpleHTTPServer.SimpleHTTPRequestHandler): @@ -307,7 +266,7 @@ def do_HEAD(self): if self.path.startswith('/walletnotify?'): txid = self.path[len(pages[0]):] - txd = btc.deserialize(self.btcinterface.fetchtx(txid)) + txd = btc.deserialize(self.rpc(['getrawtransaction', txid]).strip()) tx_output_set = set([(sv['script'], sv['value']) for sv in txd['outs']]) unconfirmfun, confirmfun = None, None @@ -458,22 +417,9 @@ def add_tx_notify(self, txd, unconfirmfun, confirmfun): tx_output_set = set([(sv['script'], sv['value']) for sv in txd['outs']]) self.txnotify_fun.append((tx_output_set, unconfirmfun, confirmfun)) - def fetchtx(self, txid): - return self.rpc(['getrawtransaction', txid]).strip() - def pushtx(self, txhex): return self.rpc(['sendrawtransaction', txhex]).strip() - def is_output_suitable(self, txout): - if not isinstance(txout, list): - txout = [txout] - - for txo in txout: - ret = self.rpc(['gettxout', txo[:64], txo[65:], 'false']) - if ret == '': - return False, 'tx ' + txo + ' not found. Unconfirmed or already spent.' - return True, 'success' - def query_utxo_set(self, txout): if not isinstance(txout, list): txout = [txout] @@ -486,7 +432,7 @@ def query_utxo_set(self, txout): data = json.loads(ret) result.append({'value': int(Decimal(str(data['value']))*Decimal('1e8')), 'address': data['scriptPubKey']['addresses'][0], 'script': data['scriptPubKey']['hex']}) - return result if len(result) > 1 else result[0] + return result #class for regtest chain access diff --git a/lib/common.py b/lib/common.py index 7aa5d8e3..8ea0e7da 100644 --- a/lib/common.py +++ b/lib/common.py @@ -89,12 +89,6 @@ def debug_dump_object(obj, skip_fields=[]): else: debug(str(v)) -def get_addr_from_utxo(txhash, index): - '''return the bitcoin address of the outpoint at - the specified index for the transaction with specified hash. - Return None if no such index existed for that transaction.''' - return btc.script_to_address(btc.deserialize(bc_interface.fetchtx(txhash))['outs'][index]['script'], get_addr_vbyte()) - class Wallet(object): def __init__(self, seedarg, max_mix_depth=2): self.max_mix_depth = max_mix_depth @@ -233,15 +227,6 @@ def calc_cj_fee(ordertype, cjfee, cj_amount): raise RuntimeError('unknown order type: ' + str(ordertype)) return real_cjfee -#TODO this function is used once, it has no point existing -def calc_total_input_value(utxos): - input_sum = 0 - for utxo in utxos: - #tx = btc.blockr_fetchtx(utxo[:64], get_network()) - tx = bc_interface.fetchtx(utxo[:64]) - input_sum += int(btc.deserialize(tx)['outs'][int(utxo[65:])]['value']) - return input_sum - def weighted_order_choose(orders, n, feekey): ''' Algorithm for choosing the weighting function diff --git a/lib/maker.py b/lib/maker.py index 3040d381..618f053e 100644 --- a/lib/maker.py +++ b/lib/maker.py @@ -29,12 +29,16 @@ def __init__(self, maker, nick, oid, amount, taker_pk): self.utxos, self.cj_addr, self.change_addr = maker.oid_to_order(oid, amount) #check nothing has messed up with the wallet code, remove this code after a while import pprint - pprint.pprint(self.utxos) - for utxo, va in self.utxos.iteritems(): - txd = btc.deserialize(common.bc_interface.fetchtx(utxo[:64])) - value = txd['outs'][int(utxo[65:])]['value'] - if value != va['value']: - debug('wrongly labeled utxo, expected value ' + str(va['value']) + ' got ' + str(value)) + debug('maker utxos = ' + pprint.pformat(self.utxos)) + utxo_list = self.utxos.keys() + utxo_data = common.bc_interface.query_utxo_set(utxo_list) + if None in utxo_data: + debug('wrongly using an already spent utxo. utxo_data = ' + pprint.pformat(utxo_data)) + sys.exit(0) + for utxo, data in zip(utxo_list, utxo_data): + if self.utxos[utxo]['value'] != data['value']: + debug('wrongly labeled utxo, expected value ' + + str(self.utxos[utxo]['value']) + ' got ' + str(data['value'])) sys.exit(0) self.ordertype = order['ordertype'] @@ -114,10 +118,11 @@ def verify_unsigned_tx(self, txd): tx_utxo_set = set([ins['outpoint']['hash'] + ':' \ + str(ins['outpoint']['index']) for ins in txd['ins']]) #complete authentication: check the tx input uses the authing pubkey - if not btc.pubtoaddr(self.i_utxo_pubkey, get_addr_vbyte()) \ - in [get_addr_from_utxo(i['outpoint']['hash'], i['outpoint']['index']) \ - for i in txd['ins']]: - return False, "authenticating bitcoin address is not contained" + input_utxo_data = common.bc_interface.query_utxo_set(list(tx_utxo_set)) + input_addresses = [u['address'] for u in input_utxo_data] + if btc.pubtoaddr(self.i_utxo_pubkey, get_addr_vbyte())\ + not in input_addresses: + return False, "authenticating bitcoin address is not contained" my_utxo_set = set(self.utxos.keys()) wallet_utxos = set(self.maker.wallet.unspent) if not tx_utxo_set.issuperset(my_utxo_set): diff --git a/lib/taker.py b/lib/taker.py index d72ce15b..e5ee7582 100644 --- a/lib/taker.py +++ b/lib/taker.py @@ -5,7 +5,7 @@ import enc_wrapper import bitcoin as btc -import sqlite3, base64, threading, time, random +import sqlite3, base64, threading, time, random, pprint class CoinJoinTX(object): #soon the taker argument will be removed and just be replaced by wallet or some other interface @@ -64,13 +64,13 @@ def recv_txio(self, nick, utxo_list, cj_pub, change_addr): self.utxos[nick] = utxo_list order = self.db.execute('SELECT ordertype, txfee, cjfee FROM ' 'orderbook WHERE oid=? AND counterparty=?', - (self.active_orders[nick], nick)).fetchone() - goodoutputs, errmsg = common.bc_interface.is_output_suitable(utxo_list) - if not goodoutputs: - common.debug('ERROR bad outputs: ' + errmsg) + (self.active_orders[nick], nick)).fetchone() + utxo_data = common.bc_interface.query_utxo_set(self.utxos[nick]) + if None in utxo_data: + common.debug('ERROR outputs unconfirmed or already spent. utxo_data=' + + pprint.pformat(utxo_data)) raise RuntimeError('killing taker, TODO handle this error') - self.nonrespondants.remove(nick) - total_input = calc_total_input_value(self.utxos[nick]) + total_input = sum([d['value'] for d in utxo_data]) real_cjfee = calc_cj_fee(order['ordertype'], order['cjfee'], self.cj_amount) self.outputs.append({'address': change_addr, 'value': total_input - self.cj_amount - order['txfee'] + real_cjfee}) @@ -79,6 +79,7 @@ def recv_txio(self, nick, utxo_list, cj_pub, change_addr): cj_addr = btc.pubtoaddr(cj_pub, get_addr_vbyte()) self.outputs.append({'address': cj_addr, 'value': self.cj_amount}) self.cjfee_total += real_cjfee + self.nonrespondants.remove(nick) if len(self.nonrespondants) > 0: debug('nonrespondants = ' + str(self.nonrespondants)) return @@ -102,7 +103,6 @@ def recv_txio(self, nick, utxo_list, cj_pub, change_addr): utxo_tx = [dict([('output', u)]) for u in sum(self.utxos.values(), [])] random.shuffle(self.outputs) tx = btc.mktx(utxo_tx, self.outputs) - import pprint debug('obtained tx\n' + pprint.pformat(btc.deserialize(tx))) self.msgchan.send_tx(self.active_orders.keys(), tx) @@ -122,9 +122,11 @@ def add_signature(self, sigb64): for index, ins in enumerate(self.latest_tx['ins']): if ins['script'] != '': continue - ftx = common.bc_interface.fetchtx(ins['outpoint']['hash']) - src_val = btc.deserialize(ftx)['outs'][ ins['outpoint']['index'] ] - sig_good = btc.verify_tx_input(tx, index, src_val['script'], *btc.deserialize_script(sig)) + utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) + utxo_data = common.bc_interface.query_utxo_set(utxo) + if utxo_data[0] == None: + continue + sig_good = btc.verify_tx_input(tx, index, utxo_data[0]['script'], *btc.deserialize_script(sig)) if sig_good: debug('found good sig at index=%d' % (index)) ins['script'] = sig From 1612544b699967a04a32a17bf2d64ef22e35b811 Mon Sep 17 00:00:00 2001 From: Belcher Date: Thu, 16 Apr 2015 01:50:57 +0100 Subject: [PATCH 199/409] several joinmarket bots can be run on the same machine with a single bitcoin core instance, untested --- lib/blockchaininterface.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/lib/blockchaininterface.py b/lib/blockchaininterface.py index f5707c0e..7af06968 100644 --- a/lib/blockchaininterface.py +++ b/lib/blockchaininterface.py @@ -1,7 +1,7 @@ #from joinmarket import * import subprocess import unittest -import json, threading, abc, pprint, time, random, sys +import json, threading, abc, pprint, time, random, sys, os import BaseHTTPServer, SimpleHTTPServer from decimal import Decimal import bitcoin as btc @@ -291,6 +291,8 @@ def do_HEAD(self): common.alert_message = self.path[len(pages[1]):] common.debug('Got an alert!\nMessage=' + common.alert_message) + os.system('wget -q --spider --timeout=0.5 --tries=1 http://localhost:' + + str(self.base_server.server_address[1] + 1) + self.path) self.send_response(200) #self.send_header('Connection', 'close') self.end_headers() @@ -302,14 +304,19 @@ def __init__(self, btcinterface): self.btcinterface = btcinterface def run(self): - common.debug('started bitcoin core notify listening thread') - hostport = ('localhost', 62602) - httpd = BaseHTTPServer.HTTPServer(hostport, NotifyRequestHeader) - httpd.btcinterface = self.btcinterface - httpd.serve_forever() - -#must run bitcoind with -txindex=1 -server -#-walletnotify="wget --spider -q http://localhost:62602/walletnotify?%s" + for inc in range(10): + hostport = ('localhost', 62602 + inc) + try: + httpd = BaseHTTPServer.HTTPServer(hostport, NotifyRequestHeader) + except Exception: + continue + httpd.btcinterface = self.btcinterface + common.debug('started bitcoin core notify listening thread, port=' + str(hostport[1])) + httpd.serve_forever() + common.debug('failed to bind for bitcoin core notify listening') + +#must run bitcoind with -server +#-walletnotify="wget -q --spider --timeout=0.5 --tries=1 http://localhost:62602/walletnotify?%s" #and make sure wget is installed #TODO must add the tx addresses as watchonly if case we ever broadcast a tx From aa25d8ec8290f783b5355dcf0f6278d33ef1b303 Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 17 Apr 2015 21:07:25 +0100 Subject: [PATCH 200/409] checks for import error with slowaes and outputs a message to the user to install it --- lib/common.py | 7 ++++++- wallet-tool.py | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/common.py b/lib/common.py index 8ea0e7da..c0451e36 100644 --- a/lib/common.py +++ b/lib/common.py @@ -2,7 +2,7 @@ import bitcoin as btc from decimal import Decimal from math import factorial -import sys, datetime, json, time, pprint, threading, aes, getpass +import sys, datetime, json, time, pprint, threading, getpass import numpy as np import blockchaininterface from ConfigParser import SafeConfigParser @@ -117,6 +117,11 @@ def get_seed(self, seedarg): debug('seedarg interpreted as seed') return seedarg debug('seedarg interpreted as wallet file name') + try: + import aes + except ImportError: + print 'You must install slowaes\nTry running: sudo pip install slowaes' + sys.exit(0) fd = open(path, 'r') walletfile = fd.read() fd.close() diff --git a/wallet-tool.py b/wallet-tool.py index fff08547..1e0a7962 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -1,6 +1,6 @@ import sys, os -import getpass, aes, json, datetime +import getpass, json, datetime from optparse import OptionParser data_dir = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(data_dir, 'lib')) @@ -90,6 +90,11 @@ total_balance += balance_depth print 'total balance = %.8fbtc' % (total_balance/1e8) elif method == 'generate' or method == 'recover': + try: + import aes + except ImportError: + print 'You must install slowaes\nTry running: sudo pip install slowaes' + sys.exit(0) if method == 'generate': seed = btc.sha256(os.urandom(64))[:32] words = old_mnemonic.mn_encode(seed) From 671b34e1448ef39a8e39b02c64adabe0ee79747e Mon Sep 17 00:00:00 2001 From: Belcher Date: Sat, 18 Apr 2015 01:30:59 +0100 Subject: [PATCH 201/409] fixed stupid bug --- lib/blockchaininterface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/blockchaininterface.py b/lib/blockchaininterface.py index 7af06968..87b6f0b7 100644 --- a/lib/blockchaininterface.py +++ b/lib/blockchaininterface.py @@ -266,7 +266,7 @@ def do_HEAD(self): if self.path.startswith('/walletnotify?'): txid = self.path[len(pages[0]):] - txd = btc.deserialize(self.rpc(['getrawtransaction', txid]).strip()) + txd = btc.deserialize(self.btcinterface.rpc(['getrawtransaction', txid]).strip()) tx_output_set = set([(sv['script'], sv['value']) for sv in txd['outs']]) unconfirmfun, confirmfun = None, None From 66745d0162e23b361e4546fbb1c838597f6b1b2e Mon Sep 17 00:00:00 2001 From: Belcher Date: Sat, 18 Apr 2015 01:44:45 +0100 Subject: [PATCH 202/409] irc responds to /ctcp version now --- lib/irc.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/irc.py b/lib/irc.py index ded1345d..878e15f2 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -271,8 +271,7 @@ def __handle_privmsg(self, source, target, message): if endindex == -1: return ctcp = message[1:endindex + 1] - #self.send_raw('PRIVMSG ' + nick + ' :\x01VERSION - #TODO ctcp version here, since some servers dont let you get on without + self.send_raw('PRIVMSG ' + nick + ' :\x01VERSION xchat 2.8.8 Ubuntu\x01') if target == self.nick: if nick not in self.built_privmsg: if message[0] != COMMAND_PREFIX: From 6f282908fbf232157366f042e004f57d98c7269f Mon Sep 17 00:00:00 2001 From: Belcher Date: Sat, 18 Apr 2015 03:04:56 +0100 Subject: [PATCH 203/409] implemented SASL authentisation for IRC, allowing tor connections --- lib/irc.py | 24 +++++++++++++++++++++++- yield-generator.py | 9 +++------ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/lib/irc.py b/lib/irc.py index 878e15f2..426cc1a1 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -315,6 +315,23 @@ def __handle_line(self, line): return chunks = line.split(' ') + if self.password: + if chunks[1] == 'CAP': + if chunks[3] != 'ACK': + debug('server does not support SASL, quitting') + self.shutdown() + self.send_raw('AUTHENTICATE PLAIN') + elif chunks[0] == 'AUTHENTICATE': + self.send_raw('AUTHENTICATE ' + base64.b64encode(self.nick + '\x00' + self.nick + '\x00' + self.password)) + elif chunks[1] == '903': + debug('Successfully authenticated') + self.password = None + self.send_raw('CAP END') + elif chunks[1] == '904': + debug('Failed authentication, wrong password') + self.shutdown() + return + if chunks[1] == 'PRIVMSG': self.__handle_privmsg(chunks[0], chunks[2], get_irc_text(line)) if chunks[1] == 'PONG': @@ -368,13 +385,16 @@ def __handle_line(self, line): self.motd_fd.close() ''' - def __init__(self, nick, username='username', realname='realname'): + def __init__(self, nick, username='username', realname='realname', password=None): MessageChannel.__init__(self) self.cjpeer = None #subclasses have to set this to self self.nick = nick self.serverport = (config.get("MESSAGING","host"), int(config.get("MESSAGING","port"))) self.channel = '#'+ config.get("MESSAGING","channel") self.userrealname = (username, realname) + if password and len(password) == 0: + password = None + self.password = password def run(self): self.connect_attempts = 0 @@ -391,6 +411,8 @@ def run(self): self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.sock.connect(self.serverport) self.fd = self.sock.makefile() + if self.password: + self.send_raw('CAP REQ :sasl') self.send_raw('USER %s b c :%s' % self.userrealname) self.send_raw('NICK ' + self.nick) while 1: diff --git a/yield-generator.py b/yield-generator.py index c697408d..4ff931c5 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -34,11 +34,7 @@ class YieldGenerator(Maker): def __init__(self, msgchan, wallet): Maker.__init__(self, msgchan, wallet) self.msgchan.register_channel_callbacks(self.on_welcome, self.on_set_topic, - self.on_connect, None, self.on_nick_leave, None) - - def on_connect(self): - if len(nickserv_password) > 0: - self.msgchan.send_raw('PRIVMSG NickServ :identify ' + nickserv_password) + None, None, self.on_nick_leave, None) def create_my_orders(self): mix_balance = self.wallet.get_balance_by_mixdepth() @@ -92,7 +88,8 @@ def main(): wallet.print_debug_wallet_info() common.nickname = nickname - irc = IRCMessageChannel(common.nickname, realname='btcint=' + common.config.get("BLOCKCHAIN", "blockchain_source")) + irc = IRCMessageChannel(common.nickname, realname='btcint=' + common.config.get("BLOCKCHAIN", "blockchain_source"), + password=nickserv_password) maker = YieldGenerator(irc, wallet) try: debug('connecting to irc') From 2b4eb6c43d05259825ec3284af663a3aa1132124 Mon Sep 17 00:00:00 2001 From: esh Date: Sat, 18 Apr 2015 15:10:51 +0300 Subject: [PATCH 204/409] added ssl --- joinmarket.cfg | 4 ++-- lib/irc.py | 17 +++++++++++------ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/joinmarket.cfg b/joinmarket.cfg index c0faa4d2..0853bb8b 100644 --- a/joinmarket.cfg +++ b/joinmarket.cfg @@ -5,7 +5,7 @@ network = testnet bitcoin_cli_cmd = bitcoin-cli [MESSAGING] -host = irc.freenode.net +host = chat.freenode.net channel = joinmarket-pit-test -port = 6667 +port = 6697 #More stuff to come diff --git a/lib/irc.py b/lib/irc.py index 426cc1a1..7da341c3 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -3,7 +3,7 @@ from message_channel import MessageChannel from message_channel import CJPeerError -import socket, threading, time +import socket, threading, time, ssl import base64, os import enc_wrapper @@ -144,7 +144,8 @@ def __privmsg(self, nick, cmd, message): def send_raw(self, line): #if not line.startswith('PING LAG'): # debug('sendraw ' + line) - self.sock.sendall(line + '\r\n') + #self.sock.sendall(line + '\r\n') + self.sslsock.send(line + '\r\n') def check_for_orders(self, nick, chunks): if chunks[0] in ordername_list: @@ -308,10 +309,11 @@ def __handle_privmsg(self, source, target, message): self.__on_pubmsg(nick, message) def __handle_line(self, line): - line = line.rstrip() + line = line.strip('\r\n') #print('<< ' + line) if line.startswith('PING '): - self.send_raw(line.replace('PING', 'PONG')) + #self.send_raw(line.replace('PING', 'PONG')) + self.send_raw('PONG '+line.split()[1]) return chunks = line.split(' ') @@ -409,8 +411,11 @@ def run(self): try: debug('connecting') self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.sock.connect(self.serverport) - self.fd = self.sock.makefile() + #self.sock.connect(self.serverport) + #self.fd = self.sock.makefile() + self.sslsock = ssl.wrap_socket(self.sock) + self.sslsock.connect(self.serverport) + self.fd = self.sslsock.makefile() if self.password: self.send_raw('CAP REQ :sasl') self.send_raw('USER %s b c :%s' % self.userrealname) From 6b18e0d0ee1a9a839e3fc7347ae783b5667c4ced Mon Sep 17 00:00:00 2001 From: esh Date: Sat, 18 Apr 2015 16:11:18 +0300 Subject: [PATCH 205/409] check for 'usessl = true' in .cfg file and use ssl if applicable --- joinmarket.cfg | 1 + lib/irc.py | 18 ++++++++---------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/joinmarket.cfg b/joinmarket.cfg index 0853bb8b..37a792b4 100644 --- a/joinmarket.cfg +++ b/joinmarket.cfg @@ -8,4 +8,5 @@ bitcoin_cli_cmd = bitcoin-cli host = chat.freenode.net channel = joinmarket-pit-test port = 6697 +usessl = true #More stuff to come diff --git a/lib/irc.py b/lib/irc.py index 7da341c3..105cd49b 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -144,8 +144,7 @@ def __privmsg(self, nick, cmd, message): def send_raw(self, line): #if not line.startswith('PING LAG'): # debug('sendraw ' + line) - #self.sock.sendall(line + '\r\n') - self.sslsock.send(line + '\r\n') + self.sock.sendall(line + '\r\n') def check_for_orders(self, nick, chunks): if chunks[0] in ordername_list: @@ -309,11 +308,10 @@ def __handle_privmsg(self, source, target, message): self.__on_pubmsg(nick, message) def __handle_line(self, line): - line = line.strip('\r\n') + line = line.rstrip() #print('<< ' + line) if line.startswith('PING '): - #self.send_raw(line.replace('PING', 'PONG')) - self.send_raw('PONG '+line.split()[1]) + self.send_raw(line.replace('PING', 'PONG')) return chunks = line.split(' ') @@ -411,11 +409,11 @@ def run(self): try: debug('connecting') self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - #self.sock.connect(self.serverport) - #self.fd = self.sock.makefile() - self.sslsock = ssl.wrap_socket(self.sock) - self.sslsock.connect(self.serverport) - self.fd = self.sslsock.makefile() + if config.get("MESSAGING","usessl").lower() == 'true': + + self.sock = ssl.wrap_socket(self.sock) + self.sock.connect(self.serverport) + self.fd = self.sock.makefile() if self.password: self.send_raw('CAP REQ :sasl') self.send_raw('USER %s b c :%s' % self.userrealname) From 3434ee03505c4118c52fb2711aa7a7f809356242 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sat, 18 Apr 2015 21:08:20 +0100 Subject: [PATCH 206/409] yield generator avoids outputs below dust threshold of 5430 satoshi --- lib/common.py | 3 ++- lib/irc.py | 1 + lib/maker.py | 15 +++++++++------ lib/taker.py | 5 ++++- patientsendpayment.py | 2 +- yield-generator.py | 21 +++++++++++++++++---- 6 files changed, 34 insertions(+), 13 deletions(-) diff --git a/lib/common.py b/lib/common.py index c0451e36..c6dfc508 100644 --- a/lib/common.py +++ b/lib/common.py @@ -9,6 +9,7 @@ import os nickname = '' MAX_PRIVMSG_LEN = 400 +DUST_THRESHOLD = 5430 bc_interface = None ordername_list = ["absorder", "relorder"] debug_file_handle = None @@ -186,7 +187,7 @@ def get_utxos_by_mixdepth(self): ''' returns a list of utxos sorted by different mix levels ''' - debug('wallet.unspent = \n' + pprint.pformat(self.unspent)) + debug('get_utxos_by_mixdepth wallet.unspent = \n' + pprint.pformat(self.unspent)) mix_utxo_list = {} for m in range(self.max_mix_depth): mix_utxo_list[m] = {} diff --git a/lib/irc.py b/lib/irc.py index 105cd49b..7b231875 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -226,6 +226,7 @@ def __on_privmsg(self, nick, message): self.on_seen_tx(nick, txhex) except CJPeerError: #TODO proper error handling + debug('cj peer error TODO handle') continue def __on_pubmsg(self, nick, message): diff --git a/lib/maker.py b/lib/maker.py index 618f053e..2b0708e0 100644 --- a/lib/maker.py +++ b/lib/maker.py @@ -26,7 +26,14 @@ def __init__(self, maker, nick, oid, amount, taker_pk): order = order_s[0] if amount < order['minsize'] or amount > order['maxsize']: self.maker.msgchan.send_error(nick, 'amount out of range') - self.utxos, self.cj_addr, self.change_addr = maker.oid_to_order(oid, amount) + self.ordertype = order['ordertype'] + self.txfee = order['txfee'] + self.cjfee = order['cjfee'] + debug('new cjorder nick=%s oid=%d amount=%d' % (nick, oid, amount)) + self.utxos, self.cj_addr, self.change_addr = maker.oid_to_order(self, oid, amount) + if not self.utxos: + self.maker.msgchan.send_error(nick, 'unable to fill order constrained by dust avoidance') + #TODO make up orders offers in a way that this error cant appear #check nothing has messed up with the wallet code, remove this code after a while import pprint debug('maker utxos = ' + pprint.pformat(self.utxos)) @@ -41,10 +48,6 @@ def __init__(self, maker, nick, oid, amount, taker_pk): str(self.utxos[utxo]['value']) + ' got ' + str(data['value'])) sys.exit(0) - self.ordertype = order['ordertype'] - self.txfee = order['txfee'] - self.cjfee = order['cjfee'] - debug('new cjorder nick=%s oid=%d amount=%d' % (nick, oid, amount)) #always a new address even if the order ends up never being # furfilled, you dont want someone pretending to fill all your # orders to find out which addresses you use @@ -271,7 +274,7 @@ def create_my_orders(self): #has to return a list of utxos and mixing depth the cj address will be in # the change address will be in mixing_depth-1 - def oid_to_order(self, oid, amount): + def oid_to_order(self, cjorder, oid, amount): ''' unspent = [] for utxo, addrvalue in self.wallet.unspent.iteritems(): diff --git a/lib/taker.py b/lib/taker.py index e5ee7582..fbb1bb7b 100644 --- a/lib/taker.py +++ b/lib/taker.py @@ -118,9 +118,11 @@ def recv_txio(self, nick, utxo_list, cj_pub, change_addr): def add_signature(self, sigb64): sig = base64.b64decode(sigb64).encode('hex') inserted_sig = False + unsigned_input_count = sum([len(u) for u in self.utxos.values()]) tx = btc.serialize(self.latest_tx) for index, ins in enumerate(self.latest_tx['ins']): if ins['script'] != '': + unsigned_input_count -= 1 continue utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) utxo_data = common.bc_interface.query_utxo_set(utxo) @@ -128,7 +130,8 @@ def add_signature(self, sigb64): continue sig_good = btc.verify_tx_input(tx, index, utxo_data[0]['script'], *btc.deserialize_script(sig)) if sig_good: - debug('found good sig at index=%d' % (index)) + unsigned_input_count -= 1 + debug('found good sig at index=%d remaining=%d' % (index, unsigned_input_count)) ins['script'] = sig inserted_sig = True break diff --git a/patientsendpayment.py b/patientsendpayment.py index c794e29a..f0b8f7a0 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -76,7 +76,7 @@ def create_my_orders(self): 'maxsize': self.amount, 'txfee': self.txfee, 'cjfee': self.cjfee} return [order] - def oid_to_order(self, oid, amount): + def oid_to_order(self, cjorder, oid, amount): #TODO race condition (kinda) #if an order arrives and before it finishes another order arrives # its possible this bot will end up paying to the destaddr more than it diff --git a/yield-generator.py b/yield-generator.py index 4ff931c5..9dc802c4 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -13,7 +13,7 @@ from socket import gethostname txfee = 1000 -cjfee = '0.002' # 1% fee +cjfee = '0.002' # 0.2% fee mix_levels = 5 nickname = 'yigen-'+binascii.hexlify(os.urandom(4)) nickserv_password = '' @@ -48,20 +48,33 @@ def create_my_orders(self): 'maxsize': mix_balance[max_mix], 'txfee': txfee, 'cjfee': cjfee} return [order] - def oid_to_order(self, oid, amount): + def oid_to_order(self, cjorder, oid, amount): mix_balance = self.wallet.get_balance_by_mixdepth() max_mix = max(mix_balance, key=mix_balance.get) #algo attempts to make the largest-balance mixing depth get an even larger balance + debug('finding suitable mixdepth') mixdepth = (max_mix - 1) % self.wallet.max_mix_depth while True: - if mixdepth in mix_balance and mix_balance[mixdepth] > amount: + if mixdepth in mix_balance and mix_balance[mixdepth] >= amount: break mixdepth = (mixdepth - 1) % self.wallet.max_mix_depth #mixdepth is the chosen depth we'll be spending from - utxos = self.wallet.select_utxos(mixdepth, amount) cj_addr = self.wallet.get_receive_addr((mixdepth + 1) % self.wallet.max_mix_depth) change_addr = self.wallet.get_change_addr(mixdepth) + + utxos = self.wallet.select_utxos(mixdepth, amount) + my_total_in = sum([va['value'] for va in utxos.values()]) + real_cjfee = calc_cj_fee(cjorder.ordertype, cjorder.cjfee, amount) + change_value = my_total_in - amount - cjorder.txfee + real_cjfee + if change_value <= common.DUST_THRESHOLD: + debug('change value=%d below dust threshold, finding new utxos' % (change_value)) + try: + utxos = self.wallet.select_utxos(mixdepth, amount + common.DUST_THRESHOLD) + except Exception: + debug('dont have the required UTXOs to make a output above the dust threshold, quitting') + return None, None, None + return utxos, cj_addr, change_addr def on_tx_unconfirmed(self, cjorder, txid, removed_utxos): From 6733ceb97676e0c07587f1f958975719cb8fddcd Mon Sep 17 00:00:00 2001 From: Belcher Date: Sat, 18 Apr 2015 21:14:53 +0100 Subject: [PATCH 207/409] edited formatting of wallet-tool output --- wallet-tool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wallet-tool.py b/wallet-tool.py index 1e0a7962..3d315090 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -72,7 +72,7 @@ privkey = btc.encode_privkey(wallet.get_key(m, forchange, k), 'wif_compressed', get_addr_vbyte()) if options.showprivkey else '' if method == 'displayall' or balance > 0 or used == ' new': - print ' m/0/%d/%d/%02d %s %s %.8fbtc %s' % (m, forchange, k, addr, used, balance/1e8, privkey) + print ' m/0/%d/%d/%03d %s %s %.8f btc %s' % (m, forchange, k, addr, used, balance/1e8, privkey) print 'for mixdepth=%d balance=%.8fbtc' % (m, balance_depth/1e8) total_balance += balance_depth print 'total balance = %.8fbtc' % (total_balance/1e8) From b5191974e12ba8bb0c9bf1e859aaf57d18b7e671 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sat, 18 Apr 2015 21:45:21 +0100 Subject: [PATCH 208/409] added message showing when bot connected --- lib/irc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/irc.py b/lib/irc.py index 7b231875..9659e43c 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -354,6 +354,7 @@ def __handle_line(self, line): elif chunks[1] == '332' or chunks[1] == 'TOPIC': #channel topic topic = get_irc_text(line) self.on_set_topic(topic) + debug('Connected to IRC and joined channel') elif chunks[1] == 'QUIT': nick = get_irc_nick(chunks[0]) if nick == self.nick: From 85c2d956093c8fb349e45b01ceb65ac9ae5f5858 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sat, 18 Apr 2015 23:17:00 +0100 Subject: [PATCH 209/409] made announce_orders() use many commands on the same irc line --- lib/common.py | 1 - lib/irc.py | 27 ++++++++++++++++----------- lib/maker.py | 9 +++++---- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/lib/common.py b/lib/common.py index c6dfc508..2b915159 100644 --- a/lib/common.py +++ b/lib/common.py @@ -8,7 +8,6 @@ from ConfigParser import SafeConfigParser import os nickname = '' -MAX_PRIVMSG_LEN = 400 DUST_THRESHOLD = 5430 bc_interface = None ordername_list = ["absorder", "relorder"] diff --git a/lib/irc.py b/lib/irc.py index 9659e43c..b3220a71 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -7,6 +7,7 @@ import base64, os import enc_wrapper +MAX_PRIVMSG_LEN = 350 COMMAND_PREFIX = '!' PING_INTERVAL = 40 PING_TIMEOUT = 10 @@ -92,14 +93,18 @@ def send_tx(self, nick_list, txhex): def announce_orders(self, orderlist, nick=None): #nick=None means announce publicly order_keys = ['oid', 'minsize', 'maxsize', 'txfee', 'cjfee'] - orderline = '' - for order in orderlist: - #TODO send all the orders on one line - elem_list = [str(order[k]) for k in order_keys] - if nick: - self.__privmsg(nick, order['ordertype'], ' '.join(elem_list)) - else: - self.__pubmsg(COMMAND_PREFIX + order['ordertype'] + ' ' + ' '.join(elem_list)) + header = 'PRIVMSG ' + (nick if nick else self.channel) + ' :' + orderlines = [] + for i, order in enumerate(orderlist): + orderparams = COMMAND_PREFIX + order['ordertype'] +\ + ' ' + ' '.join([str(order[k]) for k in order_keys]) + orderlines.append(orderparams) + line = header + ''.join(orderlines) + if len(line) > MAX_PRIVMSG_LEN or i == len(orderlist)-1: + if i < len(orderlist)-1: + line = header + ''.join(orderlines[:-1]) + self.send_raw(line) + orderlines = [orderlines[-1]] def cancel_orders(self, oid_list): clines = [COMMAND_PREFIX + 'cancel ' + str(oid) for oid in oid_list] @@ -130,8 +135,8 @@ def __privmsg(self, nick, cmd, message): if box: message = enc_wrapper.encrypt_encode(message, box) - if len(message) > 350: - message_chunks = chunks(message, 350) + if len(message) > MAX_PRIVMSG_LEN: + message_chunks = chunks(message, MAX_PRIVMSG_LEN) else: message_chunks = [message] @@ -351,10 +356,10 @@ def __handle_line(self, line): self.connect_attempts = 0 if self.on_welcome: self.on_welcome() + debug('Connected to IRC and joined channel') elif chunks[1] == '332' or chunks[1] == 'TOPIC': #channel topic topic = get_irc_text(line) self.on_set_topic(topic) - debug('Connected to IRC and joined channel') elif chunks[1] == 'QUIT': nick = get_irc_nick(chunks[0]) if nick == self.nick: diff --git a/lib/maker.py b/lib/maker.py index 2b0708e0..921c4b40 100644 --- a/lib/maker.py +++ b/lib/maker.py @@ -263,7 +263,7 @@ def create_my_orders(self): #each utxo is a single absolute-fee order orderlist = [] for utxo, addrvalue in self.wallet.unspent.iteritems(): - order = {'oid': self.get_next_oid(), 'ordertype': 'absorder', 'minsize': 0, + order = {'oid': self.get_next_oid(), 'ordertype': 'absorder', 'minsize': 12000, 'maxsize': addrvalue['value'], 'txfee': 10000, 'cjfee': 100000, 'utxo': utxo, 'mixdepth': self.wallet.addr_cache[addrvalue['address']][0]} orderlist.append(order) @@ -308,12 +308,12 @@ def on_tx_confirmed(self, cjorder, confirmations, txid): for i, out in enumerate(cjorder.tx['outs']): addr = btc.script_to_address(out['script'], get_addr_vbyte()) if addr == cjorder.change_addr: - neworder = {'oid': self.get_next_oid(), 'ordertype': 'absorder', 'minsize': 0, + neworder = {'oid': self.get_next_oid(), 'ordertype': 'absorder', 'minsize': 12000, 'maxsize': out['value'], 'txfee': 10000, 'cjfee': 100000, 'utxo': txid + ':' + str(i)} to_announce.append(neworder) if addr == cjorder.cj_addr: - neworder = {'oid': self.get_next_oid(), 'ordertype': 'absorder', 'minsize': 0, + neworder = {'oid': self.get_next_oid(), 'ordertype': 'absorder', 'minsize': 12000, 'maxsize': out['value'], 'txfee': 10000, 'cjfee': 100000, 'utxo': txid + ':' + str(i)} to_announce.append(neworder) @@ -326,8 +326,9 @@ def main(): import sys seed = sys.argv[1] #btc.sha256('dont use brainwallets except for holding testnet coins') + common.load_program_config() wallet = Wallet(seed, max_mix_depth=5) - wallet.sync_wallet() + common.bc_interface.sync_wallet(wallet) from irc import IRCMessageChannel irc = IRCMessageChannel(nickname) From 6c487eeed848f3b722bc258758389e9ed2321cf8 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 19 Apr 2015 00:33:40 +0100 Subject: [PATCH 210/409] fixed bug in orders-on-one-line code --- lib/irc.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/irc.py b/lib/irc.py index b3220a71..9dddba9c 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -99,10 +99,10 @@ def announce_orders(self, orderlist, nick=None): orderparams = COMMAND_PREFIX + order['ordertype'] +\ ' ' + ' '.join([str(order[k]) for k in order_keys]) orderlines.append(orderparams) - line = header + ''.join(orderlines) + line = header + ''.join(orderlines) + ' ~' if len(line) > MAX_PRIVMSG_LEN or i == len(orderlist)-1: if i < len(orderlist)-1: - line = header + ''.join(orderlines[:-1]) + line = header + ''.join(orderlines[:-1]) + ' ~' self.send_raw(line) orderlines = [orderlines[-1]] @@ -143,7 +143,8 @@ def __privmsg(self, nick, cmd, message): for m in message_chunks: trailer = ' ~' if m==message_chunks[-1] else ' ;' header = "PRIVMSG " + nick + " :" - if m==message_chunks[0]: header += COMMAND_PREFIX + cmd + ' ' + if m==message_chunks[0]: + header += COMMAND_PREFIX + cmd + ' ' self.send_raw(header + m + trailer) def send_raw(self, line): @@ -277,10 +278,11 @@ def __handle_privmsg(self, source, target, message): if endindex == -1: return ctcp = message[1:endindex + 1] - self.send_raw('PRIVMSG ' + nick + ' :\x01VERSION xchat 2.8.8 Ubuntu\x01') + self.send_raw('PRIVMSG ' + nick + ' :\x01VERSION xchat 2.8.8 Ubuntu\x01') if target == self.nick: if nick not in self.built_privmsg: if message[0] != COMMAND_PREFIX: + debug('message not a cmd') return #new message starting cmd_string = message[1:].split(' ')[0] @@ -309,9 +311,11 @@ def __handle_privmsg(self, source, target, message): else: #drop the bad nick del self.built_privmsg[nick] - else: + elif target == self.channel: debug("< Date: Sun, 19 Apr 2015 00:41:09 +0100 Subject: [PATCH 211/409] fixed ctcp version bug where it would reply regardless of request --- lib/irc.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/irc.py b/lib/irc.py index 9dddba9c..332ce625 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -273,13 +273,16 @@ def __get_encryption_box(self, cmd, nick): def __handle_privmsg(self, source, target, message): nick = get_irc_nick(source) - if message[0] == '\x01': - endindex = message[1:].find('\x01') - if endindex == -1: - return - ctcp = message[1:endindex + 1] - self.send_raw('PRIVMSG ' + nick + ' :\x01VERSION xchat 2.8.8 Ubuntu\x01') if target == self.nick: + if message[0] == '\x01': + endindex = message[1:].find('\x01') + if endindex == -1: + return + ctcp = message[1:endindex + 1] + if ctcp.upper() == 'VERSION': + self.send_raw('PRIVMSG ' + nick + ' :\x01VERSION xchat 2.8.8 Ubuntu\x01') + return + if nick not in self.built_privmsg: if message[0] != COMMAND_PREFIX: debug('message not a cmd') From 361e6469ccfe6e5ad36c0c5a696076c3b186698f Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 19 Apr 2015 00:43:25 +0100 Subject: [PATCH 212/409] whoops, stupid typo --- lib/irc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/irc.py b/lib/irc.py index 332ce625..bad90042 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -281,7 +281,7 @@ def __handle_privmsg(self, source, target, message): ctcp = message[1:endindex + 1] if ctcp.upper() == 'VERSION': self.send_raw('PRIVMSG ' + nick + ' :\x01VERSION xchat 2.8.8 Ubuntu\x01') - return + return if nick not in self.built_privmsg: if message[0] != COMMAND_PREFIX: From e0c410f41e2166ab1b47f5e2d04c58085acf265d Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 19 Apr 2015 00:52:52 +0100 Subject: [PATCH 213/409] removed remaining sigs in add_signature(), it wouldnt always be correct --- lib/taker.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/taker.py b/lib/taker.py index fbb1bb7b..e5ee7582 100644 --- a/lib/taker.py +++ b/lib/taker.py @@ -118,11 +118,9 @@ def recv_txio(self, nick, utxo_list, cj_pub, change_addr): def add_signature(self, sigb64): sig = base64.b64decode(sigb64).encode('hex') inserted_sig = False - unsigned_input_count = sum([len(u) for u in self.utxos.values()]) tx = btc.serialize(self.latest_tx) for index, ins in enumerate(self.latest_tx['ins']): if ins['script'] != '': - unsigned_input_count -= 1 continue utxo = ins['outpoint']['hash'] + ':' + str(ins['outpoint']['index']) utxo_data = common.bc_interface.query_utxo_set(utxo) @@ -130,8 +128,7 @@ def add_signature(self, sigb64): continue sig_good = btc.verify_tx_input(tx, index, utxo_data[0]['script'], *btc.deserialize_script(sig)) if sig_good: - unsigned_input_count -= 1 - debug('found good sig at index=%d remaining=%d' % (index, unsigned_input_count)) + debug('found good sig at index=%d' % (index)) ins['script'] = sig inserted_sig = True break From fd9b3ed6534dec1b110a421b736f6934daa3d7d4 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 19 Apr 2015 00:57:13 +0100 Subject: [PATCH 214/409] renamed guitaker to order book watcher, ob-watcher.py, its more accurate --- README.txt | 17 ++++++++--------- gui-taker.py => ob-watcher.py | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) rename gui-taker.py => ob-watcher.py (99%) diff --git a/README.txt b/README.txt index 8007c130..289e54d6 100644 --- a/README.txt +++ b/README.txt @@ -11,25 +11,24 @@ HOWTO try 1. You will need libsodium installed Get it here: http://doc.libsodium.org/installation/README.html -2. Come up with a wallet seed. This is a bit like a brainwallet, it can be any string. - For real bitcoins you would probably generate it from 128 bits of entropy and encode - in a 12-word mnemonic. For testnet just use anything. +2. run python wallet-tool.py generate + to create your encrypted wallet file, make sure to save the 12 word seed -$ python gui-taker.py +$ python ob-watcher.py Starts a local http server which you can connect to and will display the orderbook as well as some graphs -$ python wallet-tool.py [seed] +$ python wallet-tool.py [wallet] To print out a bunch of addresses, send some testnet coins to an address -$ python sendpayment.py -N 1 [seed] [amount-in-satoshi] [destination address] - Chooses the cheapest offer to do a 2-party coinjoin to send money to a destination address +$ python sendpayment.py -N 3 [wallet] [amount-in-satoshi] [destination address] + Chooses the cheapest offer to do a coinjoin with 3 other parties to send money to a destination address If you're a frugal user and don't want to pay for a coinjoin if you dont have to, use this command -$ python patientsendpayments.py -N 1 -w 2 [wallet seed] [amount in satoshi] [destination address] +$ python patientsendpayments.py -N 1 -w 2 [wallet] [amount in satoshi] [destination address] Announces orders and waits to coinjoin for a maximum of 2 hours. Once that time it up cancels the orders and pays to do a 2-party coinjoin. -$ python yield-generator.py [seed] +$ python yield-generator.py [wallet] Becomes an investor bot, being online indefinitely and doing coinjoin for the purpose of profit. Edit the file to change the IRC nick, offered fee, nickserv password and so on diff --git a/gui-taker.py b/ob-watcher.py similarity index 99% rename from gui-taker.py rename to ob-watcher.py index c511c1d0..d3f4776a 100644 --- a/gui-taker.py +++ b/ob-watcher.py @@ -202,7 +202,7 @@ def main(): import bitcoin as btc import common import binascii, os - common.nickname = 'guitaker-' +binascii.hexlify(os.urandom(4)) + common.nickname = 'watcher' +binascii.hexlify(os.urandom(4)) common.load_program_config() irc = IRCMessageChannel(common.nickname) From 17210c7c499a8a94c931642dac47f534b9939480 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 19 Apr 2015 02:31:13 +0100 Subject: [PATCH 215/409] edited irc __privmsg() to not exceed MAX_PRIVMSG_LEN when sending --- lib/irc.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/irc.py b/lib/irc.py index bad90042..90f54893 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -7,7 +7,7 @@ import base64, os import enc_wrapper -MAX_PRIVMSG_LEN = 350 +MAX_PRIVMSG_LEN = 400 COMMAND_PREFIX = '!' PING_INTERVAL = 40 PING_TIMEOUT = 10 @@ -134,17 +134,18 @@ def __privmsg(self, nick, cmd, message): #encrypt before chunking if box: message = enc_wrapper.encrypt_encode(message, box) - - if len(message) > MAX_PRIVMSG_LEN: - message_chunks = chunks(message, MAX_PRIVMSG_LEN) + + header = "PRIVMSG " + nick + " :" + max_chunk_len = MAX_PRIVMSG_LEN - len(header) - len(cmd) - 4 + #1 for command prefix 1 for space 2 for trailer + if len(message) > max_chunk_len: + message_chunks = chunks(message, max_chunk_len) else: message_chunks = [message] - for m in message_chunks: trailer = ' ~' if m==message_chunks[-1] else ' ;' - header = "PRIVMSG " + nick + " :" if m==message_chunks[0]: - header += COMMAND_PREFIX + cmd + ' ' + m = COMMAND_PREFIX + cmd + ' ' + m self.send_raw(header + m + trailer) def send_raw(self, line): From 919e97fd2d58a9758f60fec350fd6888210945f9 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 19 Apr 2015 16:14:07 +0100 Subject: [PATCH 216/409] added code for possibility of maker pushing the fully-signed tx, see issue#56 --- lib/irc.py | 14 +++++++++++++- lib/maker.py | 7 ++++++- lib/message_channel.py | 5 ++++- lib/taker.py | 7 +++++-- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/lib/irc.py b/lib/irc.py index 90f54893..05d40eea 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -12,7 +12,7 @@ PING_INTERVAL = 40 PING_TIMEOUT = 10 encrypted_commands = ["auth", "ioauth", "tx", "sig"] -plaintext_commands = ["fill", "error", "pubkey", "orderbook", "relorder", "absorder"] +plaintext_commands = ["fill", "error", "pubkey", "orderbook", "relorder", "absorder", "push"] def get_irc_text(line): return line[line[1:].find(':') + 2:] @@ -89,6 +89,10 @@ def send_tx(self, nick_list, txhex): self.__privmsg(nick, 'tx', txb64) time.sleep(1) #HACK! really there should be rate limiting, see issue#31 + def push_tx(self, nick, txhex): + txb64 = base64.b64encode(txhex.decode('hex')) + self.__privmsg(nick, 'push', txb64) + #Maker callbacks def announce_orders(self, orderlist, nick=None): #nick=None means announce publicly @@ -231,6 +235,14 @@ def __on_privmsg(self, nick, message): self.send_error(nick, 'bad base64 tx. ' + repr(e)) if self.on_seen_tx: self.on_seen_tx(nick, txhex) + elif chunks[0] == 'push': + b64tx = chunks[1] + try: + txhex = base64.b64decode(b64tx).encode('hex') + except TypeError as e: + self.send_error(nick, 'bad base64 tx. ' + repr(e)) + if self.on_push_tx: + self.on_push_tx(nick, txhex) except CJPeerError: #TODO proper error handling debug('cj peer error TODO handle') diff --git a/lib/maker.py b/lib/maker.py index 921c4b40..ca7b6258 100644 --- a/lib/maker.py +++ b/lib/maker.py @@ -166,7 +166,7 @@ def __init__(self, msgchan, wallet): self.msgchan.register_channel_callbacks(self.on_welcome, self.on_set_topic, None, None, self.on_nick_leave, None) msgchan.register_maker_callbacks(self.on_orderbook_requested, - self.on_order_fill, self.on_seen_auth, self.on_seen_tx) + self.on_order_fill, self.on_seen_auth, self.on_seen_tx, self.on_push_tx) msgchan.cjpeer = self self.active_orders = {} @@ -207,6 +207,11 @@ def on_seen_tx(self, nick, txhex): finally: self.wallet_unspent_lock.release() + def on_push_tx(self, nick, txhex): + debug('received txhex from ' + nick + ' to push\n' + txhex) + txid = common.bc_interface.pushtx(txhex) + debug('pushed tx ' + str(txid)) + def on_welcome(self): self.msgchan.announce_orders(self.orderlist) self.active_orders = {} diff --git a/lib/message_channel.py b/lib/message_channel.py index 56023020..a4c53006 100644 --- a/lib/message_channel.py +++ b/lib/message_channel.py @@ -28,6 +28,7 @@ def __init__(self): self.on_order_fill = None self.on_seen_auth = None self.on_seen_tx = None + self.on_push_tx = None def run(self): pass def shutdown(self): pass @@ -61,14 +62,16 @@ def register_taker_callbacks(self, on_error=None, on_pubkey=None, on_ioauth=None def fill_orders(self, nickoid_dict, cj_amount, taker_pubkey): pass def send_auth(self, nick, pubkey, sig): pass def send_tx(self, nick_list, txhex): pass + def push_tx(self, nick, txhex): pass #maker commands def register_maker_callbacks(self, on_orderbook_requested=None, on_order_fill=None, - on_seen_auth=None, on_seen_tx=None): + on_seen_auth=None, on_seen_tx=None, on_push_tx=None): self.on_orderbook_requested = on_orderbook_requested self.on_order_fill = on_order_fill self.on_seen_auth = on_seen_auth self.on_seen_tx = on_seen_tx + self.on_push_tx = on_push_tx def announce_orders(self, orderlist, nick=None): pass #nick=None means announce publicly def cancel_orders(self, oid_list): pass def send_pubkey(self, nick, pubkey): pass diff --git a/lib/taker.py b/lib/taker.py index e5ee7582..11c85fc0 100644 --- a/lib/taker.py +++ b/lib/taker.py @@ -145,9 +145,12 @@ def add_signature(self, sigb64): if not tx_signed: return debug('the entire tx is signed, ready to pushtx()') + txhex = btc.serialize(self.latest_tx) + debug('\n' + txhex) - debug('\n' + btc.serialize(self.latest_tx)) - txid = common.bc_interface.pushtx(btc.serialize(self.latest_tx)) + #TODO send to a random maker or push myself + #self.msgchan.push_tx(self.active_orders.keys()[0], txhex) + txid = common.bc_interface.pushtx(txhex) debug('pushed tx ' + str(txid)) if self.finishcallback != None: self.finishcallback(self) From 4da8eb066d86202942fbfb88611ec9930c8ae026 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 19 Apr 2015 16:14:52 +0100 Subject: [PATCH 217/409] fixed bug with bitcoin core rpc code --- lib/blockchaininterface.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/blockchaininterface.py b/lib/blockchaininterface.py index 87b6f0b7..6e2096a6 100644 --- a/lib/blockchaininterface.py +++ b/lib/blockchaininterface.py @@ -408,6 +408,8 @@ def sync_unspent(self, wallet): wallet.unspent = {} unspent_list = json.loads(self.rpc(['listunspent'])) for u in unspent_list: + if 'account' not in u: + continue if u['account'] != wallet_name: continue if u['address'] not in wallet.addr_cache: From 935c9a370971a46ce50e0f8481774e85eedd99e0 Mon Sep 17 00:00:00 2001 From: Belcher Date: Wed, 22 Apr 2015 23:59:07 +0100 Subject: [PATCH 218/409] yield generator writes to an income statement spreadsheet after each successful coinjoin --- lib/irc.py | 7 +++++-- lib/maker.py | 6 +++--- yield-generator.py | 29 +++++++++++++++++++++++++---- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/lib/irc.py b/lib/irc.py index 05d40eea..282bdb28 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -460,8 +460,11 @@ def run(self): except IOError as e: debug(repr(e)) finally: - self.fd.close() - self.sock.close() + try: + self.fd.close() + self.sock.close() + except Exception as e: + print repr(e) if self.on_disconnect: self.on_disconnect() debug('disconnected irc') diff --git a/lib/maker.py b/lib/maker.py index ca7b6258..094229e6 100644 --- a/lib/maker.py +++ b/lib/maker.py @@ -134,10 +134,10 @@ def verify_unsigned_tx(self, txd): return False, 'my utxos already spent' my_total_in = sum([va['value'] for va in self.utxos.values()]) - real_cjfee = calc_cj_fee(self.ordertype, self.cjfee, self.cj_amount) + self.real_cjfee = calc_cj_fee(self.ordertype, self.cjfee, self.cj_amount) expected_change_value = (my_total_in - self.cj_amount - - self.txfee + real_cjfee) - debug('earned = ' + str(real_cjfee - self.txfee)) + - self.txfee + self.real_cjfee) + debug('earned = ' + str(self.real_cjfee - self.txfee)) debug('mycjaddr, mychange = ' + self.cj_addr + ', ' + self.change_addr) times_seen_cj_addr = 0 diff --git a/yield-generator.py b/yield-generator.py index 9dc802c4..b0f66b82 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -1,6 +1,6 @@ #! /usr/bin/env python -import time, os, binascii, sys +import time, os, binascii, sys, datetime import pprint data_dir = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(data_dir, 'lib')) @@ -14,10 +14,10 @@ txfee = 1000 cjfee = '0.002' # 0.2% fee -mix_levels = 5 nickname = 'yigen-'+binascii.hexlify(os.urandom(4)) nickserv_password = '' minsize = int(1.2 * txfee / float(cjfee)) #minimum size is such that you always net profit at least 20% of the miner fee +mix_levels = 5 @@ -35,6 +35,21 @@ def __init__(self, msgchan, wallet): Maker.__init__(self, msgchan, wallet) self.msgchan.register_channel_callbacks(self.on_welcome, self.on_set_topic, None, None, self.on_nick_leave, None) + self.tx_unconfirm_timestamp = {} + + def log_statement(self, data): + data = [str(d) for d in data] + self.income_statement = open('yield-generator-income-statement.csv', 'aw') + self.income_statement.write(','.join(data) + '\n') + self.income_statement.close() + + def on_welcome(self): + Maker.on_welcome(self) + self.log_statement(['timestamp', 'cj amount/satoshi', 'my input count', + 'my input value/satoshi', 'cjfee/satoshi', 'earned/satoshi', + 'confirm time/min', 'notes']) + timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S") + self.log_statement([timestamp, '', '', '', '', '', '', 'Connected']) def create_my_orders(self): mix_balance = self.wallet.get_balance_by_mixdepth() @@ -78,6 +93,7 @@ def oid_to_order(self, cjorder, oid, amount): return utxos, cj_addr, change_addr def on_tx_unconfirmed(self, cjorder, txid, removed_utxos): + self.tx_unconfirm_timestamp[cjorder.cj_addr] = int(time.time()) #if the balance of the highest-balance mixing depth change then reannounce it oldorder = self.orderlist[0] if len(self.orderlist) > 0 else None neworders = self.create_my_orders() @@ -90,12 +106,17 @@ def on_tx_unconfirmed(self, cjorder, txid, removed_utxos): return ([], [neworders[0]]) def on_tx_confirmed(self, cjorder, confirmations, txid): - return self.on_tx_unconfirmed(None, None, None) + confirm_time = int(time.time()) - self.tx_unconfirm_timestamp[cjorder.cj_addr] + timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S") + self.log_statement([timestamp, cjorder.cj_amount, len(cjorder.utxos), + sum([av['value'] for av in cjorder.utxos.values()]), cjorder.real_cjfee, + cjorder.real_cjfee - cjorder.txfee, round(confirm_time / 60.0, 2), '']) + return self.on_tx_unconfirmed(cjorder, txid, None) def main(): common.load_program_config() import sys - seed = sys.argv[1] #btc.sha256('dont use brainwallets except for holding testnet coins') + seed = sys.argv[1] wallet = Wallet(seed, max_mix_depth = mix_levels) common.bc_interface.sync_wallet(wallet) wallet.print_debug_wallet_info() From 54e1a95f937ea8edbfd8e49b600dc918d19dae58 Mon Sep 17 00:00:00 2001 From: Belcher Date: Thu, 23 Apr 2015 01:43:50 +0100 Subject: [PATCH 219/409] added network field in wallet.json --- lib/common.py | 4 ++++ wallet-tool.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/common.py b/lib/common.py index 2b915159..a99aa17d 100644 --- a/lib/common.py +++ b/lib/common.py @@ -126,6 +126,10 @@ def get_seed(self, seedarg): walletfile = fd.read() fd.close() walletdata = json.loads(walletfile) + if walletdata['network'] != get_network(): + print 'wallet network(%s) does not match joinmarket configured network(%s)' % ( + walletdata['network'], get_network()) + sys.exit(0) password = getpass.getpass('Enter wallet decryption passphrase: ') password_key = btc.bin_dbl_sha256(password) decrypted_seed = aes.decryptData(password_key, walletdata['encrypted_seed'] diff --git a/wallet-tool.py b/wallet-tool.py index 3d315090..57390774 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -72,7 +72,7 @@ privkey = btc.encode_privkey(wallet.get_key(m, forchange, k), 'wif_compressed', get_addr_vbyte()) if options.showprivkey else '' if method == 'displayall' or balance > 0 or used == ' new': - print ' m/0/%d/%d/%03d %s %s %.8f btc %s' % (m, forchange, k, addr, used, balance/1e8, privkey) + print ' m/0/%d/%d/%03d %-35s%s %.8f btc %s' % (m, forchange, k, addr, used, balance/1e8, privkey) print 'for mixdepth=%d balance=%.8fbtc' % (m, balance_depth/1e8) total_balance += balance_depth print 'total balance = %.8fbtc' % (total_balance/1e8) @@ -113,7 +113,7 @@ encrypted_seed = aes.encryptData(password_key, seed.decode('hex')) timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S") walletfile = json.dumps({'creator': 'joinmarket project', 'creation_time': timestamp, - 'encrypted_seed': encrypted_seed.encode('hex')}) + 'encrypted_seed': encrypted_seed.encode('hex'), 'network': common.get_network()}) walletname = raw_input('Input wallet file name (default: wallet.json): ') if len(walletname) == 0: walletname = 'wallet.json' From 405bd4589b257e23146b66312303b35231ab1f74 Mon Sep 17 00:00:00 2001 From: Belcher Date: Thu, 23 Apr 2015 02:37:58 +0100 Subject: [PATCH 220/409] slight improvements with wallet-tool.py --- lib/common.py | 2 +- wallet-tool.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/common.py b/lib/common.py index a99aa17d..07f2fa45 100644 --- a/lib/common.py +++ b/lib/common.py @@ -116,7 +116,7 @@ def get_seed(self, seedarg): if not os.path.isfile(path): debug('seedarg interpreted as seed') return seedarg - debug('seedarg interpreted as wallet file name') + #debug('seedarg interpreted as wallet file name') try: import aes except ImportError: diff --git a/wallet-tool.py b/wallet-tool.py index 57390774..c591e47e 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -71,7 +71,7 @@ used = ('used' if k < wallet.index[m][forchange] else ' new') privkey = btc.encode_privkey(wallet.get_key(m, forchange, k), 'wif_compressed', get_addr_vbyte()) if options.showprivkey else '' - if method == 'displayall' or balance > 0 or used == ' new': + if method == 'displayall' or balance > 0 or (used == ' new' and forchange==0): print ' m/0/%d/%d/%03d %-35s%s %.8f btc %s' % (m, forchange, k, addr, used, balance/1e8, privkey) print 'for mixdepth=%d balance=%.8fbtc' % (m, balance_depth/1e8) total_balance += balance_depth @@ -98,7 +98,7 @@ if method == 'generate': seed = btc.sha256(os.urandom(64))[:32] words = old_mnemonic.mn_encode(seed) - print 'Write down this wallet recovery seed\n' + ' '.join(words) + '\n' + print 'Write down this wallet recovery seed\n\n' + ' '.join(words) + '\n' elif method == 'recover': words = raw_input('Input 12 word recovery seed: ') words = words.split(' ') @@ -125,4 +125,4 @@ hexseed = wallet.seed print 'hexseed = ' + hexseed words = old_mnemonic.mn_encode(hexseed) - print 'Wallet recovery seed\n' + ' '.join(words) + '\n' + print 'Wallet recovery seed\n\n' + ' '.join(words) + '\n' From e23dfbe3fabc27e70fed6d9c38da3a90be3b8ee0 Mon Sep 17 00:00:00 2001 From: Belcher Date: Thu, 23 Apr 2015 02:48:29 +0100 Subject: [PATCH 221/409] updated README with info about the irc channel and similar --- README.txt | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/README.txt b/README.txt index 289e54d6..e7b9c909 100644 --- a/README.txt +++ b/README.txt @@ -1,17 +1,30 @@ + +IRC Channel: +#joinmarket on irc.freenode.net +[https://webchat.freenode.net/?channels=%23joinmarket] + Bitcointalk thread: -https://bitcointalk.org/index.php?topic=919116.msg10096563 +[https://bitcointalk.org/index.php?topic=919116.msg10096563] +Subreddit: +[www.reddit.com/r/joinmarket] -FIRST IMPLEMENTATION OF JOINMARKET +Twitter: +[www.twitter.com/joinmarket] -you will need to know python somewhat to play around with it - also get some testnet coins +Wiki page for more detailed articles: +[https://github.com/chris-belcher/joinmarket/wiki] -HOWTO try +INSTALLING +0. You will need python 2.7 1. You will need libsodium installed Get it here: http://doc.libsodium.org/installation/README.html +2. You will need slowaes installed + sudo pip install slowaes +3. Get some testnet coins -2. run python wallet-tool.py generate +RUNNING +$ python wallet-tool.py generate to create your encrypted wallet file, make sure to save the 12 word seed $ python ob-watcher.py From f59c597b7af65c2840686a5d3c19fc8fa96c68c2 Mon Sep 17 00:00:00 2001 From: chris belcher Date: Thu, 23 Apr 2015 18:01:03 +0100 Subject: [PATCH 222/409] bugfix and edit in tumbler.py --- tumbler.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tumbler.py b/tumbler.py index dd8b5b7d..3c5a52b1 100644 --- a/tumbler.py +++ b/tumbler.py @@ -46,7 +46,7 @@ def generate_tumbler_tx(destaddrs, options): tx_list.append(tx) total_dest_addr = len(destaddrs) + options.addrask - external_dest_addrs = destaddrs + ['addrask']*options.addrask + external_dest_addrs = ['addrask']*options.addrask + destaddrs if total_dest_addr > options.mixdepthcount: print 'not enough mixing depths to pay to all destination addresses' return None @@ -62,10 +62,8 @@ def generate_tumbler_tx(destaddrs, options): for tx in tx_list: if tx['srcmixdepth'] == srcmix: if tx['dest'] == 'internal': - print 'removing tx = ' + str(tx) tx_list_remove.append(tx) else: - print 'setting amount to 1' tx['amount_ratio'] = 1.0 [tx_list.remove(t) for t in tx_list_remove] return tx_list @@ -102,7 +100,7 @@ def send_tx(self, tx, balance, sweep): addr_valid, errormsg = validate_address(destaddr) if addr_valid: break - print 'Address ' + addr + ' invalid. ' + errormsg + ' try again' + print 'Address ' + destaddr + ' invalid. ' + errormsg + ' try again' else: destaddr = tx['dest'] @@ -256,8 +254,8 @@ def main(): (options, args) = parser.parse_args() #TODO somehow implement a lower limit - if len(args) < 2: - parser.error('Needs a seed and destination addresses') + if len(args) < 1: + parser.error('Needs a seed') sys.exit(0) seed = args[0] destaddrs = args[1:] @@ -275,8 +273,6 @@ def main(): print 'this is very bad for privacy' print '='*50 - print 'seed=' + seed - print 'destaddrs=' + str(destaddrs) print str(options) tx_list = generate_tumbler_tx(destaddrs, options) if not tx_list: From cfe17be1f41340016d7bfe1f7c1d307589808de5 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sat, 25 Apr 2015 02:38:55 +0100 Subject: [PATCH 223/409] fixed bug where blockr query_utxo_set() would crash for more than 20 utxos --- lib/blockchaininterface.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/lib/blockchaininterface.py b/lib/blockchaininterface.py index 6e2096a6..c3b9bf14 100644 --- a/lib/blockchaininterface.py +++ b/lib/blockchaininterface.py @@ -61,6 +61,8 @@ def query_utxo_set(self, txouts): #address and output script contain the same information btw class BlockrInterface(BlockchainInterface): + BLOCKR_MAX_ADDR_REQ_COUNT = 20 + def __init__(self, testnet = False): super(BlockrInterface, self).__init__() self.network = 'testnet' if testnet else 'btc' #see bci.py in bitcoin module @@ -70,13 +72,12 @@ def __init__(self, testnet = False): def sync_addresses(self, wallet, gaplimit=6): common.debug('downloading wallet history') #sets Wallet internal indexes to be at the next unused address - addr_req_count = 20 for mix_depth in range(wallet.max_mix_depth): for forchange in [0, 1]: unused_addr_count = 0 last_used_addr = '' while unused_addr_count < gaplimit: - addrs = [wallet.get_new_addr(mix_depth, forchange) for i in range(addr_req_count)] + addrs = [wallet.get_new_addr(mix_depth, forchange) for i in range(self.BLOCKR_MAX_ADDR_REQ_COUNT)] #TODO send a pull request to pybitcointools # because this surely should be possible with a function from it @@ -103,7 +104,6 @@ def sync_unspent(self, wallet): common.debug('blockr sync_unspent() happened too recently (%dsec), skipping' % (st - self.last_sync_unspent)) return wallet.unspent = {} - addr_req_count = 20 addrs = wallet.addr_cache.keys() if len(addrs) == 0: @@ -111,7 +111,7 @@ def sync_unspent(self, wallet): return i = 0 while i < len(addrs): - inc = min(len(addrs) - i, addr_req_count) + inc = min(len(addrs) - i, self.BLOCKR_MAX_ADDR_REQ_COUNT) req = addrs[i:i + inc] i += inc @@ -238,11 +238,19 @@ def query_utxo_set(self, txout): if not isinstance(txout, list): txout = [txout] txids = [h[:64] for h in txout] - txids_dupremoved = list(set(txids)) - blockr_url = 'http://' + self.blockr_domain + '.blockr.io/api/v1/tx/info/' - data = json.loads(btc.make_request(blockr_url + ','.join(txids_dupremoved)))['data'] - if not isinstance(data, list): - data = [data] + txids = list(set(txids)) #remove duplicates + #self.BLOCKR_MAX_ADDR_REQ_COUNT = 2 + if len(txids) > self.BLOCKR_MAX_ADDR_REQ_COUNT: + txids = common.chunks(txids, self.BLOCKR_MAX_ADDR_REQ_COUNT) + else: + txids = [txids] + data = [] + for ids in txids: + blockr_url = 'http://' + self.blockr_domain + '.blockr.io/api/v1/tx/info/' + blockr_data = json.loads(btc.make_request(blockr_url + ','.join(ids)))['data'] + if not isinstance(blockr_data, list): + blockr_data = [blockr_data] + data += blockr_data result = [] for txo in txout: txdata = [d for d in data if d['tx'] == txo[:64]][0] From eedbfd4fe3c2825e6fc5e6a6189217cd98b768aa Mon Sep 17 00:00:00 2001 From: Belcher Date: Sat, 25 Apr 2015 02:50:00 +0100 Subject: [PATCH 224/409] slowed down sending of signatures, temporary rate limiting until issue#31 --- lib/irc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/irc.py b/lib/irc.py index 282bdb28..4a7c49c6 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -126,6 +126,7 @@ def send_sigs(self, nick, sig_list): #TODO make it send the sigs on one line if there's space for s in sig_list: self.__privmsg(nick, 'sig', s) + time.sleep(0.5) #HACK! really there should be rate limiting, see issue#31 def __pubmsg(self, message): debug('>>pubmsg ' + message) From 32fa7ac11c41b283fd1b446527fc7201ecf5bd78 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 26 Apr 2015 01:19:40 +0100 Subject: [PATCH 225/409] made regtest work, will tick forward the chain after 15 seconds to enable fast testing --- lib/blockchaininterface.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/blockchaininterface.py b/lib/blockchaininterface.py index c3b9bf14..b84a2f83 100644 --- a/lib/blockchaininterface.py +++ b/lib/blockchaininterface.py @@ -458,12 +458,19 @@ def query_utxo_set(self, txout): #with > 100 blocks. class RegtestBitcoinCoreInterface(BitcoinCoreInterface): def __init__(self, bitcoin_cli_cmd): - super(BitcoinCoreInterface, self).__init__(bitcoin_cli_cmd, False) + super(RegtestBitcoinCoreInterface, self).__init__(bitcoin_cli_cmd, False) self.command_params = bitcoin_cli_cmd + ['-regtest'] def pushtx(self, txhex): - ret = super(RegtestBitcoinCoreInterface, self).send_tx(txhex) - self.tick_forward_chain(1) + ret = super(RegtestBitcoinCoreInterface, self).pushtx(txhex) + class TickChainThread(threading.Thread): + def __init__(self, bcinterface): + threading.Thread.__init__(self) + self.bcinterface = bcinterface + def run(self): + time.sleep(15) + self.bcinterface.tick_forward_chain(1) + TickChainThread(self).start() return ret def tick_forward_chain(self, n): From de733c92262f994a66ca068b17012986ca64f07a Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 26 Apr 2015 18:22:46 +0100 Subject: [PATCH 226/409] increased number of addresses requested from bitcoin core at once, added tumbler.py label for tx list --- lib/blockchaininterface.py | 2 +- tumbler.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/blockchaininterface.py b/lib/blockchaininterface.py index b84a2f83..e20c9b8f 100644 --- a/lib/blockchaininterface.py +++ b/lib/blockchaininterface.py @@ -357,7 +357,7 @@ def add_watchonly_addresses(self, addr_list, wallet_name): def sync_addresses(self, wallet, gaplimit=6): common.debug('requesting wallet history') wallet_name = 'joinmarket-wallet-' + btc.dbl_sha256(wallet.keys[0][0])[:6] - addr_req_count = 20 + addr_req_count = 50 wallet_addr_list = [] for mix_depth in range(wallet.max_mix_depth): for forchange in [0, 1]: diff --git a/tumbler.py b/tumbler.py index 3c5a52b1..d9f17c3b 100644 --- a/tumbler.py +++ b/tumbler.py @@ -289,6 +289,7 @@ def main(): dbg_tx_list = [] for srcmixdepth, txlist in tx_dict.iteritems(): dbg_tx_list.append({'srcmixdepth': srcmixdepth, 'tx': txlist}) + print 'tumbler transaction list' pprint(dbg_tx_list) total_wait = sum([tx['wait'] for tx in tx_list]) From 7f28964403263d6ed1fe7cf6320f2806813aac97 Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 26 Apr 2015 22:12:19 +0100 Subject: [PATCH 227/409] changed names in tumbler.py for easier understanding --- tumbler.py | 24 ++++++++++++------------ wallet-tool.py | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tumbler.py b/tumbler.py index d9f17c3b..38cee007 100644 --- a/tumbler.py +++ b/tumbler.py @@ -32,17 +32,17 @@ def generate_tumbler_tx(destaddrs, options): tx_list = [] for m, txcount in enumerate(txcounts): #assume that the sizes of outputs will follow a power law - amount_ratios = 1.0 - np.random.power(options.amountpower, txcount) - amount_ratios /= sum(amount_ratios) + amount_fractions = 1.0 - np.random.power(options.amountpower, txcount) + amount_fractions /= sum(amount_fractions) #transaction times are uncorrelated #time between events in a poisson process followed exp waits = np.random.exponential(options.timelambda, txcount) #number of makers to use follows a normal distribution makercounts = np.random.normal(options.makercountrange[0], options.makercountrange[1], txcount) makercounts = lower_bounded_int(makercounts, 2) - for amount_ratio, wait, makercount in zip(amount_ratios, waits, makercounts): - tx = {'amount_ratio': amount_ratio, 'wait': round(wait, 2), - 'srcmixdepth': m + options.mixdepthsrc, 'makercount': makercount, 'dest': 'internal'} + for amount_fraction, wait, makercount in zip(amount_fractions, waits, makercounts): + tx = {'amount_fraction': amount_fraction, 'wait': round(wait, 2), + 'srcmixdepth': m + options.mixdepthsrc, 'makercount': makercount, 'destination': 'internal'} tx_list.append(tx) total_dest_addr = len(destaddrs) + options.addrask @@ -54,17 +54,17 @@ def generate_tumbler_tx(destaddrs, options): srcmix = options.mixdepthsrc + options.mixdepthcount - mix_offset - 1 for tx in reversed(tx_list): if tx['srcmixdepth'] == srcmix: - tx['dest'] = external_dest_addrs[mix_offset] + tx['destination'] = external_dest_addrs[mix_offset] break if mix_offset == 0: #setting last mixdepth to send all to dest tx_list_remove = [] for tx in tx_list: if tx['srcmixdepth'] == srcmix: - if tx['dest'] == 'internal': + if tx['destination'] == 'internal': tx_list_remove.append(tx) else: - tx['amount_ratio'] = 1.0 + tx['amount_fraction'] = 1.0 [tx_list.remove(t) for t in tx_list_remove] return tx_list @@ -92,9 +92,9 @@ def finishcallback(self, coinjointx): def send_tx(self, tx, balance, sweep): destaddr = None - if tx['dest'] == 'internal': + if tx['destination'] == 'internal': destaddr = self.taker.wallet.get_receive_addr(tx['srcmixdepth'] + 1) - elif tx['dest'] == 'addrask': + elif tx['destination'] == 'addrask': while True: destaddr = raw_input('insert new address: ') addr_valid, errormsg = validate_address(destaddr) @@ -102,7 +102,7 @@ def send_tx(self, tx, balance, sweep): break print 'Address ' + destaddr + ' invalid. ' + errormsg + ' try again' else: - destaddr = tx['dest'] + destaddr = tx['destination'] if sweep: print 'sweeping' @@ -126,7 +126,7 @@ def send_tx(self, tx, balance, sweep): self.taker.start_cj(self.taker.wallet, cjamount, orders, all_utxos, destaddr, None, self.taker.txfee, self.finishcallback) else: - amount = int(tx['amount_ratio'] * balance) + amount = int(tx['amount_fraction'] * balance) if amount < self.taker.mincjamount: print 'cj amount too low, bringing up' amount = self.taker.mincjamount diff --git a/wallet-tool.py b/wallet-tool.py index c591e47e..6fede164 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -42,8 +42,8 @@ if len(args) < 1: parser.error('Needs a seed, wallet file or method') sys.exit(0) - load_program_config() + if args[0] in noseed_methods: method = args[0] else: From 8e97da549589650b6def841c4156ca1c8abc5e5c Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 26 Apr 2015 23:54:50 +0100 Subject: [PATCH 228/409] bitcoin core interface checks whether an address is already in the wallet so -walletnotify would actually fire, adds address if not --- lib/blockchaininterface.py | 14 +++++++++++--- lib/maker.py | 2 +- lib/taker.py | 1 + tumbler.py | 2 +- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/lib/blockchaininterface.py b/lib/blockchaininterface.py index e20c9b8f..c4d244a1 100644 --- a/lib/blockchaininterface.py +++ b/lib/blockchaininterface.py @@ -42,7 +42,7 @@ def sync_unspent(self, wallet): pass @abc.abstractmethod - def add_tx_notify(self, txd, unconfirmfun, confirmfun): + def add_tx_notify(self, txd, unconfirmfun, confirmfun, notifyaddr): '''Invokes unconfirmfun and confirmfun when tx is seen on the network''' pass @@ -134,7 +134,7 @@ def sync_unspent(self, wallet): self.last_sync_unspent = time.time() common.debug('blockr sync_unspent took ' + str((self.last_sync_unspent - st)) + 'sec') - def add_tx_notify(self, txd, unconfirmfun, confirmfun): + def add_tx_notify(self, txd, unconfirmfun, confirmfun, notifyaddr): unconfirm_timeout = 10*60 #seconds unconfirm_poll_period = 5 confirm_timeout = 2*60*60 @@ -427,10 +427,18 @@ def sync_unspent(self, wallet): et = time.time() common.debug('bitcoind sync_unspent took ' + str((et - st)) + 'sec') - def add_tx_notify(self, txd, unconfirmfun, confirmfun): + def add_tx_notify(self, txd, unconfirmfun, confirmfun, notifyaddr): if not self.notifythread: self.notifythread = BitcoinCoreNotifyThread(self) self.notifythread.start() + one_addr_imported = False + for outs in txd['outs']: + addr = btc.script_to_address(outs['script'], common.get_addr_vbyte()) + if self.rpc(['getaccount', addr]) != '': + one_addr_imported = True + break + if not one_addr_imported: + self.rpc(['importaddress', notifyaddr, 'joinmarket-notify', 'false']) tx_output_set = set([(sv['script'], sv['value']) for sv in txd['outs']]) self.txnotify_fun.append((tx_output_set, unconfirmfun, confirmfun)) diff --git a/lib/maker.py b/lib/maker.py index 094229e6..761d12ed 100644 --- a/lib/maker.py +++ b/lib/maker.py @@ -91,7 +91,7 @@ def recv_tx(self, nick, txhex): sigs.append(base64.b64encode(btc.deserialize(txs)['ins'][index]['script'].decode('hex'))) #len(sigs) > 0 guarenteed since i did verify_unsigned_tx() - common.bc_interface.add_tx_notify(self.tx, self.unconfirm_callback, self.confirm_callback) + common.bc_interface.add_tx_notify(self.tx, self.unconfirm_callback, self.confirm_callback, self.cj_addr) debug('sending sigs ' + str(sigs)) self.maker.msgchan.send_sigs(nick, sigs) self.maker.active_orders[nick] = None diff --git a/lib/taker.py b/lib/taker.py index 11c85fc0..ac4bf37f 100644 --- a/lib/taker.py +++ b/lib/taker.py @@ -28,6 +28,7 @@ def __init__(self, msgchan, wallet, db, cj_amount, orders, input_utxos, my_cj_ad self.finishcallback = finishcallback self.my_txfee = my_txfee self.outputs = [{'address': my_cj_addr, 'value': self.cj_amount}] + self.my_cj_addr = my_cj_addr self.my_change_addr = my_change_addr self.cjfee_total = 0 self.latest_tx = None diff --git a/tumbler.py b/tumbler.py index 38cee007..01512041 100644 --- a/tumbler.py +++ b/tumbler.py @@ -87,7 +87,7 @@ def confirm_callback(self, txd, txid, confirmations): def finishcallback(self, coinjointx): common.bc_interface.add_tx_notify(coinjointx.latest_tx, - self.unconfirm_callback, self.confirm_callback) + self.unconfirm_callback, self.confirm_callback, coinjointx.my_cj_addr) self.taker.wallet.remove_old_utxos(coinjointx.latest_tx) def send_tx(self, tx, balance, sweep): From eaa3baf27172fdc6b10d4fa2c3de847a72975918 Mon Sep 17 00:00:00 2001 From: Belcher Date: Mon, 27 Apr 2015 22:07:45 +0100 Subject: [PATCH 229/409] increased default tumbler lambda to 20 minutes --- tumbler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tumbler.py b/tumbler.py index 01512041..15fa9117 100644 --- a/tumbler.py +++ b/tumbler.py @@ -243,10 +243,10 @@ def main(): 'This option controlls the parameters of that normal curve. (mean, standard deviation). default=(3, 1)') parser.add_option('--amountpower', type='float', dest='amountpower', default=100.0, help='The output amounts follow a power law distribution, this is the power, default=100.0') - parser.add_option('-l', '--timelambda', type='float', dest='timelambda', default=2, + parser.add_option('-l', '--timelambda', type='float', dest='timelambda', default=20, help='Average the number of minutes to wait between transactions. Randomly chosen ' ' following an exponential distribution, which describes the time between uncorrelated' - ' events. default=5') + ' events. default=20') parser.add_option('-w', '--wait-time', action='store', type='float', dest='waittime', help='wait time in seconds to allow orders to arrive, default=5', default=5) parser.add_option('-s', '--mincjamount', type='float', dest='mincjamount', default=0.0001, From 652ee06612a750e60d75cec30b8e7160816b4e96 Mon Sep 17 00:00:00 2001 From: Belcher Date: Mon, 27 Apr 2015 22:56:49 +0100 Subject: [PATCH 230/409] reduced dust threshold to 543, in line with electrum --- lib/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/common.py b/lib/common.py index 07f2fa45..4c6dcb40 100644 --- a/lib/common.py +++ b/lib/common.py @@ -8,7 +8,7 @@ from ConfigParser import SafeConfigParser import os nickname = '' -DUST_THRESHOLD = 5430 +DUST_THRESHOLD = 543 bc_interface = None ordername_list = ["absorder", "relorder"] debug_file_handle = None From 7e39cdbfb525b7165da2b9f5b7fd588c19032d1f Mon Sep 17 00:00:00 2001 From: chris belcher Date: Tue, 28 Apr 2015 11:30:03 +0100 Subject: [PATCH 231/409] fixed links in readme --- README.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.txt b/README.txt index e7b9c909..09959dd9 100644 --- a/README.txt +++ b/README.txt @@ -1,24 +1,24 @@ IRC Channel: #joinmarket on irc.freenode.net -[https://webchat.freenode.net/?channels=%23joinmarket] +https://webchat.freenode.net/?channels=%23joinmarket Bitcointalk thread: -[https://bitcointalk.org/index.php?topic=919116.msg10096563] +https://bitcointalk.org/index.php?topic=919116.msg10096563 Subreddit: -[www.reddit.com/r/joinmarket] +www.reddit.com/r/joinmarket Twitter: -[www.twitter.com/joinmarket] +www.twitter.com/joinmarket Wiki page for more detailed articles: -[https://github.com/chris-belcher/joinmarket/wiki] +https://github.com/chris-belcher/joinmarket/wiki INSTALLING 0. You will need python 2.7 1. You will need libsodium installed - Get it here: http://doc.libsodium.org/installation/README.html + Get it here: http://doc.libsodium.org/ 2. You will need slowaes installed sudo pip install slowaes 3. Get some testnet coins From 2afd70512a14fdeb1f1632742f4cbbdbb52be1cb Mon Sep 17 00:00:00 2001 From: chris belcher Date: Wed, 29 Apr 2015 16:15:55 +0100 Subject: [PATCH 232/409] irc bots get their nickname from a random wikipedia article --- lib/common.py | 2 +- lib/irc.py | 22 +++++++++++++++++----- ob-watcher.py | 4 ++-- patientsendpayment.py | 7 +++---- sendpayment.py | 5 +++-- tumbler.py | 5 +++-- yield-generator.py | 7 ++++--- 7 files changed, 33 insertions(+), 19 deletions(-) diff --git a/lib/common.py b/lib/common.py index 07f2fa45..2a2847fe 100644 --- a/lib/common.py +++ b/lib/common.py @@ -48,7 +48,7 @@ def debug(msg): print 'Alert Message: ' + alert_message print outmsg if nickname: #debugs before creating bot nick won't be handled like this - debug_file_handle.write(outmsg + '\n') + debug_file_handle.write(outmsg + '\r\n') def chunks(d, n): return [d[x: x+n] for x in xrange(0, len(d), n)] diff --git a/lib/irc.py b/lib/irc.py index 4a7c49c6..7fdded9b 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -4,7 +4,7 @@ from message_channel import CJPeerError import socket, threading, time, ssl -import base64, os +import base64, os, urllib2, re import enc_wrapper MAX_PRIVMSG_LEN = 400 @@ -14,6 +14,16 @@ encrypted_commands = ["auth", "ioauth", "tx", "sig"] plaintext_commands = ["fill", "error", "pubkey", "orderbook", "relorder", "absorder", "push"] +def random_nick(): + random_article_url = 'http://en.wikipedia.org/wiki/Special:Random' + page = urllib2.urlopen(random_article_url).read() + page_title = page[page.index('') + 7 : page.index('')] + title = page_title[:page_title.index(' - Wikipedia')] + ircnick = ''.join([s.capitalize() for s in re.split('\\W+', title)]) + if re.match('\\d', ircnick[0]): + ircnick = '_' + ircnick + return ircnick + def get_irc_text(line): return line[line[1:].find(':') + 2:] @@ -371,7 +381,8 @@ def __handle_line(self, line): self.on_connect() self.send_raw('JOIN ' + self.channel) elif chunks[1] == '433': #nick in use - self.nick += '_' + #self.nick = random_nick() + self.nick += '_' #helps keep identity constant if just _ added self.send_raw('NICK ' + self.nick) elif chunks[1] == '366': #end of names list self.connect_attempts = 0 @@ -413,10 +424,11 @@ def __handle_line(self, line): self.motd_fd.close() ''' - def __init__(self, nick, username='username', realname='realname', password=None): + def __init__(self, given_nick, username='username', realname='realname', password=None): MessageChannel.__init__(self) self.cjpeer = None #subclasses have to set this to self - self.nick = nick + self.given_nick = given_nick + self.nick = given_nick self.serverport = (config.get("MESSAGING","host"), int(config.get("MESSAGING","port"))) self.channel = '#'+ config.get("MESSAGING","channel") self.userrealname = (username, realname) @@ -445,7 +457,7 @@ def run(self): if self.password: self.send_raw('CAP REQ :sasl') self.send_raw('USER %s b c :%s' % self.userrealname) - self.send_raw('NICK ' + self.nick) + self.send_raw('NICK ' + self.given_nick) while 1: try: line = self.fd.readline() diff --git a/ob-watcher.py b/ob-watcher.py index d3f4776a..252acda3 100644 --- a/ob-watcher.py +++ b/ob-watcher.py @@ -6,7 +6,7 @@ sys.path.insert(0, os.path.join(data_dir, 'lib')) import taker -from irc import IRCMessageChannel +from irc import IRCMessageChannel, random_nick from common import * @@ -202,7 +202,7 @@ def main(): import bitcoin as btc import common import binascii, os - common.nickname = 'watcher' +binascii.hexlify(os.urandom(4)) + common.nickname =random_nick() #watcher' +binascii.hexlify(os.urandom(4)) common.load_program_config() irc = IRCMessageChannel(common.nickname) diff --git a/patientsendpayment.py b/patientsendpayment.py index f0b8f7a0..3b4df8d7 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -9,7 +9,7 @@ import common import taker import maker -from irc import IRCMessageChannel +from irc import IRCMessageChannel, random_nick import bitcoin as btc class TakerThread(threading.Thread): @@ -141,7 +141,6 @@ def main(): return waittime = timedelta(hours=options.waittime).total_seconds() - print 'Running patient sender of a payment' print 'txfee=%d cjfee=%d waittime=%s makercount=%d' % (options.txfee, options.cjfee, str(timedelta(hours=options.waittime)), options.makercount) @@ -153,7 +152,8 @@ def main(): print 'not enough money at mixdepth=%d, exiting' % (options.mixdepth) return - common.nickname = 'ppayer-' +binascii.hexlify(os.urandom(4)) + common.nickname = random_nick() + debug('Running patient sender of a payment') irc = IRCMessageChannel(common.nickname) bot = PatientSendPayment(irc, wallet, destaddr, amount, options.makercount, @@ -162,7 +162,6 @@ def main(): irc.run() except: debug('CRASHING, DUMPING EVERYTHING') - debug('wallet seed = ' + seed) debug_dump_object(wallet, ['addr_cache', 'keys', 'seed']) debug_dump_object(taker) import traceback diff --git a/sendpayment.py b/sendpayment.py index 85a92bac..777b589f 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -8,7 +8,7 @@ from common import * import common import taker as takermodule -from irc import IRCMessageChannel +from irc import IRCMessageChannel, random_nick import bitcoin as btc @@ -94,8 +94,9 @@ def main(): print 'ERROR: Address invalid. ' + errormsg return + common.nickname = random_nick() + debug('starting sendpayment') import binascii, os - common.nickname = 'payer-' +binascii.hexlify(os.urandom(4)) wallet = Wallet(seed, options.mixdepth + 1) common.bc_interface.sync_wallet(wallet) diff --git a/tumbler.py b/tumbler.py index 01512041..a5767562 100644 --- a/tumbler.py +++ b/tumbler.py @@ -6,7 +6,7 @@ import taker as takermodule import common from common import * -from irc import IRCMessageChannel +from irc import IRCMessageChannel, random_nick from optparse import OptionParser import numpy as np @@ -318,7 +318,8 @@ def main(): common.bc_interface.sync_wallet(wallet) wallet.print_debug_wallet_info() - common.nickname = 'tumbler-'+binascii.hexlify(os.urandom(4)) + common.nickname = random_nick() + debug('starting tumbler') irc = IRCMessageChannel(common.nickname) tumbler = Tumbler(irc, wallet, tx_list, options.txfee, options.maxcjfee, options.mincjamount) try: diff --git a/yield-generator.py b/yield-generator.py index b0f66b82..3875e133 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -6,7 +6,7 @@ sys.path.insert(0, os.path.join(data_dir, 'lib')) from maker import * -from irc import IRCMessageChannel +from irc import IRCMessageChannel, random_nick import bitcoin as btc import common @@ -14,7 +14,7 @@ txfee = 1000 cjfee = '0.002' # 0.2% fee -nickname = 'yigen-'+binascii.hexlify(os.urandom(4)) +nickname = random_nick() nickserv_password = '' minsize = int(1.2 * txfee / float(cjfee)) #minimum size is such that you always net profit at least 20% of the miner fee mix_levels = 5 @@ -120,8 +120,9 @@ def main(): wallet = Wallet(seed, max_mix_depth = mix_levels) common.bc_interface.sync_wallet(wallet) wallet.print_debug_wallet_info() - + common.nickname = nickname + debug('starting yield generator') irc = IRCMessageChannel(common.nickname, realname='btcint=' + common.config.get("BLOCKCHAIN", "blockchain_source"), password=nickserv_password) maker = YieldGenerator(irc, wallet) From 9364ed058296d6856bdd41836acfb55bf63c36db Mon Sep 17 00:00:00 2001 From: chris belcher Date: Fri, 1 May 2015 11:58:49 +0100 Subject: [PATCH 233/409] length limit to irc nick --- lib/irc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/irc.py b/lib/irc.py index 7fdded9b..29309551 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -22,6 +22,7 @@ def random_nick(): ircnick = ''.join([s.capitalize() for s in re.split('\\W+', title)]) if re.match('\\d', ircnick[0]): ircnick = '_' + ircnick + ircnick = ircnick[:9] return ircnick def get_irc_text(line): From b19ed269919cb0768c0cec71c8fed2b73d958b3c Mon Sep 17 00:00:00 2001 From: Belcher Date: Mon, 4 May 2015 23:38:53 +0100 Subject: [PATCH 234/409] rewrote irc alerts code, useful for getting people to update --- lib/blockchaininterface.py | 4 ++-- lib/common.py | 11 ++++++++--- lib/maker.py | 8 -------- lib/taker.py | 27 +++++++++++++++++++-------- 4 files changed, 29 insertions(+), 21 deletions(-) diff --git a/lib/blockchaininterface.py b/lib/blockchaininterface.py index c4d244a1..25430eea 100644 --- a/lib/blockchaininterface.py +++ b/lib/blockchaininterface.py @@ -296,8 +296,8 @@ def do_HEAD(self): common.debug('ran confirmfun') elif self.path.startswith('/alertnotify?'): - common.alert_message = self.path[len(pages[1]):] - common.debug('Got an alert!\nMessage=' + common.alert_message) + common.core_alert = self.path[len(pages[1]):] + common.debug('Got an alert!\nMessage=' + common.core_alert) os.system('wget -q --spider --timeout=0.5 --tries=1 http://localhost:' + str(self.base_server.server_address[1] + 1) + self.path) diff --git a/lib/common.py b/lib/common.py index f11c1d88..972b9431 100644 --- a/lib/common.py +++ b/lib/common.py @@ -7,12 +7,15 @@ import blockchaininterface from ConfigParser import SafeConfigParser import os + +JM_VERSION = 1 nickname = '' DUST_THRESHOLD = 543 bc_interface = None ordername_list = ["absorder", "relorder"] debug_file_handle = None -alert_message = None +core_alert = None +joinmarket_alert = None config = SafeConfigParser() config_location = 'joinmarket.cfg' @@ -44,8 +47,10 @@ def debug(msg): if nickname and not debug_file_handle: debug_file_handle = open(nickname+'.log','ab') outmsg = datetime.datetime.now().strftime("[%Y/%m/%d %H:%M:%S] ") + msg - if alert_message: - print 'Alert Message: ' + alert_message + if core_alert: + print 'Core Alert Message: ' + core_alert + if joinmarket_alert: + print 'JoinMarket Alert Message: ' + joinmarket_alert print outmsg if nickname: #debugs before creating bot nick won't be handled like this debug_file_handle.write(outmsg + '\r\n') diff --git a/lib/maker.py b/lib/maker.py index 761d12ed..43392fda 100644 --- a/lib/maker.py +++ b/lib/maker.py @@ -216,14 +216,6 @@ def on_welcome(self): self.msgchan.announce_orders(self.orderlist) self.active_orders = {} - def on_set_topic(self, newtopic): - chunks = newtopic.split('|') - if len(chunks) > 1: - print '=' * 60 - print 'MESSAGE FROM BELCHER!' - print chunks[1].strip() - print '=' * 60 - def on_nick_leave(self, nick): self.active_orders[nick] = None diff --git a/lib/taker.py b/lib/taker.py index ac4bf37f..f614fbb6 100644 --- a/lib/taker.py +++ b/lib/taker.py @@ -163,6 +163,25 @@ def __init__(self, msgchan): def get_crypto_box_from_nick(self, nick): raise Exception() + def on_set_topic(self, newtopic): + chunks = newtopic.split('|') + for msg in chunks[1:]: + try: + msg = msg.strip() + params = msg.split(' ') + min_version = int(params[0]) + max_version = int(params[1]) + alert = msg[msg.index(params[1]) + len(params[1]):].strip() + except ValueError, IndexError: + continue + if min_version < common.JM_VERSION and max_version > common.JM_VERSION: + print '=' * 60 + print 'JOINMARKET ALERT' + print alert + print '=' * 60 + common.joinmarket_alert = alert + + class OrderbookWatch(CoinJoinerPeer): def __init__(self, msgchan): CoinJoinerPeer.__init__(self, msgchan) @@ -196,14 +215,6 @@ def on_nick_leave(self, nick): def on_disconnect(self): self.db.execute('DELETE FROM orderbook;') - def on_set_topic(self, newtopic): - chunks = newtopic.split('|') - if len(chunks) > 1: - print '=' * 60 - print 'MESSAGE FROM BELCHER!' - print chunks[1].strip() - print '=' * 60 - #assume this only has one open cj tx at a time class Taker(OrderbookWatch): def __init__(self, msgchan): From c2a1a42bcfb30eb7295a6be0b22b791b8270cb59 Mon Sep 17 00:00:00 2001 From: Belcher Date: Tue, 5 May 2015 00:21:01 +0100 Subject: [PATCH 235/409] for non-testnet, using a seed is not supported --- lib/common.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/common.py b/lib/common.py index 972b9431..22be1551 100644 --- a/lib/common.py +++ b/lib/common.py @@ -119,8 +119,11 @@ def __init__(self, seedarg, max_mix_depth=2): def get_seed(self, seedarg): path = os.path.join('wallets', seedarg) if not os.path.isfile(path): - debug('seedarg interpreted as seed') - return seedarg + if get_network() == 'testnet': + debug('seedarg interpreted as seed, only available in testnet because this probably has lower entropy') + return seedarg + else: + raise IOError('wallet file not found') #debug('seedarg interpreted as wallet file name') try: import aes From 1888da6f713996e606960550283bd8bd740335e3 Mon Sep 17 00:00:00 2001 From: Belcher Date: Wed, 6 May 2015 20:53:40 +0100 Subject: [PATCH 236/409] moved pit to snoonet irc, it hides bots ip by default, has a tor onion --- joinmarket.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/joinmarket.cfg b/joinmarket.cfg index 37a792b4..61981230 100644 --- a/joinmarket.cfg +++ b/joinmarket.cfg @@ -5,7 +5,7 @@ network = testnet bitcoin_cli_cmd = bitcoin-cli [MESSAGING] -host = chat.freenode.net +host = irc.snoonet.org channel = joinmarket-pit-test port = 6697 usessl = true From bd02ba14cdff31845c348bccd6e89e67617e1bc8 Mon Sep 17 00:00:00 2001 From: Belcher Date: Wed, 6 May 2015 22:07:39 +0100 Subject: [PATCH 237/409] made installation instructions slightly clearer and removed info which is now in the wiki --- README.txt | 36 ++++++------------------------------ 1 file changed, 6 insertions(+), 30 deletions(-) diff --git a/README.txt b/README.txt index 09959dd9..8a082a6a 100644 --- a/README.txt +++ b/README.txt @@ -19,35 +19,11 @@ INSTALLING 0. You will need python 2.7 1. You will need libsodium installed Get it here: http://doc.libsodium.org/ -2. You will need slowaes installed + use this line to check it was installed correctly + python lib/enc_wrapper.py +2. You will need slowaes installed for encrypting your wallet sudo pip install slowaes -3. Get some testnet coins - -RUNNING -$ python wallet-tool.py generate - to create your encrypted wallet file, make sure to save the 12 word seed - -$ python ob-watcher.py - Starts a local http server which you can connect to and will display the orderbook as well as some graphs - -$ python wallet-tool.py [wallet] - To print out a bunch of addresses, send some testnet coins to an address - -$ python sendpayment.py -N 3 [wallet] [amount-in-satoshi] [destination address] - Chooses the cheapest offer to do a coinjoin with 3 other parties to send money to a destination address - -If you're a frugal user and don't want to pay for a coinjoin if you dont have to, use this command -$ python patientsendpayments.py -N 1 -w 2 [wallet] [amount in satoshi] [destination address] - Announces orders and waits to coinjoin for a maximum of 2 hours. Once that time it up cancels the - orders and pays to do a 2-party coinjoin. - -$ python yield-generator.py [wallet] - Becomes an investor bot, being online indefinitely and doing coinjoin for the purpose of profit. - Edit the file to change the IRC nick, offered fee, nickserv password and so on - -Watch the output of your bot(s), soon enough the taker will say it has completed - a transaction, maker will wait for the transaction to be seen and confirmed -If there are no orders, you could run two bots from the same machine. Be sure to use - two seperate wallet seeds though. - +3. you will need numpy 1.7 or later installed +4. (optional) matplotlib for displaying the graphs in orderbook-watcher +Read the wiki for more detailed articles on how to use From 5f8fe9065801dfd4c6f16e7da4bc6f61602b6172 Mon Sep 17 00:00:00 2001 From: Belcher Date: Wed, 6 May 2015 22:45:42 +0100 Subject: [PATCH 238/409] implemented the idea that the testnet channel is main channel + -test --- joinmarket.cfg | 2 +- lib/common.py | 13 ++++++++----- lib/irc.py | 3 +-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/joinmarket.cfg b/joinmarket.cfg index 61981230..defbaa38 100644 --- a/joinmarket.cfg +++ b/joinmarket.cfg @@ -6,7 +6,7 @@ bitcoin_cli_cmd = bitcoin-cli [MESSAGING] host = irc.snoonet.org -channel = joinmarket-pit-test +channel = joinmarket-pit port = 6697 usessl = true #More stuff to come diff --git a/lib/common.py b/lib/common.py index 22be1551..b72eff92 100644 --- a/lib/common.py +++ b/lib/common.py @@ -42,6 +42,12 @@ def load_program_config(): global bc_interface bc_interface = blockchaininterface.get_blockchain_interface_instance(config) +def get_config_irc_channel(): + channel = '#'+ config.get("MESSAGING","channel") + if get_network() == 'testnet': + channel += '-test' + return channel + def debug(msg): global debug_file_handle if nickname and not debug_file_handle: @@ -59,11 +65,8 @@ def chunks(d, n): return [d[x: x+n] for x in xrange(0, len(d), n)] def get_network(): - '''Returns network name as required by pybitcointools''' - if config.get("BLOCKCHAIN","network") == 'testnet': - return 'testnet' - else: - raise Exception("Only testnet is currently implemented") + '''Returns network name''' + return config.get("BLOCKCHAIN","network") def get_addr_vbyte(): if get_network() == 'testnet': diff --git a/lib/irc.py b/lib/irc.py index 29309551..8650816d 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -431,7 +431,7 @@ def __init__(self, given_nick, username='username', realname='realname', passwor self.given_nick = given_nick self.nick = given_nick self.serverport = (config.get("MESSAGING","host"), int(config.get("MESSAGING","port"))) - self.channel = '#'+ config.get("MESSAGING","channel") + self.channel = get_config_irc_channel() self.userrealname = (username, realname) if password and len(password) == 0: password = None @@ -451,7 +451,6 @@ def run(self): debug('connecting') self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if config.get("MESSAGING","usessl").lower() == 'true': - self.sock = ssl.wrap_socket(self.sock) self.sock.connect(self.serverport) self.fd = self.sock.makefile() From 659df430dcce730f6cb5f573cea3a8c7d7bfed54 Mon Sep 17 00:00:00 2001 From: Belcher Date: Wed, 6 May 2015 23:25:47 +0100 Subject: [PATCH 239/409] added instructions on how to run with mainnet --- README.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.txt b/README.txt index 8a082a6a..95cdb602 100644 --- a/README.txt +++ b/README.txt @@ -26,4 +26,8 @@ INSTALLING 3. you will need numpy 1.7 or later installed 4. (optional) matplotlib for displaying the graphs in orderbook-watcher +in the joinmarket.cfg configuration file, set + network = mainnet +for the actual bitcoin mainnet + Read the wiki for more detailed articles on how to use From ed5ed146cdac9d5de687cdb2a979de51ec602085 Mon Sep 17 00:00:00 2001 From: Belcher Date: Thu, 7 May 2015 00:16:52 +0100 Subject: [PATCH 240/409] added link to the wiki in the joinmarket.cfg --- joinmarket.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/joinmarket.cfg b/joinmarket.cfg index defbaa38..5f740b33 100644 --- a/joinmarket.cfg +++ b/joinmarket.cfg @@ -1,6 +1,7 @@ [BLOCKCHAIN] blockchain_source = blockr #options: blockr, json-rpc, regtest +#before using json-rpc read https://github.com/chris-belcher/joinmarket/wiki/Running-JoinMarket-with-Bitcoin-Core-full-node network = testnet bitcoin_cli_cmd = bitcoin-cli From 9a5dad2fe5f6c1a8a876a16642f3ea7de845f419 Mon Sep 17 00:00:00 2001 From: Belcher Date: Thu, 7 May 2015 00:28:39 +0100 Subject: [PATCH 241/409] removed the word seed from documentation of wallet-tool because thats only used for testnet --- joinmarket.cfg | 2 +- wallet-tool.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/joinmarket.cfg b/joinmarket.cfg index 5f740b33..cb276331 100644 --- a/joinmarket.cfg +++ b/joinmarket.cfg @@ -2,7 +2,7 @@ blockchain_source = blockr #options: blockr, json-rpc, regtest #before using json-rpc read https://github.com/chris-belcher/joinmarket/wiki/Running-JoinMarket-with-Bitcoin-Core-full-node -network = testnet +network = mainnet bitcoin_cli_cmd = bitcoin-cli [MESSAGING] diff --git a/wallet-tool.py b/wallet-tool.py index 6fede164..a74c592d 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -20,7 +20,7 @@ # m/0/n/1/k kth change address, for mixing depth n -parser = OptionParser(usage='usage: %prog [options] [seed / wallet file] [method]', +parser = OptionParser(usage='usage: %prog [options] [wallet file] [method]', description='Does useful little tasks involving your bip32 wallet. The' + ' method is one of the following: display- shows addresses and balances.' + ' displayall - shows ALL addresses and balances.' @@ -40,7 +40,7 @@ methods = ['display', 'displayall', 'summary'] + noseed_methods if len(args) < 1: - parser.error('Needs a seed, wallet file or method') + parser.error('Needs a wallet file or method') sys.exit(0) load_program_config() From fe535a9b8b086b5f9fd86948f3b98a23caa6de5a Mon Sep 17 00:00:00 2001 From: Emilian Ursu Date: Thu, 7 May 2015 06:14:36 -0400 Subject: [PATCH 242/409] Implemented socks5 proxy --- joinmarket.cfg | 3 +++ lib/irc.py | 13 ++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/joinmarket.cfg b/joinmarket.cfg index cb276331..76a1fa68 100644 --- a/joinmarket.cfg +++ b/joinmarket.cfg @@ -10,4 +10,7 @@ host = irc.snoonet.org channel = joinmarket-pit port = 6697 usessl = true +socks5 = true +socks5_host = 127.0.0.1 +socks5_port = 1500 #More stuff to come diff --git a/lib/irc.py b/lib/irc.py index 8650816d..5d4134ab 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -3,7 +3,7 @@ from message_channel import MessageChannel from message_channel import CJPeerError -import socket, threading, time, ssl +import socket, threading, time, ssl, socks import base64, os, urllib2, re import enc_wrapper @@ -431,6 +431,8 @@ def __init__(self, given_nick, username='username', realname='realname', passwor self.given_nick = given_nick self.nick = given_nick self.serverport = (config.get("MESSAGING","host"), int(config.get("MESSAGING","port"))) + self.socks5_host = config.get("MESSAGING","socks5_host") + self.socks5_port = int(config.get("MESSAGING","socks5_port")) self.channel = get_config_irc_channel() self.userrealname = (username, realname) if password and len(password) == 0: @@ -449,11 +451,16 @@ def run(self): while self.connect_attempts < 10 and not self.give_up: try: debug('connecting') - self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if config.get("MESSAGING","socks5").lower() == 'true': + debug("Using socks5 proxy %s:%s" % (self.socks5_host, self.socks5_port)) + socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, self.socks5_host, self.socks5_port, True) + self.sock = socks.socksocket() + else: + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if config.get("MESSAGING","usessl").lower() == 'true': self.sock = ssl.wrap_socket(self.sock) - self.sock.connect(self.serverport) self.fd = self.sock.makefile() + self.sock.connect(self.serverport) if self.password: self.send_raw('CAP REQ :sasl') self.send_raw('USER %s b c :%s' % self.userrealname) From 45ca1cd17562714c0d3f53ce7baa9f4c41d100c8 Mon Sep 17 00:00:00 2001 From: Belcher Date: Thu, 7 May 2015 21:34:47 +0100 Subject: [PATCH 243/409] yield generator takes into account dust_threshold when announcing order maxsize --- yield-generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yield-generator.py b/yield-generator.py index 3875e133..a7e0f955 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -60,7 +60,7 @@ def create_my_orders(self): #print mix_balance max_mix = max(mix_balance, key=mix_balance.get) order = {'oid': 0, 'ordertype': 'relorder', 'minsize': minsize, - 'maxsize': mix_balance[max_mix], 'txfee': txfee, 'cjfee': cjfee} + 'maxsize': mix_balance[max_mix] - common.DUST_THRESHOLD, 'txfee': txfee, 'cjfee': cjfee} return [order] def oid_to_order(self, cjorder, oid, amount): From 8e240f0c17dd3f37177becb1a8e2bb9954f5f712 Mon Sep 17 00:00:00 2001 From: Belcher Date: Thu, 7 May 2015 23:10:30 +0100 Subject: [PATCH 244/409] added nicer error message when not enough liquidity --- sendpayment.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sendpayment.py b/sendpayment.py index 777b589f..04503022 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -42,6 +42,9 @@ def run(self): self.taker.destaddr, None, self.taker.txfee, self.finishcallback) else: orders, total_cj_fee = choose_order(self.taker.db, self.taker.amount, self.taker.makercount) + if not orders: + debug('ERROR not enough liquidity in the orderbook, exiting') + return print 'chosen orders to fill ' + str(orders) + ' totalcjfee=' + str(total_cj_fee) total_amount = self.taker.amount + total_cj_fee + self.taker.txfee print 'total amount spent = ' + str(total_amount) From c2f32c84aaeec47148e7bfa530a90fc8d40f318b Mon Sep 17 00:00:00 2001 From: Belcher Date: Thu, 7 May 2015 23:11:48 +0100 Subject: [PATCH 245/409] slightly clearer error message --- lib/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/common.py b/lib/common.py index b72eff92..33ca53a9 100644 --- a/lib/common.py +++ b/lib/common.py @@ -280,7 +280,7 @@ def choose_order(db, cj_amount, n): for o in sqlorders if cj_amount >= o['minsize'] and cj_amount <= o['maxsize']] counterparties = set([o[0] for o in orders]) if n > len(counterparties): - debug('ERROR not enough liquidity in the orderbook n=%d counterparties=%d' + debug('ERROR not enough liquidity in the orderbook n=%d suitable-counterparties=%d' % (n, len(counterparties))) return None, 0 #TODO handle not enough liquidity better, maybe an Exception orders = sorted(orders, key=lambda k: k[2]) From 05be93d0df5c77f14d8d9f6599f2d7f54e3021fb Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 8 May 2015 00:08:48 +0100 Subject: [PATCH 246/409] added donation address to readme --- README.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.txt b/README.txt index 95cdb602..db747dbc 100644 --- a/README.txt +++ b/README.txt @@ -1,4 +1,3 @@ - IRC Channel: #joinmarket on irc.freenode.net https://webchat.freenode.net/?channels=%23joinmarket @@ -12,6 +11,9 @@ www.reddit.com/r/joinmarket Twitter: www.twitter.com/joinmarket +Donation address: +1AZgQZWYRteh6UyF87hwuvyWj73NvWKpL + Wiki page for more detailed articles: https://github.com/chris-belcher/joinmarket/wiki From 93dfad116f05ba3adb9d991818d8f3aa799a5709 Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 8 May 2015 00:38:16 +0100 Subject: [PATCH 247/409] added protection against coinjoin amounts below dust threshold --- lib/maker.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/maker.py b/lib/maker.py index 43392fda..41b40e0c 100644 --- a/lib/maker.py +++ b/lib/maker.py @@ -12,6 +12,8 @@ def __init__(self, maker, nick, oid, amount, taker_pk): self.maker = maker self.oid = oid self.cj_amount = amount + if self.cj_amount <= common.DUST_THRESHOLD: + self.maker.msgchan.send_error(nick, 'amount below dust threshold') #the btc pubkey of the utxo that the taker plans to use as input self.taker_pk = taker_pk #create DH keypair on the fly for this Order object From 6b3a945d027015991c1c161448af88ad0a9f00fd Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 8 May 2015 22:08:10 +0100 Subject: [PATCH 248/409] changed name of button in ob-watcher.py html to reduce confusion --- ob-watcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ob-watcher.py b/ob-watcher.py index 252acda3..e5113bab 100644 --- a/ob-watcher.py +++ b/ob-watcher.py @@ -26,7 +26,7 @@ shutdownform = '
    ' shutdownpage = '

    Successfully Shut down

    ' -refresh_orderbook_form = '
    ' +refresh_orderbook_form = '
    ' def calc_depth_data(db, value): pass From d53bc4e9acb703430fc429a7463da1ed4cd7a8d3 Mon Sep 17 00:00:00 2001 From: Belcher Date: Fri, 8 May 2015 23:28:30 +0100 Subject: [PATCH 249/409] added --pre to instructions for pip install --- README.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.txt b/README.txt index db747dbc..df7474ab 100644 --- a/README.txt +++ b/README.txt @@ -24,7 +24,7 @@ INSTALLING use this line to check it was installed correctly python lib/enc_wrapper.py 2. You will need slowaes installed for encrypting your wallet - sudo pip install slowaes + sudo pip install --pre slowaes 3. you will need numpy 1.7 or later installed 4. (optional) matplotlib for displaying the graphs in orderbook-watcher From a068d706e6e93bfba28ec30c723f4f0c1fa2d5fe Mon Sep 17 00:00:00 2001 From: Belcher Date: Sun, 10 May 2015 00:49:29 +0100 Subject: [PATCH 250/409] sendpayment now displays chosen orders and asks for confirmation before starting, --yes will skip --- sendpayment.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/sendpayment.py b/sendpayment.py index 04503022..a6d5b1f2 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -38,6 +38,10 @@ def run(self): utxo_list = self.taker.wallet.get_utxos_by_mixdepth()[self.taker.mixdepth] total_value = sum([va['value'] for va in utxo_list.values()]) orders, cjamount = choose_sweep_order(self.taker.db, total_value, self.taker.txfee, self.taker.makercount) + if not self.taker.answeryes: + if raw_input('send with these orders? (y/n):')[0] != 'y': + self.finishcallback(None) + return self.taker.start_cj(self.taker.wallet, cjamount, orders, utxo_list, self.taker.destaddr, None, self.taker.txfee, self.finishcallback) else: @@ -46,6 +50,10 @@ def run(self): debug('ERROR not enough liquidity in the orderbook, exiting') return print 'chosen orders to fill ' + str(orders) + ' totalcjfee=' + str(total_cj_fee) + if not self.taker.answeryes: + if raw_input('send with these orders? (y/n):')[0] != 'y': + self.finishcallback(None) + return total_amount = self.taker.amount + total_cj_fee + self.taker.txfee print 'total amount spent = ' + str(total_amount) @@ -55,7 +63,7 @@ def run(self): self.finishcallback) class SendPayment(takermodule.Taker): - def __init__(self, msgchan, wallet, destaddr, amount, makercount, txfee, waittime, mixdepth): + def __init__(self, msgchan, wallet, destaddr, amount, makercount, txfee, waittime, mixdepth, answeryes): takermodule.Taker.__init__(self, msgchan) self.wallet = wallet self.destaddr = destaddr @@ -64,6 +72,7 @@ def __init__(self, msgchan, wallet, destaddr, amount, makercount, txfee, waittim self.txfee = txfee self.waittime = waittime self.mixdepth = mixdepth + self.answeryes = answeryes def on_welcome(self): takermodule.Taker.on_welcome(self) @@ -82,6 +91,8 @@ def main(): help='how many makers to coinjoin with, default=2', default=2) parser.add_option('-m', '--mixdepth', action='store', type='int', dest='mixdepth', help='mixing depth to spend from, default=0', default=0) + parser.add_option('--yes', action='store_true', dest='answeryes', default=False, + help='answer yes to everything') (options, args) = parser.parse_args() if len(args) < 3: @@ -107,7 +118,7 @@ def main(): irc = IRCMessageChannel(common.nickname) taker = SendPayment(irc, wallet, destaddr, amount, options.makercount, options.txfee, - options.waittime, options.mixdepth) + options.waittime, options.mixdepth, options.answeryes) try: debug('starting irc') irc.run() From e74c384beb21755988afe46f941106efd0695a42 Mon Sep 17 00:00:00 2001 From: Belcher Date: Tue, 12 May 2015 00:43:14 +0100 Subject: [PATCH 251/409] bug fix, would crash on unconfirmed or already spent utxos instead of sending error to taker --- lib/maker.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/maker.py b/lib/maker.py index 41b40e0c..cc0eacbe 100644 --- a/lib/maker.py +++ b/lib/maker.py @@ -124,6 +124,8 @@ def verify_unsigned_tx(self, txd): + str(ins['outpoint']['index']) for ins in txd['ins']]) #complete authentication: check the tx input uses the authing pubkey input_utxo_data = common.bc_interface.query_utxo_set(list(tx_utxo_set)) + if None in input_utxo_data: + return False, 'some utxos already spent or not confirmed yet' input_addresses = [u['address'] for u in input_utxo_data] if btc.pubtoaddr(self.i_utxo_pubkey, get_addr_vbyte())\ not in input_addresses: From 6fdc00d815880cebdf16ebd22bcc29ab3653826b Mon Sep 17 00:00:00 2001 From: Belcher Date: Wed, 13 May 2015 02:24:23 +0100 Subject: [PATCH 252/409] doesnt crash on /move/ transactions in the internal bitcoin core accounts --- lib/blockchaininterface.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/blockchaininterface.py b/lib/blockchaininterface.py index 25430eea..34d6d8c9 100644 --- a/lib/blockchaininterface.py +++ b/lib/blockchaininterface.py @@ -289,6 +289,10 @@ def do_HEAD(self): txdata = json.loads(self.btcinterface.rpc(['gettxout', txid, '0', 'true'])) if txdata['confirmations'] == 0: unconfirmfun(txd, txid) + #TODO pass the total transfered amount value here somehow + #wallet_name = self.get_wallet_name() + #amount = + #bitcoin-cli move wallet_name "" amount common.debug('ran unconfirmfun') else: confirmfun(txd, txid, txdata['confirmations']) @@ -338,6 +342,9 @@ def __init__(self, bitcoin_cli_cmd, testnet = False): self.notifythread = None self.txnotify_fun = [] + def get_wallet_name(self): + return 'joinmarket-wallet-' + btc.dbl_sha256(wallet.keys[0][0])[:6] + def rpc(self, args): try: if args[0] != 'importaddress': @@ -356,7 +363,7 @@ def add_watchonly_addresses(self, addr_list, wallet_name): def sync_addresses(self, wallet, gaplimit=6): common.debug('requesting wallet history') - wallet_name = 'joinmarket-wallet-' + btc.dbl_sha256(wallet.keys[0][0])[:6] + wallet_name = self.get_wallet_name() addr_req_count = 50 wallet_addr_list = [] for mix_depth in range(wallet.max_mix_depth): @@ -373,7 +380,7 @@ def sync_addresses(self, wallet, gaplimit=6): txs = json.loads(ret) if len(txs) == 1000: raise Exception('time to stop putting off this bug and actually fix it, see the TODO') - used_addr_list = [tx['address'] for tx in txs] + used_addr_list = [tx['address'] for tx in txs if tx['category'] == 'receive'] too_few_addr_mix_change = [] for mix_depth in range(wallet.max_mix_depth): for forchange in [0, 1]: @@ -412,7 +419,7 @@ def sync_addresses(self, wallet, gaplimit=6): def sync_unspent(self, wallet): st = time.time() - wallet_name = 'joinmarket-wallet-' + btc.dbl_sha256(wallet.keys[0][0])[:6] + wallet_name = self.get_wallet_name() wallet.unspent = {} unspent_list = json.loads(self.rpc(['listunspent'])) for u in unspent_list: From 3b89d91fcbcbc0ca9c3add8ce5fae9d5baa14c5a Mon Sep 17 00:00:00 2001 From: Belcher Date: Wed, 13 May 2015 02:34:26 +0100 Subject: [PATCH 253/409] threading lock the debug file writes --- lib/common.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/lib/common.py b/lib/common.py index 33ca53a9..7529b1f6 100644 --- a/lib/common.py +++ b/lib/common.py @@ -13,6 +13,8 @@ DUST_THRESHOLD = 543 bc_interface = None ordername_list = ["absorder", "relorder"] + +debug_file_lock = threading.Lock() debug_file_handle = None core_alert = None joinmarket_alert = None @@ -50,16 +52,17 @@ def get_config_irc_channel(): def debug(msg): global debug_file_handle - if nickname and not debug_file_handle: - debug_file_handle = open(nickname+'.log','ab') - outmsg = datetime.datetime.now().strftime("[%Y/%m/%d %H:%M:%S] ") + msg - if core_alert: - print 'Core Alert Message: ' + core_alert - if joinmarket_alert: - print 'JoinMarket Alert Message: ' + joinmarket_alert - print outmsg - if nickname: #debugs before creating bot nick won't be handled like this - debug_file_handle.write(outmsg + '\r\n') + with debug_file_lock: + if nickname and not debug_file_handle: + debug_file_handle = open(nickname+'.log','ab') + outmsg = datetime.datetime.now().strftime("[%Y/%m/%d %H:%M:%S] ") + msg + if core_alert: + print 'Core Alert Message: ' + core_alert + if joinmarket_alert: + print 'JoinMarket Alert Message: ' + joinmarket_alert + print outmsg + if nickname: #debugs before creating bot nick won't be handled like this + debug_file_handle.write(outmsg + '\r\n') def chunks(d, n): return [d[x: x+n] for x in xrange(0, len(d), n)] From 7a179d05eefc31e7c679908e32d16ee1f5cf8fc2 Mon Sep 17 00:00:00 2001 From: Belcher Date: Wed, 13 May 2015 02:36:20 +0100 Subject: [PATCH 254/409] socks5 proxy off by default --- joinmarket.cfg | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/joinmarket.cfg b/joinmarket.cfg index 76a1fa68..7b34a37d 100644 --- a/joinmarket.cfg +++ b/joinmarket.cfg @@ -10,7 +10,6 @@ host = irc.snoonet.org channel = joinmarket-pit port = 6697 usessl = true -socks5 = true +socks5 = false socks5_host = 127.0.0.1 -socks5_port = 1500 -#More stuff to come +socks5_port = 9150 From e6739bdcc4a4040f751e3b01ea46d7c0df9febc2 Mon Sep 17 00:00:00 2001 From: Belcher Date: Wed, 13 May 2015 03:06:01 +0100 Subject: [PATCH 255/409] bugfix with other commit, whoops --- lib/blockchaininterface.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/blockchaininterface.py b/lib/blockchaininterface.py index 34d6d8c9..67583583 100644 --- a/lib/blockchaininterface.py +++ b/lib/blockchaininterface.py @@ -342,7 +342,7 @@ def __init__(self, bitcoin_cli_cmd, testnet = False): self.notifythread = None self.txnotify_fun = [] - def get_wallet_name(self): + def get_wallet_name(self, wallet): return 'joinmarket-wallet-' + btc.dbl_sha256(wallet.keys[0][0])[:6] def rpc(self, args): @@ -363,7 +363,7 @@ def add_watchonly_addresses(self, addr_list, wallet_name): def sync_addresses(self, wallet, gaplimit=6): common.debug('requesting wallet history') - wallet_name = self.get_wallet_name() + wallet_name = self.get_wallet_name(wallet) addr_req_count = 50 wallet_addr_list = [] for mix_depth in range(wallet.max_mix_depth): @@ -419,7 +419,7 @@ def sync_addresses(self, wallet, gaplimit=6): def sync_unspent(self, wallet): st = time.time() - wallet_name = self.get_wallet_name() + wallet_name = self.get_wallet_name(wallet) wallet.unspent = {} unspent_list = json.loads(self.rpc(['listunspent'])) for u in unspent_list: From 9fce906116b9a6bc78320906d6d0aa920ebc3376 Mon Sep 17 00:00:00 2001 From: Belcher Date: Wed, 13 May 2015 21:04:24 +0100 Subject: [PATCH 256/409] fixed bug in socks5 debug line --- lib/irc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/irc.py b/lib/irc.py index 5d4134ab..2709ddb9 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -452,7 +452,7 @@ def run(self): try: debug('connecting') if config.get("MESSAGING","socks5").lower() == 'true': - debug("Using socks5 proxy %s:%s" % (self.socks5_host, self.socks5_port)) + debug("Using socks5 proxy %s:%d" % (self.socks5_host, self.socks5_port)) socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, self.socks5_host, self.socks5_port, True) self.sock = socks.socksocket() else: From d02167fe9ba6515aac56be6cdda96695ec0690db Mon Sep 17 00:00:00 2001 From: Belcher Date: Wed, 13 May 2015 21:39:37 +0100 Subject: [PATCH 257/409] failed pushtx error is handled correctly instead of crashing the program --- lib/blockchaininterface.py | 16 ++++++++++++---- lib/maker.py | 2 ++ lib/taker.py | 6 ++++-- sendpayment.py | 1 - 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/lib/blockchaininterface.py b/lib/blockchaininterface.py index 67583583..3d3f12dd 100644 --- a/lib/blockchaininterface.py +++ b/lib/blockchaininterface.py @@ -48,7 +48,7 @@ def add_tx_notify(self, txd, unconfirmfun, confirmfun, notifyaddr): @abc.abstractmethod def pushtx(self, txhex): - '''pushes tx to the network, returns txhash''' + '''pushes tx to the network, returns txhash, or None if failed''' pass @abc.abstractmethod @@ -227,9 +227,13 @@ def run(self): NotifyThread(self.blockr_domain, txd, unconfirmfun, confirmfun).start() def pushtx(self, txhex): - data = json.loads(btc.blockr_pushtx(txhex, self.network)) + try: + json_str = btc.blockr_pushtx(txhex, self.network) + except Exception: + common.debug('failed blockr.io pushtx') + return None + data = json.loads(json_str) if data['status'] != 'success': - #error message generally useless so there might not be a point returning common.debug(data) return None return data['data'] @@ -450,7 +454,11 @@ def add_tx_notify(self, txd, unconfirmfun, confirmfun, notifyaddr): self.txnotify_fun.append((tx_output_set, unconfirmfun, confirmfun)) def pushtx(self, txhex): - return self.rpc(['sendrawtransaction', txhex]).strip() + try: + return self.rpc(['sendrawtransaction', txhex]).strip() + except subprocess.CalledProcessError, e: + common.debug('failed pushtx, error ' + repr(e)) + return None def query_utxo_set(self, txout): if not isinstance(txout, list): diff --git a/lib/maker.py b/lib/maker.py index cc0eacbe..3485846d 100644 --- a/lib/maker.py +++ b/lib/maker.py @@ -215,6 +215,8 @@ def on_push_tx(self, nick, txhex): debug('received txhex from ' + nick + ' to push\n' + txhex) txid = common.bc_interface.pushtx(txhex) debug('pushed tx ' + str(txid)) + if txid == None: + self.send_error(nick, 'Unable to push tx') def on_welcome(self): self.msgchan.announce_orders(self.orderlist) diff --git a/lib/taker.py b/lib/taker.py index f614fbb6..72669bec 100644 --- a/lib/taker.py +++ b/lib/taker.py @@ -151,8 +151,10 @@ def add_signature(self, sigb64): #TODO send to a random maker or push myself #self.msgchan.push_tx(self.active_orders.keys()[0], txhex) - txid = common.bc_interface.pushtx(txhex) - debug('pushed tx ' + str(txid)) + self.txid = common.bc_interface.pushtx(txhex) + debug('pushed tx ' + str(self.txid)) + if self.txid == None: + debug('unable to pushtx') if self.finishcallback != None: self.finishcallback(self) diff --git a/sendpayment.py b/sendpayment.py index a6d5b1f2..64e496e0 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -110,7 +110,6 @@ def main(): common.nickname = random_nick() debug('starting sendpayment') - import binascii, os wallet = Wallet(seed, options.mixdepth + 1) common.bc_interface.sync_wallet(wallet) From 8638268ca48af4b45321bd5d2d6b759994366b3a Mon Sep 17 00:00:00 2001 From: Anduck Date: Tue, 12 May 2015 02:46:21 +0300 Subject: [PATCH 258/409] add options to choose cheapest orders or manually pick orders (only works while not sweeping) --- lib/common.py | 36 +++++++++++++++++++++++++++++++----- sendpayment.py | 26 ++++++++++++++++++++++---- 2 files changed, 53 insertions(+), 9 deletions(-) diff --git a/lib/common.py b/lib/common.py index 33ca53a9..13da902c 100644 --- a/lib/common.py +++ b/lib/common.py @@ -274,7 +274,32 @@ def weighted_order_choose(orders, n, feekey): chosen_order_index = np.random.choice(len(orders), p=weight) return orders[chosen_order_index] -def choose_order(db, cj_amount, n): +def cheapest_order_choose(orders, n, feekey): + ''' + Return the cheapest order from the orders. + ''' + return sorted(orders, key=feekey)[0] + +def pick_order(orders, n, feekey): + i = -1 + print("Considered orders:"); + for o in orders: + i+=1 + print(" "+str(i)+". "+str(o[0])+", CJ fee: "+str(o[2])) + pickedOrderIndex = -1 + if i==0: + print("Only one possible pick, picking it.") + return orders[0] + while pickedOrderIndex == -1: + pickedOrderIndex = raw_input('Pick an order between 0 and '+str(i)+': ') + if pickedOrderIndex!='' and pickedOrderIndex[0].isdigit(): + pickedOrderIndex = int(pickedOrderIndex[0]) + if pickedOrderIndex>=0 and pickedOrderIndex= o['minsize'] and cj_amount <= o['maxsize']] @@ -283,12 +308,12 @@ def choose_order(db, cj_amount, n): debug('ERROR not enough liquidity in the orderbook n=%d suitable-counterparties=%d' % (n, len(counterparties))) return None, 0 #TODO handle not enough liquidity better, maybe an Exception - orders = sorted(orders, key=lambda k: k[2]) + orders = sorted(orders, key=lambda k: k[2]) #sort from smallest to biggest cj fee debug('considered orders = ' + str(orders)) total_cj_fee = 0 chosen_orders = [] for i in range(n): - chosen_order = weighted_order_choose(orders, n, lambda k: k[2]) + chosen_order = chooseOrdersBy(orders, n, lambda k: k[2]) orders = [o for o in orders if o[0] != chosen_order[0]] #remove all orders from that same counterparty chosen_orders.append(chosen_order) total_cj_fee += chosen_order[2] @@ -330,7 +355,7 @@ def create_combination(li, n): assert len(result) == nCk(len(li), n) return result -def choose_sweep_order(db, my_total_input, my_tx_fee, n): +def choose_sweep_order(db, my_total_input, my_tx_fee, n, chooseOrdersBy): ''' choose an order given that we want to be left with no change i.e. sweep an entire group of utxos @@ -377,7 +402,8 @@ def is_amount_in_range(ordercombo, cjamount): if len(ordercombos) == 0: debug('ERROR not enough liquidity in the orderbook') return None, 0 #TODO handle not enough liquidity better, maybe an Exception - ordercombo = weighted_order_choose(ordercombos, n, lambda k: k[1][1]) + + ordercombo = chooseOrdersBy(ordercombos, n, lambda k: k[1][1]) #index [1][1] = cjfee orders = dict([(o['counterparty'], o['oid']) for o in ordercombo[0]]) cjamount = ordercombo[1][0] debug('chosen orders = ' + str(orders)) diff --git a/sendpayment.py b/sendpayment.py index a6d5b1f2..ab30351b 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -37,7 +37,12 @@ def run(self): if self.taker.amount == 0: utxo_list = self.taker.wallet.get_utxos_by_mixdepth()[self.taker.mixdepth] total_value = sum([va['value'] for va in utxo_list.values()]) - orders, cjamount = choose_sweep_order(self.taker.db, total_value, self.taker.txfee, self.taker.makercount) + if self.taker.choosecheapest: #choose cheapest + chooseOrdersBy = cheapest_order_choose + else: #choose randomly (weighted) + chooseOrdersBy = weighted_order_choose + + orders, cjamount = choose_sweep_order(self.taker.db, total_value, self.taker.txfee, self.taker.makercount, chooseOrdersBy) if not self.taker.answeryes: if raw_input('send with these orders? (y/n):')[0] != 'y': self.finishcallback(None) @@ -45,7 +50,14 @@ def run(self): self.taker.start_cj(self.taker.wallet, cjamount, orders, utxo_list, self.taker.destaddr, None, self.taker.txfee, self.finishcallback) else: - orders, total_cj_fee = choose_order(self.taker.db, self.taker.amount, self.taker.makercount) + if self.taker.pickorders: #pick orders manually + chooseOrdersBy = pick_order + elif self.taker.choosecheapest: #choose cheapest + chooseOrdersBy = cheapest_order_choose + else: #choose randomly (weighted) + chooseOrdersBy = weighted_order_choose + + orders, total_cj_fee = choose_order(self.taker.db, self.taker.amount, self.taker.makercount, chooseOrdersBy) if not orders: debug('ERROR not enough liquidity in the orderbook, exiting') return @@ -63,7 +75,7 @@ def run(self): self.finishcallback) class SendPayment(takermodule.Taker): - def __init__(self, msgchan, wallet, destaddr, amount, makercount, txfee, waittime, mixdepth, answeryes): + def __init__(self, msgchan, wallet, destaddr, amount, makercount, txfee, waittime, mixdepth, answeryes, choosecheapest, pickorders): takermodule.Taker.__init__(self, msgchan) self.wallet = wallet self.destaddr = destaddr @@ -73,6 +85,8 @@ def __init__(self, msgchan, wallet, destaddr, amount, makercount, txfee, waittim self.waittime = waittime self.mixdepth = mixdepth self.answeryes = answeryes + self.choosecheapest = choosecheapest + self.pickorders = pickorders def on_welcome(self): takermodule.Taker.on_welcome(self) @@ -89,6 +103,10 @@ def main(): help='wait time in seconds to allow orders to arrive, default=5', default=5) parser.add_option('-N', '--makercount', action='store', type='int', dest='makercount', help='how many makers to coinjoin with, default=2', default=2) + parser.add_option('-C','--choose-cheapest', action='store_true', dest='choosecheapest', default=False, + help='override weightened offers picking and choose cheapest') + parser.add_option('-P','--pick-orders', action='store_true', dest='pickorders', default=False, + help='manually pick which orders to take. doesn\'t work while sweeping.') parser.add_option('-m', '--mixdepth', action='store', type='int', dest='mixdepth', help='mixing depth to spend from, default=0', default=0) parser.add_option('--yes', action='store_true', dest='answeryes', default=False, @@ -118,7 +136,7 @@ def main(): irc = IRCMessageChannel(common.nickname) taker = SendPayment(irc, wallet, destaddr, amount, options.makercount, options.txfee, - options.waittime, options.mixdepth, options.answeryes) + options.waittime, options.mixdepth, options.answeryes, options.choosecheapest, options.pickorders) try: debug('starting irc') irc.run() From 326bb8d71a49e998b3caf3a4ece63f8c4bd31646 Mon Sep 17 00:00:00 2001 From: Belcher Date: Thu, 14 May 2015 00:01:36 +0100 Subject: [PATCH 259/409] tweaked order choosing parameters --- lib/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/common.py b/lib/common.py index 6e55ff2a..546f37bc 100644 --- a/lib/common.py +++ b/lib/common.py @@ -265,7 +265,7 @@ def weighted_order_choose(orders, n, feekey): unless M < orderbook size, then phi goes up to the last order ''' minfee = feekey(orders[0]) - M = 2*n + M = n if len(orders) > M: phi = feekey(orders[M]) - minfee else: From 074846eaae3aeb6dcd6ad0393bb13d5aa1a5a698 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Thu, 14 May 2015 14:33:24 +0300 Subject: [PATCH 260/409] option gaplimit type set to int --- wallet-tool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wallet-tool.py b/wallet-tool.py index a74c592d..3a36c640 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -32,7 +32,7 @@ help='print private key along with address, default false') parser.add_option('-m', '--maxmixdepth', action='store', type='int', dest='maxmixdepth', default=5, help='maximum mixing depth to look for, default=5') -parser.add_option('-g', '--gap-limit', action='store', dest='gaplimit', +parser.add_option('-g', '--gap-limit', type="int", action='store', dest='gaplimit', help='gap limit for wallet, default=6', default=6) (options, args) = parser.parse_args() From f1f64cae2a5a28da81ed96c62b823e08b7d5310e Mon Sep 17 00:00:00 2001 From: Adlai Chandrasekhar Date: Thu, 14 May 2015 13:18:51 +0300 Subject: [PATCH 261/409] separate directory for debug logs and income statement --- .gitignore | 1 - lib/common.py | 2 +- logs/.gitignore | 4 ++++ yield-generator.py | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 logs/.gitignore diff --git a/.gitignore b/.gitignore index bd95d0df..c9b568f7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ *.pyc *.swp -*.log diff --git a/lib/common.py b/lib/common.py index 546f37bc..4167e895 100644 --- a/lib/common.py +++ b/lib/common.py @@ -54,7 +54,7 @@ def debug(msg): global debug_file_handle with debug_file_lock: if nickname and not debug_file_handle: - debug_file_handle = open(nickname+'.log','ab') + debug_file_handle = open(os.path.join('logs', nickname+'.log'),'ab') outmsg = datetime.datetime.now().strftime("[%Y/%m/%d %H:%M:%S] ") + msg if core_alert: print 'Core Alert Message: ' + core_alert diff --git a/logs/.gitignore b/logs/.gitignore new file mode 100644 index 00000000..86d0cb27 --- /dev/null +++ b/logs/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore \ No newline at end of file diff --git a/yield-generator.py b/yield-generator.py index a7e0f955..22e29ba3 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -39,7 +39,7 @@ def __init__(self, msgchan, wallet): def log_statement(self, data): data = [str(d) for d in data] - self.income_statement = open('yield-generator-income-statement.csv', 'aw') + self.income_statement = open(os.path.join('logs', 'yigen-statement.csv'), 'aw') self.income_statement.write(','.join(data) + '\n') self.income_statement.close() From 157b0468f6a9c8f472ab957f560e6c62039ad2dd Mon Sep 17 00:00:00 2001 From: Adlai Chandrasekhar Date: Thu, 14 May 2015 13:11:00 +0300 Subject: [PATCH 262/409] delete empty keys from Maker.active_orders --- lib/maker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/maker.py b/lib/maker.py index 3485846d..0e62ea89 100644 --- a/lib/maker.py +++ b/lib/maker.py @@ -223,7 +223,8 @@ def on_welcome(self): self.active_orders = {} def on_nick_leave(self, nick): - self.active_orders[nick] = None + if nick in self.active_orders: + del self.active_orders[nick] def modify_orders(self, to_cancel, to_announce): debug('modifying orders. to_cancel=' + str(to_cancel) + '\nto_announce=' + str(to_announce)) From b613f1679ffd485b7096686d5fd2b3d5a22e00d5 Mon Sep 17 00:00:00 2001 From: Emilian Ursu Date: Fri, 15 May 2015 07:21:31 -0400 Subject: [PATCH 263/409] Sanity check for relorder params --- lib/irc.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/irc.py b/lib/irc.py index 2709ddb9..c94a14e6 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -179,6 +179,10 @@ def check_for_orders(self, nick, chunks): maxsize = chunks[3] txfee = chunks[4] cjfee = chunks[5] + if any(t < 0 for t in [oid, minsize, maxsize, txfee, cjfee]): + return + if minsize > maxsize: + return if self.on_order_seen: self.on_order_seen(counterparty, oid, ordertype, minsize, maxsize, txfee, cjfee) From cc7b77d414265dd3e4379bf04edc9ac9cad27fc9 Mon Sep 17 00:00:00 2001 From: Emilian Ursu Date: Fri, 15 May 2015 08:25:07 -0400 Subject: [PATCH 264/409] convert to float --- lib/irc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/irc.py b/lib/irc.py index c94a14e6..dd1ee823 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -179,9 +179,9 @@ def check_for_orders(self, nick, chunks): maxsize = chunks[3] txfee = chunks[4] cjfee = chunks[5] - if any(t < 0 for t in [oid, minsize, maxsize, txfee, cjfee]): + if any(float(t) < 0 for t in [oid, minsize, maxsize, txfee, cjfee]): return - if minsize > maxsize: + if float(minsize) > float(maxsize): return if self.on_order_seen: self.on_order_seen(counterparty, oid, From 99d94ab2f07c48aff4fbbaf02a0ae719a711ef11 Mon Sep 17 00:00:00 2001 From: Emilian Ursu Date: Sun, 17 May 2015 08:06:57 -0400 Subject: [PATCH 265/409] Reverted changes --- lib/irc.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/irc.py b/lib/irc.py index dd1ee823..2709ddb9 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -179,10 +179,6 @@ def check_for_orders(self, nick, chunks): maxsize = chunks[3] txfee = chunks[4] cjfee = chunks[5] - if any(float(t) < 0 for t in [oid, minsize, maxsize, txfee, cjfee]): - return - if float(minsize) > float(maxsize): - return if self.on_order_seen: self.on_order_seen(counterparty, oid, ordertype, minsize, maxsize, txfee, cjfee) From 0fe0168cc1e230dfa24abb881628b5ad0e8e7fcc Mon Sep 17 00:00:00 2001 From: Emilian Ursu Date: Sun, 17 May 2015 08:07:21 -0400 Subject: [PATCH 266/409] Added sanity checks in OrderbookWatch --- lib/taker.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/taker.py b/lib/taker.py index 72669bec..c64ff3d3 100644 --- a/lib/taker.py +++ b/lib/taker.py @@ -199,6 +199,21 @@ def __init__(self, msgchan): + "minsize INTEGER, maxsize INTEGER, txfee INTEGER, cjfee TEXT);") def on_order_seen(self, counterparty, oid, ordertype, minsize, maxsize, txfee, cjfee): + if int(oid) < 0 or int(oid) > sys.maxint: + debug("Got invalid order: " + oid + " from " + counterparty) + return + if int(minsize) < 0 or int(minsize) > 21*10**14: + debug("Got invalid minsize: " + minsize + " from " + counterparty) + return + if int(maxsize) < 0 or int(maxsize) > 21*10**14: + debug("Got invalid maxsize: " + maxsize + " from " + counterparty) + return + if int(txfee) < 0: + debug("Got invalid txfee: " + txfee + " from " + counterparty) + return + if int(minsize) > int(maxsize): + debug("Got minsize bigger than maxsize: " + minsize + " - " + maxsize + " from " + counterparty) + return self.db.execute("DELETE FROM orderbook WHERE counterparty=? AND oid=?;", (counterparty, oid)) self.db.execute('INSERT INTO orderbook VALUES(?, ?, ?, ?, ?, ?, ?);', From 3af58792945a8f01fa3260f295c497447450027a Mon Sep 17 00:00:00 2001 From: Emilian Ursu Date: Mon, 18 May 2015 02:17:43 -0400 Subject: [PATCH 267/409] Remove joinmarket.cfg / create default config at first run --- .gitignore | 1 + joinmarket.cfg | 15 --------------- lib/common.py | 22 ++++++++++++++++++++-- 3 files changed, 21 insertions(+), 17 deletions(-) delete mode 100644 joinmarket.cfg diff --git a/.gitignore b/.gitignore index c9b568f7..0c37968c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.pyc *.swp +joinmarket.cfg diff --git a/joinmarket.cfg b/joinmarket.cfg deleted file mode 100644 index 7b34a37d..00000000 --- a/joinmarket.cfg +++ /dev/null @@ -1,15 +0,0 @@ -[BLOCKCHAIN] -blockchain_source = blockr -#options: blockr, json-rpc, regtest -#before using json-rpc read https://github.com/chris-belcher/joinmarket/wiki/Running-JoinMarket-with-Bitcoin-Core-full-node -network = mainnet -bitcoin_cli_cmd = bitcoin-cli - -[MESSAGING] -host = irc.snoonet.org -channel = joinmarket-pit -port = 6697 -usessl = true -socks5 = false -socks5_host = 127.0.0.1 -socks5_port = 9150 diff --git a/lib/common.py b/lib/common.py index 4167e895..bef7d663 100644 --- a/lib/common.py +++ b/lib/common.py @@ -6,7 +6,7 @@ import numpy as np import blockchaininterface from ConfigParser import SafeConfigParser -import os +import os, io JM_VERSION = 1 nickname = '' @@ -29,7 +29,25 @@ def load_program_config(): #detailed sanity checking : #did the file exist? if len(loadedFiles) != 1: - raise Exception("Could not find config file: "+config_location) + defaultconfig = """ +[BLOCKCHAIN] +blockchain_source = blockr +network = mainnet +bitcoin_cli_cmd = bitcoin-cli + +[MESSAGING] +host = irc.snoonet.org +channel = joinmarket-pit +port = 6697 +usessl = true +socks5 = false +socks5_host = 127.0.0.1 +socks5_port = 9150 +""" + config.readfp(io.BytesIO(defaultconfig)) + with open(config_location, "wb") as configfile: + config.write(configfile) + #check for sections for s in required_options: if s not in config.sections(): From 0c2b473719b982d8a6baf0bb5780935da9329b54 Mon Sep 17 00:00:00 2001 From: Emilian Ursu Date: Mon, 18 May 2015 02:20:50 -0400 Subject: [PATCH 268/409] wrong branch --- lib/irc.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/irc.py b/lib/irc.py index c94a14e6..2709ddb9 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -179,10 +179,6 @@ def check_for_orders(self, nick, chunks): maxsize = chunks[3] txfee = chunks[4] cjfee = chunks[5] - if any(t < 0 for t in [oid, minsize, maxsize, txfee, cjfee]): - return - if minsize > maxsize: - return if self.on_order_seen: self.on_order_seen(counterparty, oid, ordertype, minsize, maxsize, txfee, cjfee) From 9eb66d7f2a0e0d0f6a82ebcdee7b8c8f44862842 Mon Sep 17 00:00:00 2001 From: Emilian Ursu Date: Mon, 18 May 2015 02:22:37 -0400 Subject: [PATCH 269/409] Update comment --- lib/common.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/common.py b/lib/common.py index bef7d663..bf386f6b 100644 --- a/lib/common.py +++ b/lib/common.py @@ -26,8 +26,7 @@ def load_program_config(): loadedFiles = config.read([config_location]) - #detailed sanity checking : - #did the file exist? + #Create default config file if not found if len(loadedFiles) != 1: defaultconfig = """ [BLOCKCHAIN] From b3dc7c7291a38183324175326dcf70eedadb3861 Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Tue, 19 May 2015 01:58:35 +0100 Subject: [PATCH 270/409] added update_cache_index() code, which stores the pointers of the addresses we're up to in the HD wallet --- lib/blockchaininterface.py | 26 ++++++++++++++------------ lib/common.py | 29 ++++++++++++++++++++++++++--- lib/maker.py | 1 + wallet-tool.py | 4 ++-- 4 files changed, 43 insertions(+), 17 deletions(-) diff --git a/lib/blockchaininterface.py b/lib/blockchaininterface.py index 3d3f12dd..558f00cb 100644 --- a/lib/blockchaininterface.py +++ b/lib/blockchaininterface.py @@ -27,12 +27,12 @@ class BlockchainInterface(object): def __init__(self): pass - def sync_wallet(self, wallet, gaplimit=6): - self.sync_addresses(wallet, gaplimit) + def sync_wallet(self, wallet): + self.sync_addresses(wallet) self.sync_unspent(wallet) @abc.abstractmethod - def sync_addresses(self, wallet, gaplimit=6): + def sync_addresses(self, wallet): '''Finds which addresses have been used and sets wallet.index appropriately''' pass @@ -69,28 +69,31 @@ def __init__(self, testnet = False): self.blockr_domain = 'tbtc' if testnet else 'btc' self.last_sync_unspent = 0 - def sync_addresses(self, wallet, gaplimit=6): + def sync_addresses(self, wallet): common.debug('downloading wallet history') #sets Wallet internal indexes to be at the next unused address for mix_depth in range(wallet.max_mix_depth): for forchange in [0, 1]: unused_addr_count = 0 last_used_addr = '' - while unused_addr_count < gaplimit: + while unused_addr_count < wallet.gaplimit or\ + wallet.index[mix_depth][forchange] <= wallet.index_cache[mix_depth][forchange]: addrs = [wallet.get_new_addr(mix_depth, forchange) for i in range(self.BLOCKR_MAX_ADDR_REQ_COUNT)] #TODO send a pull request to pybitcointools # because this surely should be possible with a function from it blockr_url = 'http://' + self.blockr_domain + '.blockr.io/api/v1/address/txs/' + #print 'downloading, lastusedaddr = ' + last_used_addr + ' unusedaddrcount= ' + str(unused_addr_count) res = btc.make_request(blockr_url+','.join(addrs)) data = json.loads(res)['data'] for dat in data: + #if forchange == 0: + # print ' nbtxs ' + str(dat['nb_txs']) + ' addr=' + dat['address'] + ' unused=' + str(unused_addr_count) if dat['nb_txs'] != 0: last_used_addr = dat['address'] + unused_addr_count = 0 else: unused_addr_count += 1 - if unused_addr_count >= gaplimit: - break if last_used_addr == '': wallet.index[mix_depth][forchange] = 0 else: @@ -365,7 +368,7 @@ def add_watchonly_addresses(self, addr_list, wallet_name): print 'now restart bitcoind with -rescan' sys.exit(0) - def sync_addresses(self, wallet, gaplimit=6): + def sync_addresses(self, wallet): common.debug('requesting wallet history') wallet_name = self.get_wallet_name(wallet) addr_req_count = 50 @@ -392,7 +395,8 @@ def sync_addresses(self, wallet, gaplimit=6): last_used_addr = '' breakloop = False while not breakloop: - if unused_addr_count >= gaplimit: + if unused_addr_count >= wallet.gaplimit and\ + wallet.index[mix_depth][forchange] >= wallet.index_cache[mix_depth][forchange]: break mix_change_addrs = [wallet.get_new_addr(mix_depth, forchange) for i in range(addr_req_count)] for mc_addr in mix_change_addrs: @@ -402,11 +406,9 @@ def sync_addresses(self, wallet, gaplimit=6): break if mc_addr in used_addr_list: last_used_addr = mc_addr + unused_addr_count = 0 else: unused_addr_count += 1 - if unused_addr_count >= gaplimit: - breakloop = True - break if last_used_addr == '': wallet.index[mix_depth][forchange] = 0 diff --git a/lib/common.py b/lib/common.py index 4167e895..96e4525b 100644 --- a/lib/common.py +++ b/lib/common.py @@ -101,10 +101,11 @@ def debug_dump_object(obj, skip_fields=[]): debug(str(v)) class Wallet(object): - def __init__(self, seedarg, max_mix_depth=2): + def __init__(self, seedarg, max_mix_depth=2, gaplimit=6): self.max_mix_depth = max_mix_depth - self.seed = self.get_seed(seedarg) - master = btc.bip32_master_key(self.seed) + self.gaplimit = gaplimit + seed = self.get_seed(seedarg) + master = btc.bip32_master_key(seed) m_0 = btc.bip32_ckd(master, 0) mixing_depth_keys = [btc.bip32_ckd(m_0, c) for c in range(max_mix_depth)] self.keys = [(btc.bip32_ckd(m, 0), btc.bip32_ckd(m, 1)) for m in mixing_depth_keys] @@ -123,6 +124,8 @@ def __init__(self, seedarg, max_mix_depth=2): self.spent_utxos = [] def get_seed(self, seedarg): + self.path = None + self.index_cache = [[0, 0]]*self.max_mix_depth path = os.path.join('wallets', seedarg) if not os.path.isfile(path): if get_network() == 'testnet': @@ -131,6 +134,7 @@ def get_seed(self, seedarg): else: raise IOError('wallet file not found') #debug('seedarg interpreted as wallet file name') + self.path = path try: import aes except ImportError: @@ -144,12 +148,30 @@ def get_seed(self, seedarg): print 'wallet network(%s) does not match joinmarket configured network(%s)' % ( walletdata['network'], get_network()) sys.exit(0) + if 'index_cache' in walletdata: + self.index_cache = walletdata['index_cache'] + print 'index cache = ' + str(self.index_cache) password = getpass.getpass('Enter wallet decryption passphrase: ') password_key = btc.bin_dbl_sha256(password) decrypted_seed = aes.decryptData(password_key, walletdata['encrypted_seed'] .decode('hex')).encode('hex') return decrypted_seed + def update_cache_index(self): + if not self.path: + return + if not os.path.isfile(self.path): + return + fd = open(self.path, 'r') + walletfile = fd.read() + fd.close() + walletdata = json.loads(walletfile) + walletdata['index_cache'] = self.index + walletfile = json.dumps(walletdata) + fd = open(self.path, 'w') + fd.write(walletfile) + fd.close() + def get_key(self, mixing_depth, forchange, i): return btc.bip32_extract_key(btc.bip32_ckd(self.keys[mixing_depth][forchange], i)) @@ -161,6 +183,7 @@ def get_new_addr(self, mixing_depth, forchange): addr = self.get_addr(mixing_depth, forchange, index[forchange]) self.addr_cache[addr] = (mixing_depth, forchange, index[forchange]) index[forchange] += 1 + #self.update_cache_index() return addr def get_receive_addr(self, mixing_depth): diff --git a/lib/maker.py b/lib/maker.py index 0e62ea89..b14d0719 100644 --- a/lib/maker.py +++ b/lib/maker.py @@ -33,6 +33,7 @@ def __init__(self, maker, nick, oid, amount, taker_pk): self.cjfee = order['cjfee'] debug('new cjorder nick=%s oid=%d amount=%d' % (nick, oid, amount)) self.utxos, self.cj_addr, self.change_addr = maker.oid_to_order(self, oid, amount) + self.maker.wallet.update_cache_index() if not self.utxos: self.maker.msgchan.send_error(nick, 'unable to fill order constrained by dust avoidance') #TODO make up orders offers in a way that this error cant appear diff --git a/wallet-tool.py b/wallet-tool.py index 3a36c640..7be78837 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -49,9 +49,9 @@ else: seed = args[0] method = ('display' if len(args) == 1 else args[1].lower()) - wallet = Wallet(seed, options.maxmixdepth) + wallet = Wallet(seed, options.maxmixdepth, options.gaplimit) if method != 'showseed': - common.bc_interface.sync_wallet(wallet, options.gaplimit) + common.bc_interface.sync_wallet(wallet) if method == 'display' or method == 'displayall': total_balance = 0 From cfe7d24472fd72cb5dad7f16b3d4b3bd3c2ae9e1 Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Tue, 19 May 2015 01:59:17 +0100 Subject: [PATCH 271/409] Revert "added update_cache_index() code, which stores the pointers of the addresses we're up to in the HD wallet" This reverts commit b3dc7c7291a38183324175326dcf70eedadb3861. --- lib/blockchaininterface.py | 26 ++++++++++++-------------- lib/common.py | 29 +++-------------------------- lib/maker.py | 1 - wallet-tool.py | 4 ++-- 4 files changed, 17 insertions(+), 43 deletions(-) diff --git a/lib/blockchaininterface.py b/lib/blockchaininterface.py index 558f00cb..3d3f12dd 100644 --- a/lib/blockchaininterface.py +++ b/lib/blockchaininterface.py @@ -27,12 +27,12 @@ class BlockchainInterface(object): def __init__(self): pass - def sync_wallet(self, wallet): - self.sync_addresses(wallet) + def sync_wallet(self, wallet, gaplimit=6): + self.sync_addresses(wallet, gaplimit) self.sync_unspent(wallet) @abc.abstractmethod - def sync_addresses(self, wallet): + def sync_addresses(self, wallet, gaplimit=6): '''Finds which addresses have been used and sets wallet.index appropriately''' pass @@ -69,31 +69,28 @@ def __init__(self, testnet = False): self.blockr_domain = 'tbtc' if testnet else 'btc' self.last_sync_unspent = 0 - def sync_addresses(self, wallet): + def sync_addresses(self, wallet, gaplimit=6): common.debug('downloading wallet history') #sets Wallet internal indexes to be at the next unused address for mix_depth in range(wallet.max_mix_depth): for forchange in [0, 1]: unused_addr_count = 0 last_used_addr = '' - while unused_addr_count < wallet.gaplimit or\ - wallet.index[mix_depth][forchange] <= wallet.index_cache[mix_depth][forchange]: + while unused_addr_count < gaplimit: addrs = [wallet.get_new_addr(mix_depth, forchange) for i in range(self.BLOCKR_MAX_ADDR_REQ_COUNT)] #TODO send a pull request to pybitcointools # because this surely should be possible with a function from it blockr_url = 'http://' + self.blockr_domain + '.blockr.io/api/v1/address/txs/' - #print 'downloading, lastusedaddr = ' + last_used_addr + ' unusedaddrcount= ' + str(unused_addr_count) res = btc.make_request(blockr_url+','.join(addrs)) data = json.loads(res)['data'] for dat in data: - #if forchange == 0: - # print ' nbtxs ' + str(dat['nb_txs']) + ' addr=' + dat['address'] + ' unused=' + str(unused_addr_count) if dat['nb_txs'] != 0: last_used_addr = dat['address'] - unused_addr_count = 0 else: unused_addr_count += 1 + if unused_addr_count >= gaplimit: + break if last_used_addr == '': wallet.index[mix_depth][forchange] = 0 else: @@ -368,7 +365,7 @@ def add_watchonly_addresses(self, addr_list, wallet_name): print 'now restart bitcoind with -rescan' sys.exit(0) - def sync_addresses(self, wallet): + def sync_addresses(self, wallet, gaplimit=6): common.debug('requesting wallet history') wallet_name = self.get_wallet_name(wallet) addr_req_count = 50 @@ -395,8 +392,7 @@ def sync_addresses(self, wallet): last_used_addr = '' breakloop = False while not breakloop: - if unused_addr_count >= wallet.gaplimit and\ - wallet.index[mix_depth][forchange] >= wallet.index_cache[mix_depth][forchange]: + if unused_addr_count >= gaplimit: break mix_change_addrs = [wallet.get_new_addr(mix_depth, forchange) for i in range(addr_req_count)] for mc_addr in mix_change_addrs: @@ -406,9 +402,11 @@ def sync_addresses(self, wallet): break if mc_addr in used_addr_list: last_used_addr = mc_addr - unused_addr_count = 0 else: unused_addr_count += 1 + if unused_addr_count >= gaplimit: + breakloop = True + break if last_used_addr == '': wallet.index[mix_depth][forchange] = 0 diff --git a/lib/common.py b/lib/common.py index 96e4525b..4167e895 100644 --- a/lib/common.py +++ b/lib/common.py @@ -101,11 +101,10 @@ def debug_dump_object(obj, skip_fields=[]): debug(str(v)) class Wallet(object): - def __init__(self, seedarg, max_mix_depth=2, gaplimit=6): + def __init__(self, seedarg, max_mix_depth=2): self.max_mix_depth = max_mix_depth - self.gaplimit = gaplimit - seed = self.get_seed(seedarg) - master = btc.bip32_master_key(seed) + self.seed = self.get_seed(seedarg) + master = btc.bip32_master_key(self.seed) m_0 = btc.bip32_ckd(master, 0) mixing_depth_keys = [btc.bip32_ckd(m_0, c) for c in range(max_mix_depth)] self.keys = [(btc.bip32_ckd(m, 0), btc.bip32_ckd(m, 1)) for m in mixing_depth_keys] @@ -124,8 +123,6 @@ def __init__(self, seedarg, max_mix_depth=2, gaplimit=6): self.spent_utxos = [] def get_seed(self, seedarg): - self.path = None - self.index_cache = [[0, 0]]*self.max_mix_depth path = os.path.join('wallets', seedarg) if not os.path.isfile(path): if get_network() == 'testnet': @@ -134,7 +131,6 @@ def get_seed(self, seedarg): else: raise IOError('wallet file not found') #debug('seedarg interpreted as wallet file name') - self.path = path try: import aes except ImportError: @@ -148,30 +144,12 @@ def get_seed(self, seedarg): print 'wallet network(%s) does not match joinmarket configured network(%s)' % ( walletdata['network'], get_network()) sys.exit(0) - if 'index_cache' in walletdata: - self.index_cache = walletdata['index_cache'] - print 'index cache = ' + str(self.index_cache) password = getpass.getpass('Enter wallet decryption passphrase: ') password_key = btc.bin_dbl_sha256(password) decrypted_seed = aes.decryptData(password_key, walletdata['encrypted_seed'] .decode('hex')).encode('hex') return decrypted_seed - def update_cache_index(self): - if not self.path: - return - if not os.path.isfile(self.path): - return - fd = open(self.path, 'r') - walletfile = fd.read() - fd.close() - walletdata = json.loads(walletfile) - walletdata['index_cache'] = self.index - walletfile = json.dumps(walletdata) - fd = open(self.path, 'w') - fd.write(walletfile) - fd.close() - def get_key(self, mixing_depth, forchange, i): return btc.bip32_extract_key(btc.bip32_ckd(self.keys[mixing_depth][forchange], i)) @@ -183,7 +161,6 @@ def get_new_addr(self, mixing_depth, forchange): addr = self.get_addr(mixing_depth, forchange, index[forchange]) self.addr_cache[addr] = (mixing_depth, forchange, index[forchange]) index[forchange] += 1 - #self.update_cache_index() return addr def get_receive_addr(self, mixing_depth): diff --git a/lib/maker.py b/lib/maker.py index b14d0719..0e62ea89 100644 --- a/lib/maker.py +++ b/lib/maker.py @@ -33,7 +33,6 @@ def __init__(self, maker, nick, oid, amount, taker_pk): self.cjfee = order['cjfee'] debug('new cjorder nick=%s oid=%d amount=%d' % (nick, oid, amount)) self.utxos, self.cj_addr, self.change_addr = maker.oid_to_order(self, oid, amount) - self.maker.wallet.update_cache_index() if not self.utxos: self.maker.msgchan.send_error(nick, 'unable to fill order constrained by dust avoidance') #TODO make up orders offers in a way that this error cant appear diff --git a/wallet-tool.py b/wallet-tool.py index 7be78837..3a36c640 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -49,9 +49,9 @@ else: seed = args[0] method = ('display' if len(args) == 1 else args[1].lower()) - wallet = Wallet(seed, options.maxmixdepth, options.gaplimit) + wallet = Wallet(seed, options.maxmixdepth) if method != 'showseed': - common.bc_interface.sync_wallet(wallet) + common.bc_interface.sync_wallet(wallet, options.gaplimit) if method == 'display' or method == 'displayall': total_balance = 0 From 7bff63ee419af88b90fe3e61d00db81c27eaa6c2 Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Tue, 19 May 2015 02:00:19 +0100 Subject: [PATCH 272/409] Revert "Revert "added update_cache_index() code, which stores the pointers of the addresses we're up to in the HD wallet"" This reverts commit cfe7d24472fd72cb5dad7f16b3d4b3bd3c2ae9e1. --- lib/blockchaininterface.py | 26 ++++++++++++++------------ lib/common.py | 29 ++++++++++++++++++++++++++--- lib/maker.py | 1 + wallet-tool.py | 4 ++-- 4 files changed, 43 insertions(+), 17 deletions(-) diff --git a/lib/blockchaininterface.py b/lib/blockchaininterface.py index 3d3f12dd..558f00cb 100644 --- a/lib/blockchaininterface.py +++ b/lib/blockchaininterface.py @@ -27,12 +27,12 @@ class BlockchainInterface(object): def __init__(self): pass - def sync_wallet(self, wallet, gaplimit=6): - self.sync_addresses(wallet, gaplimit) + def sync_wallet(self, wallet): + self.sync_addresses(wallet) self.sync_unspent(wallet) @abc.abstractmethod - def sync_addresses(self, wallet, gaplimit=6): + def sync_addresses(self, wallet): '''Finds which addresses have been used and sets wallet.index appropriately''' pass @@ -69,28 +69,31 @@ def __init__(self, testnet = False): self.blockr_domain = 'tbtc' if testnet else 'btc' self.last_sync_unspent = 0 - def sync_addresses(self, wallet, gaplimit=6): + def sync_addresses(self, wallet): common.debug('downloading wallet history') #sets Wallet internal indexes to be at the next unused address for mix_depth in range(wallet.max_mix_depth): for forchange in [0, 1]: unused_addr_count = 0 last_used_addr = '' - while unused_addr_count < gaplimit: + while unused_addr_count < wallet.gaplimit or\ + wallet.index[mix_depth][forchange] <= wallet.index_cache[mix_depth][forchange]: addrs = [wallet.get_new_addr(mix_depth, forchange) for i in range(self.BLOCKR_MAX_ADDR_REQ_COUNT)] #TODO send a pull request to pybitcointools # because this surely should be possible with a function from it blockr_url = 'http://' + self.blockr_domain + '.blockr.io/api/v1/address/txs/' + #print 'downloading, lastusedaddr = ' + last_used_addr + ' unusedaddrcount= ' + str(unused_addr_count) res = btc.make_request(blockr_url+','.join(addrs)) data = json.loads(res)['data'] for dat in data: + #if forchange == 0: + # print ' nbtxs ' + str(dat['nb_txs']) + ' addr=' + dat['address'] + ' unused=' + str(unused_addr_count) if dat['nb_txs'] != 0: last_used_addr = dat['address'] + unused_addr_count = 0 else: unused_addr_count += 1 - if unused_addr_count >= gaplimit: - break if last_used_addr == '': wallet.index[mix_depth][forchange] = 0 else: @@ -365,7 +368,7 @@ def add_watchonly_addresses(self, addr_list, wallet_name): print 'now restart bitcoind with -rescan' sys.exit(0) - def sync_addresses(self, wallet, gaplimit=6): + def sync_addresses(self, wallet): common.debug('requesting wallet history') wallet_name = self.get_wallet_name(wallet) addr_req_count = 50 @@ -392,7 +395,8 @@ def sync_addresses(self, wallet, gaplimit=6): last_used_addr = '' breakloop = False while not breakloop: - if unused_addr_count >= gaplimit: + if unused_addr_count >= wallet.gaplimit and\ + wallet.index[mix_depth][forchange] >= wallet.index_cache[mix_depth][forchange]: break mix_change_addrs = [wallet.get_new_addr(mix_depth, forchange) for i in range(addr_req_count)] for mc_addr in mix_change_addrs: @@ -402,11 +406,9 @@ def sync_addresses(self, wallet, gaplimit=6): break if mc_addr in used_addr_list: last_used_addr = mc_addr + unused_addr_count = 0 else: unused_addr_count += 1 - if unused_addr_count >= gaplimit: - breakloop = True - break if last_used_addr == '': wallet.index[mix_depth][forchange] = 0 diff --git a/lib/common.py b/lib/common.py index 4167e895..96e4525b 100644 --- a/lib/common.py +++ b/lib/common.py @@ -101,10 +101,11 @@ def debug_dump_object(obj, skip_fields=[]): debug(str(v)) class Wallet(object): - def __init__(self, seedarg, max_mix_depth=2): + def __init__(self, seedarg, max_mix_depth=2, gaplimit=6): self.max_mix_depth = max_mix_depth - self.seed = self.get_seed(seedarg) - master = btc.bip32_master_key(self.seed) + self.gaplimit = gaplimit + seed = self.get_seed(seedarg) + master = btc.bip32_master_key(seed) m_0 = btc.bip32_ckd(master, 0) mixing_depth_keys = [btc.bip32_ckd(m_0, c) for c in range(max_mix_depth)] self.keys = [(btc.bip32_ckd(m, 0), btc.bip32_ckd(m, 1)) for m in mixing_depth_keys] @@ -123,6 +124,8 @@ def __init__(self, seedarg, max_mix_depth=2): self.spent_utxos = [] def get_seed(self, seedarg): + self.path = None + self.index_cache = [[0, 0]]*self.max_mix_depth path = os.path.join('wallets', seedarg) if not os.path.isfile(path): if get_network() == 'testnet': @@ -131,6 +134,7 @@ def get_seed(self, seedarg): else: raise IOError('wallet file not found') #debug('seedarg interpreted as wallet file name') + self.path = path try: import aes except ImportError: @@ -144,12 +148,30 @@ def get_seed(self, seedarg): print 'wallet network(%s) does not match joinmarket configured network(%s)' % ( walletdata['network'], get_network()) sys.exit(0) + if 'index_cache' in walletdata: + self.index_cache = walletdata['index_cache'] + print 'index cache = ' + str(self.index_cache) password = getpass.getpass('Enter wallet decryption passphrase: ') password_key = btc.bin_dbl_sha256(password) decrypted_seed = aes.decryptData(password_key, walletdata['encrypted_seed'] .decode('hex')).encode('hex') return decrypted_seed + def update_cache_index(self): + if not self.path: + return + if not os.path.isfile(self.path): + return + fd = open(self.path, 'r') + walletfile = fd.read() + fd.close() + walletdata = json.loads(walletfile) + walletdata['index_cache'] = self.index + walletfile = json.dumps(walletdata) + fd = open(self.path, 'w') + fd.write(walletfile) + fd.close() + def get_key(self, mixing_depth, forchange, i): return btc.bip32_extract_key(btc.bip32_ckd(self.keys[mixing_depth][forchange], i)) @@ -161,6 +183,7 @@ def get_new_addr(self, mixing_depth, forchange): addr = self.get_addr(mixing_depth, forchange, index[forchange]) self.addr_cache[addr] = (mixing_depth, forchange, index[forchange]) index[forchange] += 1 + #self.update_cache_index() return addr def get_receive_addr(self, mixing_depth): diff --git a/lib/maker.py b/lib/maker.py index 0e62ea89..b14d0719 100644 --- a/lib/maker.py +++ b/lib/maker.py @@ -33,6 +33,7 @@ def __init__(self, maker, nick, oid, amount, taker_pk): self.cjfee = order['cjfee'] debug('new cjorder nick=%s oid=%d amount=%d' % (nick, oid, amount)) self.utxos, self.cj_addr, self.change_addr = maker.oid_to_order(self, oid, amount) + self.maker.wallet.update_cache_index() if not self.utxos: self.maker.msgchan.send_error(nick, 'unable to fill order constrained by dust avoidance') #TODO make up orders offers in a way that this error cant appear diff --git a/wallet-tool.py b/wallet-tool.py index 3a36c640..7be78837 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -49,9 +49,9 @@ else: seed = args[0] method = ('display' if len(args) == 1 else args[1].lower()) - wallet = Wallet(seed, options.maxmixdepth) + wallet = Wallet(seed, options.maxmixdepth, options.gaplimit) if method != 'showseed': - common.bc_interface.sync_wallet(wallet, options.gaplimit) + common.bc_interface.sync_wallet(wallet) if method == 'display' or method == 'displayall': total_balance = 0 From 4487d2b9c47190b2edccea79866dc8e5c19111b1 Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Wed, 20 May 2015 00:10:36 +0100 Subject: [PATCH 273/409] prints generated nickname so it can be more easily found --- lib/irc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/irc.py b/lib/irc.py index 2709ddb9..185934bc 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -23,6 +23,7 @@ def random_nick(): if re.match('\\d', ircnick[0]): ircnick = '_' + ircnick ircnick = ircnick[:9] + print 'Generated random nickname: ' + ircnick #not using debug because it might not know the logfile name at this point return ircnick def get_irc_text(line): From a3e8c4c8ab7d231c8feb1c1e22f46ca6cf93900c Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Wed, 20 May 2015 00:41:53 +0100 Subject: [PATCH 274/409] added warning to yieldgen when using with blockr.io --- lib/common.py | 1 - yield-generator.py | 10 +++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/common.py b/lib/common.py index 96e4525b..15d7bad4 100644 --- a/lib/common.py +++ b/lib/common.py @@ -150,7 +150,6 @@ def get_seed(self, seedarg): sys.exit(0) if 'index_cache' in walletdata: self.index_cache = walletdata['index_cache'] - print 'index cache = ' + str(self.index_cache) password = getpass.getpass('Enter wallet decryption passphrase: ') password_key = btc.bin_dbl_sha256(password) decrypted_seed = aes.decryptData(password_key, walletdata['encrypted_seed'] diff --git a/yield-generator.py b/yield-generator.py index 22e29ba3..055dd253 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -8,7 +8,7 @@ from maker import * from irc import IRCMessageChannel, random_nick import bitcoin as btc -import common +import common, blockchaininterface from socket import gethostname @@ -117,6 +117,14 @@ def main(): common.load_program_config() import sys seed = sys.argv[1] + if isinstance(common.bc_interface, blockchaininterface.BlockrInterface): + print '\nYou are running a yield generator by polling the blockr.io website' + print 'This is quite bad for privacy. That site is owned by coinbase.com' + print 'Learn how to setup JoinMarket with Bitcoin Core: https://github.com/chris-belcher/joinmarket/wiki/Running-JoinMarket-with-Bitcoin-Core-full-node' + ret = raw_input('\nContinue? (y/n):') + if ret[0] != 'y': + return + wallet = Wallet(seed, max_mix_depth = mix_levels) common.bc_interface.sync_wallet(wallet) wallet.print_debug_wallet_info() From c0a8b74601494b273e847c0603dac7ccc4d998f0 Mon Sep 17 00:00:00 2001 From: Adlai Chandrasekhar Date: Wed, 20 May 2015 03:33:45 +0300 Subject: [PATCH 275/409] Use itertools.combinations when sweeping github magic: fixes #48 --- lib/common.py | 37 ++----------------------------------- 1 file changed, 2 insertions(+), 35 deletions(-) diff --git a/lib/common.py b/lib/common.py index 15d7bad4..e0574b04 100644 --- a/lib/common.py +++ b/lib/common.py @@ -7,6 +7,7 @@ import blockchaininterface from ConfigParser import SafeConfigParser import os +import itertools JM_VERSION = 1 nickname = '' @@ -346,40 +347,6 @@ def choose_order(db, cj_amount, n, chooseOrdersBy): chosen_orders = [o[:2] for o in chosen_orders] return dict(chosen_orders), total_cj_fee -def nCk(n, k): - ''' - n choose k - ''' - return factorial(n) / factorial(k) / factorial(n - k) - -def create_combination(li, n): - ''' - Creates a list of combinations of elements of a given list - For example, combination(['apple', 'orange', 'pear'], 2) - = [('apple', 'orange'), ('apple', 'pear'), ('orange', 'pear')] - ''' - result = [] - if n == 1: - result = [(l,) for l in li] #same thing but each order is a tuple - elif n == 2: - #this case could be removed and the function completely recurvsive - # but for n=2 this is slightly more efficent - for i, e1 in enumerate(li): - for e2 in li[i+1:]: - result.append((e1, e2)) - else: - for i, e in enumerate(li): - if len(li[i:]) < n: - #there wont be - continue - combn1 = create_combination(li[i:], n - 1) - for c in combn1: - if e not in c: - result.append((e,) + c) - - assert len(result) == nCk(len(li), n) - return result - def choose_sweep_order(db, my_total_input, my_tx_fee, n, chooseOrdersBy): ''' choose an order given that we want to be left with no change @@ -415,7 +382,7 @@ def is_amount_in_range(ordercombo, cjamount): orderkeys = ['counterparty', 'oid', 'ordertype', 'minsize', 'maxsize', 'txfee', 'cjfee'] orderlist = [dict([(k, o[k]) for k in orderkeys]) for o in sqlorders] - ordercombos = create_combination(orderlist, n) + ordercombos = [combo for combo in itertools.combinations(orderlist, n)] ordercombos = [(c, calc_zero_change_cj_amount(c)) for c in ordercombos] ordercombos = [oc for oc in ordercombos if is_amount_in_range(oc[0], oc[1][0])] From aaec24eab4d5e608b7cb53b5e5c036e6d8c4c944 Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Wed, 20 May 2015 22:42:54 +0100 Subject: [PATCH 276/409] added more debug lines to the networking code to hopefully find the source of a hang bug --- lib/irc.py | 8 ++++---- ob-watcher.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/irc.py b/lib/irc.py index 185934bc..49ec7e83 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -167,7 +167,7 @@ def __privmsg(self, nick, cmd, message): def send_raw(self, line): #if not line.startswith('PING LAG'): - # debug('sendraw ' + line) + debug('sendraw ' + line) self.sock.sendall(line + '\r\n') def check_for_orders(self, nick, chunks): @@ -335,20 +335,20 @@ def __handle_privmsg(self, source, target, message): parsed = self.built_privmsg[nick][1] #wipe the message buffer waiting for the next one del self.built_privmsg[nick] - debug("< Date: Fri, 22 May 2015 00:56:19 +0100 Subject: [PATCH 277/409] added comments to default config file --- lib/common.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/common.py b/lib/common.py index b953c717..c7f68d83 100644 --- a/lib/common.py +++ b/lib/common.py @@ -24,13 +24,12 @@ required_options = {'BLOCKCHAIN':['blockchain_source', 'network', 'bitcoin_cli_cmd'], 'MESSAGING':['host','channel','port']} -def load_program_config(): - loadedFiles = config.read([config_location]) - #Create default config file if not found - if len(loadedFiles) != 1: - defaultconfig = """ +defaultconfig =\ +""" [BLOCKCHAIN] blockchain_source = blockr +#options: blockr, json-rpc, regtest +#before using json-rpc read https://github.com/chris-belcher/joinmarket/wiki/Running-JoinMarket-with-Bitcoin-Core-full-node network = mainnet bitcoin_cli_cmd = bitcoin-cli @@ -43,9 +42,14 @@ def load_program_config(): socks5_host = 127.0.0.1 socks5_port = 9150 """ + +def load_program_config(): + loadedFiles = config.read([config_location]) + #Create default config file if not found + if len(loadedFiles) != 1: config.readfp(io.BytesIO(defaultconfig)) - with open(config_location, "wb") as configfile: - config.write(configfile) + with open(config_location, "w") as configfile: + configfile.write(defaultconfig) #check for sections for s in required_options: From 9b993097d890d86c5d6724b13a7d4f4fcb5847dc Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Sat, 23 May 2015 21:24:12 +0100 Subject: [PATCH 278/409] handled the case where len(orders) == n --- lib/common.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/common.py b/lib/common.py index c7f68d83..99e8af25 100644 --- a/lib/common.py +++ b/lib/common.py @@ -314,7 +314,10 @@ def weighted_order_choose(orders, n, feekey): else: phi = feekey(orders[-1]) - minfee fee = np.array([feekey(o) for o in orders]) - weight = np.exp(-(1.0*fee - minfee) / phi) + if phi > 0: + weight = np.exp(-(1.0*fee - minfee) / phi) + else: + weight = np.ones_like(fee) weight /= sum(weight) debug('randomly choosing orders with weighting\n' + pprint.pformat(zip(orders, weight))) chosen_order_index = np.random.choice(len(orders), p=weight) From e3728e8c7ae4aa540f62e70c2500559824c0c6e8 Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Sat, 23 May 2015 22:17:36 +0100 Subject: [PATCH 279/409] expanded the warning in yieldgen.py against not using bitcoin core --- yield-generator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/yield-generator.py b/yield-generator.py index 055dd253..e45e7aff 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -120,6 +120,8 @@ def main(): if isinstance(common.bc_interface, blockchaininterface.BlockrInterface): print '\nYou are running a yield generator by polling the blockr.io website' print 'This is quite bad for privacy. That site is owned by coinbase.com' + print 'Also your bot will run faster and more efficently, you can be immediately notified of new bitcoin network' + print ' information so your money will be working for you as hard as possible' print 'Learn how to setup JoinMarket with Bitcoin Core: https://github.com/chris-belcher/joinmarket/wiki/Running-JoinMarket-with-Bitcoin-Core-full-node' ret = raw_input('\nContinue? (y/n):') if ret[0] != 'y': From 6dbe42ba8f424539424b7b8ecee5631d605ff1f1 Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Sat, 23 May 2015 23:01:35 +0100 Subject: [PATCH 280/409] changed irc network where the pit is --- lib/common.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/common.py b/lib/common.py index 99e8af25..f9156224 100644 --- a/lib/common.py +++ b/lib/common.py @@ -34,13 +34,18 @@ bitcoin_cli_cmd = bitcoin-cli [MESSAGING] -host = irc.snoonet.org +host = irc.cyberguerrilla.org channel = joinmarket-pit port = 6697 usessl = true socks5 = false -socks5_host = 127.0.0.1 +socks5_host = localhost socks5_port = 9150 +#for tor +#host = 6dvj6v5imhny3anf.onion +#port = 6667 +#usessl = false +#socks5 = true """ def load_program_config(): From c780aa7d44359b464e11411a4dc92b44d26e1cf0 Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Sun, 24 May 2015 01:03:42 +0100 Subject: [PATCH 281/409] added MODE +B to mark bots as bots --- lib/irc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/irc.py b/lib/irc.py index 49ec7e83..508e378b 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -466,6 +466,7 @@ def run(self): self.send_raw('CAP REQ :sasl') self.send_raw('USER %s b c :%s' % self.userrealname) self.send_raw('NICK ' + self.given_nick) + self.send_raw('MODE ' + self.given_nick + ' +B') #marks as bots on unreal while 1: try: line = self.fd.readline() From ca368caf58ba5d97f26184b4e1715a13395b3fff Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Sun, 24 May 2015 01:27:46 +0100 Subject: [PATCH 282/409] fixed MODE bug, allowed unregistered nicks to private message --- lib/irc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/irc.py b/lib/irc.py index 508e378b..172a81ca 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -382,6 +382,8 @@ def __handle_line(self, line): if self.on_connect: self.on_connect() self.send_raw('JOIN ' + self.channel) + self.send_raw('MODE ' + self.given_nick + ' +B') #marks as bots on unreal + self.send_raw('MODE ' + self.given_nick + ' -R') #allows unreg'd private messages elif chunks[1] == '433': #nick in use #self.nick = random_nick() self.nick += '_' #helps keep identity constant if just _ added @@ -466,7 +468,6 @@ def run(self): self.send_raw('CAP REQ :sasl') self.send_raw('USER %s b c :%s' % self.userrealname) self.send_raw('NICK ' + self.given_nick) - self.send_raw('MODE ' + self.given_nick + ' +B') #marks as bots on unreal while 1: try: line = self.fd.readline() From 900a57abe0c1bd28d3ebf5c8cbf870df94d0aedf Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Sun, 24 May 2015 01:33:07 +0100 Subject: [PATCH 283/409] fixed bug in sasl code --- lib/irc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/irc.py b/lib/irc.py index 172a81ca..21f8c9e7 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -361,7 +361,7 @@ def __handle_line(self, line): self.shutdown() self.send_raw('AUTHENTICATE PLAIN') elif chunks[0] == 'AUTHENTICATE': - self.send_raw('AUTHENTICATE ' + base64.b64encode(self.nick + '\x00' + self.nick + '\x00' + self.password)) + self.send_raw('AUTHENTICATE ' + base64.b64encode(self.nick + '\x00' + self.userrealname[0] + '\x00' + self.password)) elif chunks[1] == '903': debug('Successfully authenticated') self.password = None From 04902c06a19c00f370673bb1c0cf075c89f9391a Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Sun, 24 May 2015 17:53:33 +0100 Subject: [PATCH 284/409] added the joinmarket alert message to the html orderbook page --- ob-watcher.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ob-watcher.py b/ob-watcher.py index b9d707ff..d8a6aa2e 100644 --- a/ob-watcher.py +++ b/ob-watcher.py @@ -8,7 +8,7 @@ import taker from irc import IRCMessageChannel, random_nick from common import * - +import common tableheading = '''
    @@ -126,20 +126,23 @@ def do_GET(self): fd = open('orderbook.html', 'r') orderbook_fmt = fd.read() fd.close() + alert_msg = '' + if common.joinmarket_alert: + alert_msg = '
    JoinMarket Alert Message:
    ' + common.joinmarket_alert if self.path == '/': ordercount, ordertable = self.create_orderbook_table() replacements = { 'PAGETITLE': 'JoinMarket Browser Interface', 'MAINHEADING': 'JoinMarket Orderbook', 'SECONDHEADING': (str(ordercount) + ' orders found by ' - + self.get_counterparty_count() + ' counterparties'), + + self.get_counterparty_count() + ' counterparties' + alert_msg), 'MAINBODY': refresh_orderbook_form + shutdownform + tableheading + ordertable + '
    \n' } elif self.path == '/ordersize': replacements = { 'PAGETITLE': 'JoinMarket Browser Interface', 'MAINHEADING': 'Order Sizes', - 'SECONDHEADING': 'Order Size Histogram', + 'SECONDHEADING': 'Order Size Histogram' + alert_msg, 'MAINBODY': create_size_histogram(self.taker.db) } elif self.path.startswith('/depth'): @@ -150,7 +153,7 @@ def do_GET(self): replacements = { 'PAGETITLE': 'JoinMarket Browser Interface', 'MAINHEADING': 'Depth Chart', - 'SECONDHEADING': 'Orderbook Depth', + 'SECONDHEADING': 'Orderbook Depth' + alert_msg, 'MAINBODY': '
    '.join(mainbody) } orderbook_page = orderbook_fmt From fde0b9ed8e66e0b3161bac66249f5b131a43ff56 Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Sun, 24 May 2015 17:53:46 +0100 Subject: [PATCH 285/409] Revert "fixed bug in sasl code", actually wasnt a bug This reverts commit 900a57abe0c1bd28d3ebf5c8cbf870df94d0aedf. --- lib/irc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/irc.py b/lib/irc.py index 21f8c9e7..172a81ca 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -361,7 +361,7 @@ def __handle_line(self, line): self.shutdown() self.send_raw('AUTHENTICATE PLAIN') elif chunks[0] == 'AUTHENTICATE': - self.send_raw('AUTHENTICATE ' + base64.b64encode(self.nick + '\x00' + self.userrealname[0] + '\x00' + self.password)) + self.send_raw('AUTHENTICATE ' + base64.b64encode(self.nick + '\x00' + self.nick + '\x00' + self.password)) elif chunks[1] == '903': debug('Successfully authenticated') self.password = None From 68756a2ed27701c79bbd6f93faa3e7d741d957f2 Mon Sep 17 00:00:00 2001 From: Adlai Chandrasekhar Date: Sun, 24 May 2015 23:51:46 +0300 Subject: [PATCH 286/409] fix depth charts for the case of a lone applicable offer --- ob-watcher.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ob-watcher.py b/ob-watcher.py index d8a6aa2e..c2a4c60c 100644 --- a/ob-watcher.py +++ b/ob-watcher.py @@ -46,7 +46,10 @@ def create_depth_chart(db, cj_amount): if len(orderfees) == 0: return 'No orders at amount ' + str(cj_amount/1e8) fig = plt.figure() - plt.hist(orderfees, 30, histtype='bar', rwidth=0.8) + if len(orderfees) == 1: + plt.hist(orderfees, 30, rwidth=0.8, range=(orderfees[0]/2, orderfees[0]*2)) + else: + plt.hist(orderfees, 30, rwidth=0.8) plt.grid() plt.title('CoinJoin Orderbook Depth Chart for amount=' + str(cj_amount/1e8) + 'btc') plt.xlabel('CoinJoin Fee / btc') From 52dd847db537b76a315625b27f7e11c707affad4 Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Sun, 24 May 2015 23:48:53 +0100 Subject: [PATCH 287/409] removed duplicate check for utxos being spent, required to remove references to wallet.unspent --- lib/maker.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/maker.py b/lib/maker.py index b14d0719..d64c5aa8 100644 --- a/lib/maker.py +++ b/lib/maker.py @@ -132,11 +132,8 @@ def verify_unsigned_tx(self, txd): not in input_addresses: return False, "authenticating bitcoin address is not contained" my_utxo_set = set(self.utxos.keys()) - wallet_utxos = set(self.maker.wallet.unspent) if not tx_utxo_set.issuperset(my_utxo_set): return False, 'my utxos are not contained' - if not wallet_utxos.issuperset(my_utxo_set): - return False, 'my utxos already spent' my_total_in = sum([va['value'] for va in self.utxos.values()]) self.real_cjfee = calc_cj_fee(self.ordertype, self.cjfee, self.cj_amount) From 03634ff6ed20bcae46fc40645a568f432f9a9c02 Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Mon, 25 May 2015 00:06:15 +0100 Subject: [PATCH 288/409] fixed bug in issue #86, and for patientsendpayment --- patientsendpayment.py | 2 +- tumbler.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/patientsendpayment.py b/patientsendpayment.py index 3b4df8d7..e3bdd5bf 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -34,7 +34,7 @@ def run(self): print 'giving up waiting' #cancel the remaining order self.tmaker.modify_orders([0], []) - orders, total_cj_fee = choose_order(self.tmaker.db, self.tmaker.amount, self.tmaker.makercount) + orders, total_cj_fee = choose_order(self.tmaker.db, self.tmaker.amount, self.tmaker.makercount, weighted_order_choose) print 'chosen orders to fill ' + str(orders) + ' totalcjfee=' + str(total_cj_fee) total_amount = self.tmaker.amount + total_cj_fee + self.tmaker.txfee print 'total amount spent = ' + str(total_amount) diff --git a/tumbler.py b/tumbler.py index 140a28e0..9b7ca491 100644 --- a/tumbler.py +++ b/tumbler.py @@ -108,10 +108,8 @@ def send_tx(self, tx, balance, sweep): print 'sweeping' all_utxos = self.taker.wallet.get_utxos_by_mixdepth()[tx['srcmixdepth']] total_value = sum([addrval['value'] for addrval in all_utxos.values()]) - orders, cjamount = choose_sweep_order(self.taker.db, total_value, self.taker.txfee, tx['makercount']) while True: - #orders, total_cj_fee = choose_order(self.taker.db, amount, tx['makercount']) - orders, cjamount = choose_sweep_order(self.taker.db, total_value, self.taker.txfee, tx['makercount']) + orders, cjamount = choose_sweep_order(self.taker.db, total_value, self.taker.txfee, tx['makercount'], weighted_order_choose) if orders == None: print 'waiting for liquidity' time.sleep(10) @@ -133,7 +131,7 @@ def send_tx(self, tx, balance, sweep): changeaddr = self.taker.wallet.get_change_addr(tx['srcmixdepth']) print 'coinjoining ' + str(amount) while True: - orders, total_cj_fee = choose_order(self.taker.db, amount, tx['makercount']) + orders, total_cj_fee = choose_order(self.taker.db, amount, tx['makercount'], weighted_order_choose) cj_fee = 1.0*total_cj_fee / tx['makercount'] / amount print 'average fee = ' + str(cj_fee) if cj_fee > self.taker.maxcjfee: From 7a82abc1234667de0627663b65b57187084c9375 Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Tue, 26 May 2015 00:23:31 +0100 Subject: [PATCH 289/409] included slowaes.py and socks.py in the repos for easier installation --- README.txt | 6 +- lib/common.py | 9 +- lib/slowaes.py | 656 +++++++++++++++++++++++++++++++++++++++++++++++++ lib/socks.py | 387 +++++++++++++++++++++++++++++ wallet-tool.py | 9 +- 5 files changed, 1049 insertions(+), 18 deletions(-) create mode 100644 lib/slowaes.py create mode 100644 lib/socks.py diff --git a/README.txt b/README.txt index df7474ab..794b2835 100644 --- a/README.txt +++ b/README.txt @@ -23,10 +23,8 @@ INSTALLING Get it here: http://doc.libsodium.org/ use this line to check it was installed correctly python lib/enc_wrapper.py -2. You will need slowaes installed for encrypting your wallet - sudo pip install --pre slowaes -3. you will need numpy 1.7 or later installed -4. (optional) matplotlib for displaying the graphs in orderbook-watcher +2. you will need numpy 1.7 or later installed +3. (optional) matplotlib for displaying the graphs in orderbook-watcher in the joinmarket.cfg configuration file, set network = mainnet diff --git a/lib/common.py b/lib/common.py index f9156224..6f328bd3 100644 --- a/lib/common.py +++ b/lib/common.py @@ -4,7 +4,7 @@ from math import factorial import sys, datetime, json, time, pprint, threading, getpass import numpy as np -import blockchaininterface +import blockchaininterface, slowaes from ConfigParser import SafeConfigParser import os, io, itertools @@ -161,11 +161,6 @@ def get_seed(self, seedarg): raise IOError('wallet file not found') #debug('seedarg interpreted as wallet file name') self.path = path - try: - import aes - except ImportError: - print 'You must install slowaes\nTry running: sudo pip install slowaes' - sys.exit(0) fd = open(path, 'r') walletfile = fd.read() fd.close() @@ -178,7 +173,7 @@ def get_seed(self, seedarg): self.index_cache = walletdata['index_cache'] password = getpass.getpass('Enter wallet decryption passphrase: ') password_key = btc.bin_dbl_sha256(password) - decrypted_seed = aes.decryptData(password_key, walletdata['encrypted_seed'] + decrypted_seed = slowaes.decryptData(password_key, walletdata['encrypted_seed'] .decode('hex')).encode('hex') return decrypted_seed diff --git a/lib/slowaes.py b/lib/slowaes.py new file mode 100644 index 00000000..e5f9e89a --- /dev/null +++ b/lib/slowaes.py @@ -0,0 +1,656 @@ +#!/usr/bin/python +# +# aes.py: implements AES - Advanced Encryption Standard +# from the SlowAES project, http://code.google.com/p/slowaes/ +# +# Copyright (c) 2008 Josh Davis ( http://www.josh-davis.org ), +# Alex Martelli ( http://www.aleax.it ) +# +# Ported from C code written by Laurent Haan ( http://www.progressive-coding.com ) +# +# Licensed under the Apache License, Version 2.0 +# http://www.apache.org/licenses/ +# +import os +import sys +import math + +def append_PKCS7_padding(s): + """return s padded to a multiple of 16-bytes by PKCS7 padding""" + numpads = 16 - (len(s)%16) + return s + numpads*chr(numpads) + +def strip_PKCS7_padding(s): + """return s stripped of PKCS7 padding""" + if len(s)%16 or not s: + raise ValueError("String of len %d can't be PCKS7-padded" % len(s)) + numpads = ord(s[-1]) + if numpads > 16: + raise ValueError("String ending with %r can't be PCKS7-padded" % s[-1]) + return s[:-numpads] + +class AES(object): + # valid key sizes + keySize = dict(SIZE_128=16, SIZE_192=24, SIZE_256=32) + + # Rijndael S-box + sbox = [0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, + 0x2b, 0xfe, 0xd7, 0xab, 0x76, 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, + 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, 0xb7, + 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, + 0x71, 0xd8, 0x31, 0x15, 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, + 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, 0x09, 0x83, + 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, + 0xe3, 0x2f, 0x84, 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, + 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, 0xd0, 0xef, 0xaa, + 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, + 0x9f, 0xa8, 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, + 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, 0xcd, 0x0c, 0x13, 0xec, + 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, + 0x73, 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, + 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, 0xe0, 0x32, 0x3a, 0x0a, 0x49, + 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, + 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, + 0xea, 0x65, 0x7a, 0xae, 0x08, 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, + 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, 0x70, + 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, + 0x86, 0xc1, 0x1d, 0x9e, 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, + 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, 0x8c, 0xa1, + 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, + 0x54, 0xbb, 0x16] + + # Rijndael Inverted S-box + rsbox = [0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, + 0x9e, 0x81, 0xf3, 0xd7, 0xfb , 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, + 0xff, 0x87, 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb , 0x54, + 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, + 0x42, 0xfa, 0xc3, 0x4e , 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, + 0xb2, 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25 , 0x72, 0xf8, + 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, + 0x65, 0xb6, 0x92 , 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, + 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84 , 0x90, 0xd8, 0xab, + 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, + 0x45, 0x06 , 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 0xc1, + 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b , 0x3a, 0x91, 0x11, 0x41, + 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, + 0x73 , 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, + 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e , 0x47, 0xf1, 0x1a, 0x71, 0x1d, + 0x29, 0xc5, 0x89, 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b , + 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, + 0xfe, 0x78, 0xcd, 0x5a, 0xf4 , 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, + 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f , 0x60, + 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, + 0x93, 0xc9, 0x9c, 0xef , 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, + 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61 , 0x17, 0x2b, + 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, + 0x21, 0x0c, 0x7d] + + def getSBoxValue(self,num): + """Retrieves a given S-Box Value""" + return self.sbox[num] + + def getSBoxInvert(self,num): + """Retrieves a given Inverted S-Box Value""" + return self.rsbox[num] + + def rotate(self, word): + """ Rijndael's key schedule rotate operation. + + Rotate a word eight bits to the left: eg, rotate(1d2c3a4f) == 2c3a4f1d + Word is an char list of size 4 (32 bits overall). + """ + return word[1:] + word[:1] + + # Rijndael Rcon + Rcon = [0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, + 0x6c, 0xd8, 0xab, 0x4d, 0x9a, 0x2f, 0x5e, 0xbc, 0x63, 0xc6, 0x97, + 0x35, 0x6a, 0xd4, 0xb3, 0x7d, 0xfa, 0xef, 0xc5, 0x91, 0x39, 0x72, + 0xe4, 0xd3, 0xbd, 0x61, 0xc2, 0x9f, 0x25, 0x4a, 0x94, 0x33, 0x66, + 0xcc, 0x83, 0x1d, 0x3a, 0x74, 0xe8, 0xcb, 0x8d, 0x01, 0x02, 0x04, + 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, 0x6c, 0xd8, 0xab, 0x4d, + 0x9a, 0x2f, 0x5e, 0xbc, 0x63, 0xc6, 0x97, 0x35, 0x6a, 0xd4, 0xb3, + 0x7d, 0xfa, 0xef, 0xc5, 0x91, 0x39, 0x72, 0xe4, 0xd3, 0xbd, 0x61, + 0xc2, 0x9f, 0x25, 0x4a, 0x94, 0x33, 0x66, 0xcc, 0x83, 0x1d, 0x3a, + 0x74, 0xe8, 0xcb, 0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, + 0x80, 0x1b, 0x36, 0x6c, 0xd8, 0xab, 0x4d, 0x9a, 0x2f, 0x5e, 0xbc, + 0x63, 0xc6, 0x97, 0x35, 0x6a, 0xd4, 0xb3, 0x7d, 0xfa, 0xef, 0xc5, + 0x91, 0x39, 0x72, 0xe4, 0xd3, 0xbd, 0x61, 0xc2, 0x9f, 0x25, 0x4a, + 0x94, 0x33, 0x66, 0xcc, 0x83, 0x1d, 0x3a, 0x74, 0xe8, 0xcb, 0x8d, + 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, 0x6c, + 0xd8, 0xab, 0x4d, 0x9a, 0x2f, 0x5e, 0xbc, 0x63, 0xc6, 0x97, 0x35, + 0x6a, 0xd4, 0xb3, 0x7d, 0xfa, 0xef, 0xc5, 0x91, 0x39, 0x72, 0xe4, + 0xd3, 0xbd, 0x61, 0xc2, 0x9f, 0x25, 0x4a, 0x94, 0x33, 0x66, 0xcc, + 0x83, 0x1d, 0x3a, 0x74, 0xe8, 0xcb, 0x8d, 0x01, 0x02, 0x04, 0x08, + 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, 0x6c, 0xd8, 0xab, 0x4d, 0x9a, + 0x2f, 0x5e, 0xbc, 0x63, 0xc6, 0x97, 0x35, 0x6a, 0xd4, 0xb3, 0x7d, + 0xfa, 0xef, 0xc5, 0x91, 0x39, 0x72, 0xe4, 0xd3, 0xbd, 0x61, 0xc2, + 0x9f, 0x25, 0x4a, 0x94, 0x33, 0x66, 0xcc, 0x83, 0x1d, 0x3a, 0x74, + 0xe8, 0xcb ] + + def getRconValue(self, num): + """Retrieves a given Rcon Value""" + return self.Rcon[num] + + def core(self, word, iteration): + """Key schedule core.""" + # rotate the 32-bit word 8 bits to the left + word = self.rotate(word) + # apply S-Box substitution on all 4 parts of the 32-bit word + for i in range(4): + word[i] = self.getSBoxValue(word[i]) + # XOR the output of the rcon operation with i to the first part + # (leftmost) only + word[0] = word[0] ^ self.getRconValue(iteration) + return word + + def expandKey(self, key, size, expandedKeySize): + """Rijndael's key expansion. + + Expands an 128,192,256 key into an 176,208,240 bytes key + + expandedKey is a char list of large enough size, + key is the non-expanded key. + """ + # current expanded keySize, in bytes + currentSize = 0 + rconIteration = 1 + expandedKey = [0] * expandedKeySize + + # set the 16, 24, 32 bytes of the expanded key to the input key + for j in range(size): + expandedKey[j] = key[j] + currentSize += size + + while currentSize < expandedKeySize: + # assign the previous 4 bytes to the temporary value t + t = expandedKey[currentSize-4:currentSize] + + # every 16,24,32 bytes we apply the core schedule to t + # and increment rconIteration afterwards + if currentSize % size == 0: + t = self.core(t, rconIteration) + rconIteration += 1 + # For 256-bit keys, we add an extra sbox to the calculation + if size == self.keySize["SIZE_256"] and ((currentSize % size) == 16): + for l in range(4): t[l] = self.getSBoxValue(t[l]) + + # We XOR t with the four-byte block 16,24,32 bytes before the new + # expanded key. This becomes the next four bytes in the expanded + # key. + for m in range(4): + expandedKey[currentSize] = expandedKey[currentSize - size] ^ \ + t[m] + currentSize += 1 + + return expandedKey + + def addRoundKey(self, state, roundKey): + """Adds (XORs) the round key to the state.""" + for i in range(16): + state[i] ^= roundKey[i] + return state + + def createRoundKey(self, expandedKey, roundKeyPointer): + """Create a round key. + Creates a round key from the given expanded key and the + position within the expanded key. + """ + roundKey = [0] * 16 + for i in range(4): + for j in range(4): + roundKey[j*4+i] = expandedKey[roundKeyPointer + i*4 + j] + return roundKey + + def galois_multiplication(self, a, b): + """Galois multiplication of 8 bit characters a and b.""" + p = 0 + for counter in range(8): + if b & 1: p ^= a + hi_bit_set = a & 0x80 + a <<= 1 + # keep a 8 bit + a &= 0xFF + if hi_bit_set: + a ^= 0x1b + b >>= 1 + return p + + # + # substitute all the values from the state with the value in the SBox + # using the state value as index for the SBox + # + def subBytes(self, state, isInv): + if isInv: getter = self.getSBoxInvert + else: getter = self.getSBoxValue + for i in range(16): state[i] = getter(state[i]) + return state + + # iterate over the 4 rows and call shiftRow() with that row + def shiftRows(self, state, isInv): + for i in range(4): + state = self.shiftRow(state, i*4, i, isInv) + return state + + # each iteration shifts the row to the left by 1 + def shiftRow(self, state, statePointer, nbr, isInv): + for i in range(nbr): + if isInv: + state[statePointer:statePointer+4] = \ + state[statePointer+3:statePointer+4] + \ + state[statePointer:statePointer+3] + else: + state[statePointer:statePointer+4] = \ + state[statePointer+1:statePointer+4] + \ + state[statePointer:statePointer+1] + return state + + # galois multiplication of the 4x4 matrix + def mixColumns(self, state, isInv): + # iterate over the 4 columns + for i in range(4): + # construct one column by slicing over the 4 rows + column = state[i:i+16:4] + # apply the mixColumn on one column + column = self.mixColumn(column, isInv) + # put the values back into the state + state[i:i+16:4] = column + + return state + + # galois multiplication of 1 column of the 4x4 matrix + def mixColumn(self, column, isInv): + if isInv: mult = [14, 9, 13, 11] + else: mult = [2, 1, 1, 3] + cpy = list(column) + g = self.galois_multiplication + + column[0] = g(cpy[0], mult[0]) ^ g(cpy[3], mult[1]) ^ \ + g(cpy[2], mult[2]) ^ g(cpy[1], mult[3]) + column[1] = g(cpy[1], mult[0]) ^ g(cpy[0], mult[1]) ^ \ + g(cpy[3], mult[2]) ^ g(cpy[2], mult[3]) + column[2] = g(cpy[2], mult[0]) ^ g(cpy[1], mult[1]) ^ \ + g(cpy[0], mult[2]) ^ g(cpy[3], mult[3]) + column[3] = g(cpy[3], mult[0]) ^ g(cpy[2], mult[1]) ^ \ + g(cpy[1], mult[2]) ^ g(cpy[0], mult[3]) + return column + + # applies the 4 operations of the forward round in sequence + def aes_round(self, state, roundKey): + state = self.subBytes(state, False) + state = self.shiftRows(state, False) + state = self.mixColumns(state, False) + state = self.addRoundKey(state, roundKey) + return state + + # applies the 4 operations of the inverse round in sequence + def aes_invRound(self, state, roundKey): + state = self.shiftRows(state, True) + state = self.subBytes(state, True) + state = self.addRoundKey(state, roundKey) + state = self.mixColumns(state, True) + return state + + # Perform the initial operations, the standard round, and the final + # operations of the forward aes, creating a round key for each round + def aes_main(self, state, expandedKey, nbrRounds): + state = self.addRoundKey(state, self.createRoundKey(expandedKey, 0)) + i = 1 + while i < nbrRounds: + state = self.aes_round(state, + self.createRoundKey(expandedKey, 16*i)) + i += 1 + state = self.subBytes(state, False) + state = self.shiftRows(state, False) + state = self.addRoundKey(state, + self.createRoundKey(expandedKey, 16*nbrRounds)) + return state + + # Perform the initial operations, the standard round, and the final + # operations of the inverse aes, creating a round key for each round + def aes_invMain(self, state, expandedKey, nbrRounds): + state = self.addRoundKey(state, + self.createRoundKey(expandedKey, 16*nbrRounds)) + i = nbrRounds - 1 + while i > 0: + state = self.aes_invRound(state, + self.createRoundKey(expandedKey, 16*i)) + i -= 1 + state = self.shiftRows(state, True) + state = self.subBytes(state, True) + state = self.addRoundKey(state, self.createRoundKey(expandedKey, 0)) + return state + + # encrypts a 128 bit input block against the given key of size specified + def encrypt(self, iput, key, size): + output = [0] * 16 + # the number of rounds + nbrRounds = 0 + # the 128 bit block to encode + block = [0] * 16 + # set the number of rounds + if size == self.keySize["SIZE_128"]: nbrRounds = 10 + elif size == self.keySize["SIZE_192"]: nbrRounds = 12 + elif size == self.keySize["SIZE_256"]: nbrRounds = 14 + else: return None + + # the expanded keySize + expandedKeySize = 16*(nbrRounds+1) + + # Set the block values, for the block: + # a0,0 a0,1 a0,2 a0,3 + # a1,0 a1,1 a1,2 a1,3 + # a2,0 a2,1 a2,2 a2,3 + # a3,0 a3,1 a3,2 a3,3 + # the mapping order is a0,0 a1,0 a2,0 a3,0 a0,1 a1,1 ... a2,3 a3,3 + # + # iterate over the columns + for i in range(4): + # iterate over the rows + for j in range(4): + block[(i+(j*4))] = iput[(i*4)+j] + + # expand the key into an 176, 208, 240 bytes key + # the expanded key + expandedKey = self.expandKey(key, size, expandedKeySize) + + # encrypt the block using the expandedKey + block = self.aes_main(block, expandedKey, nbrRounds) + + # unmap the block again into the output + for k in range(4): + # iterate over the rows + for l in range(4): + output[(k*4)+l] = block[(k+(l*4))] + return output + + # decrypts a 128 bit input block against the given key of size specified + def decrypt(self, iput, key, size): + output = [0] * 16 + # the number of rounds + nbrRounds = 0 + # the 128 bit block to decode + block = [0] * 16 + # set the number of rounds + if size == self.keySize["SIZE_128"]: nbrRounds = 10 + elif size == self.keySize["SIZE_192"]: nbrRounds = 12 + elif size == self.keySize["SIZE_256"]: nbrRounds = 14 + else: return None + + # the expanded keySize + expandedKeySize = 16*(nbrRounds+1) + + # Set the block values, for the block: + # a0,0 a0,1 a0,2 a0,3 + # a1,0 a1,1 a1,2 a1,3 + # a2,0 a2,1 a2,2 a2,3 + # a3,0 a3,1 a3,2 a3,3 + # the mapping order is a0,0 a1,0 a2,0 a3,0 a0,1 a1,1 ... a2,3 a3,3 + + # iterate over the columns + for i in range(4): + # iterate over the rows + for j in range(4): + block[(i+(j*4))] = iput[(i*4)+j] + # expand the key into an 176, 208, 240 bytes key + expandedKey = self.expandKey(key, size, expandedKeySize) + # decrypt the block using the expandedKey + block = self.aes_invMain(block, expandedKey, nbrRounds) + # unmap the block again into the output + for k in range(4): + # iterate over the rows + for l in range(4): + output[(k*4)+l] = block[(k+(l*4))] + return output + + +class AESModeOfOperation(object): + + aes = AES() + + # structure of supported modes of operation + modeOfOperation = dict(OFB=0, CFB=1, CBC=2) + + # converts a 16 character string into a number array + def convertString(self, string, start, end, mode): + if end - start > 16: end = start + 16 + if mode == self.modeOfOperation["CBC"]: ar = [0] * 16 + else: ar = [] + + i = start + j = 0 + while len(ar) < end - start: + ar.append(0) + while i < end: + ar[j] = ord(string[i]) + j += 1 + i += 1 + return ar + + # Mode of Operation Encryption + # stringIn - Input String + # mode - mode of type modeOfOperation + # hexKey - a hex key of the bit length size + # size - the bit length of the key + # hexIV - the 128 bit hex Initilization Vector + def encrypt(self, stringIn, mode, key, size, IV): + if len(key) % size: + return None + if len(IV) % 16: + return None + # the AES input/output + plaintext = [] + iput = [0] * 16 + output = [] + ciphertext = [0] * 16 + # the output cipher string + cipherOut = [] + # char firstRound + firstRound = True + if stringIn != None: + for j in range(int(math.ceil(float(len(stringIn))/16))): + start = j*16 + end = j*16+16 + if end > len(stringIn): + end = len(stringIn) + plaintext = self.convertString(stringIn, start, end, mode) + # print 'PT@%s:%s' % (j, plaintext) + if mode == self.modeOfOperation["CFB"]: + if firstRound: + output = self.aes.encrypt(IV, key, size) + firstRound = False + else: + output = self.aes.encrypt(iput, key, size) + for i in range(16): + if len(plaintext)-1 < i: + ciphertext[i] = 0 ^ output[i] + elif len(output)-1 < i: + ciphertext[i] = plaintext[i] ^ 0 + elif len(plaintext)-1 < i and len(output) < i: + ciphertext[i] = 0 ^ 0 + else: + ciphertext[i] = plaintext[i] ^ output[i] + for k in range(end-start): + cipherOut.append(ciphertext[k]) + iput = ciphertext + elif mode == self.modeOfOperation["OFB"]: + if firstRound: + output = self.aes.encrypt(IV, key, size) + firstRound = False + else: + output = self.aes.encrypt(iput, key, size) + for i in range(16): + if len(plaintext)-1 < i: + ciphertext[i] = 0 ^ output[i] + elif len(output)-1 < i: + ciphertext[i] = plaintext[i] ^ 0 + elif len(plaintext)-1 < i and len(output) < i: + ciphertext[i] = 0 ^ 0 + else: + ciphertext[i] = plaintext[i] ^ output[i] + for k in range(end-start): + cipherOut.append(ciphertext[k]) + iput = output + elif mode == self.modeOfOperation["CBC"]: + for i in range(16): + if firstRound: + iput[i] = plaintext[i] ^ IV[i] + else: + iput[i] = plaintext[i] ^ ciphertext[i] + # print 'IP@%s:%s' % (j, iput) + firstRound = False + ciphertext = self.aes.encrypt(iput, key, size) + # always 16 bytes because of the padding for CBC + for k in range(16): + cipherOut.append(ciphertext[k]) + return mode, len(stringIn), cipherOut + + # Mode of Operation Decryption + # cipherIn - Encrypted String + # originalsize - The unencrypted string length - required for CBC + # mode - mode of type modeOfOperation + # key - a number array of the bit length size + # size - the bit length of the key + # IV - the 128 bit number array Initilization Vector + def decrypt(self, cipherIn, originalsize, mode, key, size, IV): + # cipherIn = unescCtrlChars(cipherIn) + if len(key) % size: + return None + if len(IV) % 16: + return None + # the AES input/output + ciphertext = [] + iput = [] + output = [] + plaintext = [0] * 16 + # the output plain text string + stringOut = '' + # char firstRound + firstRound = True + if cipherIn != None: + for j in range(int(math.ceil(float(len(cipherIn))/16))): + start = j*16 + end = j*16+16 + if j*16+16 > len(cipherIn): + end = len(cipherIn) + ciphertext = cipherIn[start:end] + if mode == self.modeOfOperation["CFB"]: + if firstRound: + output = self.aes.encrypt(IV, key, size) + firstRound = False + else: + output = self.aes.encrypt(iput, key, size) + for i in range(16): + if len(output)-1 < i: + plaintext[i] = 0 ^ ciphertext[i] + elif len(ciphertext)-1 < i: + plaintext[i] = output[i] ^ 0 + elif len(output)-1 < i and len(ciphertext) < i: + plaintext[i] = 0 ^ 0 + else: + plaintext[i] = output[i] ^ ciphertext[i] + for k in range(end-start): + stringOut += chr(plaintext[k]) + iput = ciphertext + elif mode == self.modeOfOperation["OFB"]: + if firstRound: + output = self.aes.encrypt(IV, key, size) + firstRound = False + else: + output = self.aes.encrypt(iput, key, size) + for i in range(16): + if len(output)-1 < i: + plaintext[i] = 0 ^ ciphertext[i] + elif len(ciphertext)-1 < i: + plaintext[i] = output[i] ^ 0 + elif len(output)-1 < i and len(ciphertext) < i: + plaintext[i] = 0 ^ 0 + else: + plaintext[i] = output[i] ^ ciphertext[i] + for k in range(end-start): + stringOut += chr(plaintext[k]) + iput = output + elif mode == self.modeOfOperation["CBC"]: + output = self.aes.decrypt(ciphertext, key, size) + for i in range(16): + if firstRound: + plaintext[i] = IV[i] ^ output[i] + else: + plaintext[i] = iput[i] ^ output[i] + firstRound = False + if originalsize is not None and originalsize < end: + for k in range(originalsize-start): + stringOut += chr(plaintext[k]) + else: + for k in range(end-start): + stringOut += chr(plaintext[k]) + iput = ciphertext + return stringOut + + +def encryptData(key, data, mode=AESModeOfOperation.modeOfOperation["CBC"]): + """encrypt `data` using `key` + + `key` should be a string of bytes. + + returned cipher is a string of bytes prepended with the initialization + vector. + + """ + key = map(ord, key) + if mode == AESModeOfOperation.modeOfOperation["CBC"]: + data = append_PKCS7_padding(data) + keysize = len(key) + assert keysize in AES.keySize.values(), 'invalid key size: %s' % keysize + # create a new iv using random data + iv = [ord(i) for i in os.urandom(16)] + moo = AESModeOfOperation() + (mode, length, ciph) = moo.encrypt(data, mode, key, keysize, iv) + # With padding, the original length does not need to be known. It's a bad + # idea to store the original message length. + # prepend the iv. + return ''.join(map(chr, iv)) + ''.join(map(chr, ciph)) + +def decryptData(key, data, mode=AESModeOfOperation.modeOfOperation["CBC"]): + """decrypt `data` using `key` + + `key` should be a string of bytes. + + `data` should have the initialization vector prepended as a string of + ordinal values. + + """ + + key = map(ord, key) + keysize = len(key) + assert keysize in AES.keySize.values(), 'invalid key size: %s' % keysize + # iv is first 16 bytes + iv = map(ord, data[:16]) + data = map(ord, data[16:]) + moo = AESModeOfOperation() + decr = moo.decrypt(data, None, mode, key, keysize, iv) + if mode == AESModeOfOperation.modeOfOperation["CBC"]: + decr = strip_PKCS7_padding(decr) + return decr + +def generateRandomKey(keysize): + """Generates a key from random data of length `keysize`. + + The returned key is a string of bytes. + + """ + if keysize not in (16, 24, 32): + emsg = 'Invalid keysize, %s. Should be one of (16, 24, 32).' + raise ValueError, emsg % keysize + return os.urandom(keysize) + +if __name__ == "__main__": + moo = AESModeOfOperation() + cleartext = "This is a test!" + cypherkey = [143,194,34,208,145,203,230,143,177,246,97,206,145,92,255,84] + iv = [103,35,148,239,76,213,47,118,255,222,123,176,106,134,98,92] + mode, orig_len, ciph = moo.encrypt(cleartext, moo.modeOfOperation["CBC"], + cypherkey, moo.aes.keySize["SIZE_128"], iv) + print 'm=%s, ol=%s (%s), ciph=%s' % (mode, orig_len, len(cleartext), ciph) + decr = moo.decrypt(ciph, orig_len, mode, cypherkey, + moo.aes.keySize["SIZE_128"], iv) + print decr diff --git a/lib/socks.py b/lib/socks.py new file mode 100644 index 00000000..8792c9b5 --- /dev/null +++ b/lib/socks.py @@ -0,0 +1,387 @@ +"""SocksiPy - Python SOCKS module. +Version 1.00 + +Copyright 2006 Dan-Haim. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +3. Neither the name of Dan Haim nor the names of his contributors may be used + to endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY DAN HAIM "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL DAN HAIM OR HIS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMANGE. + + +This module provides a standard socket-like interface for Python +for tunneling connections through SOCKS proxies. + +""" + +import socket +import struct + +PROXY_TYPE_SOCKS4 = 1 +PROXY_TYPE_SOCKS5 = 2 +PROXY_TYPE_HTTP = 3 + +_defaultproxy = None +_orgsocket = socket.socket + +class ProxyError(Exception): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class GeneralProxyError(ProxyError): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class Socks5AuthError(ProxyError): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class Socks5Error(ProxyError): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class Socks4Error(ProxyError): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class HTTPError(ProxyError): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +_generalerrors = ("success", + "invalid data", + "not connected", + "not available", + "bad proxy type", + "bad input") + +_socks5errors = ("succeeded", + "general SOCKS server failure", + "connection not allowed by ruleset", + "Network unreachable", + "Host unreachable", + "Connection refused", + "TTL expired", + "Command not supported", + "Address type not supported", + "Unknown error") + +_socks5autherrors = ("succeeded", + "authentication is required", + "all offered authentication methods were rejected", + "unknown username or invalid password", + "unknown error") + +_socks4errors = ("request granted", + "request rejected or failed", + "request rejected because SOCKS server cannot connect to identd on the client", + "request rejected because the client program and identd report different user-ids", + "unknown error") + +def setdefaultproxy(proxytype=None,addr=None,port=None,rdns=True,username=None,password=None): + """setdefaultproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) + Sets a default proxy which all further socksocket objects will use, + unless explicitly changed. + """ + global _defaultproxy + _defaultproxy = (proxytype,addr,port,rdns,username,password) + +class socksocket(socket.socket): + """socksocket([family[, type[, proto]]]) -> socket object + + Open a SOCKS enabled socket. The parameters are the same as + those of the standard socket init. In order for SOCKS to work, + you must specify family=AF_INET, type=SOCK_STREAM and proto=0. + """ + + def __init__(self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, _sock=None): + _orgsocket.__init__(self,family,type,proto,_sock) + if _defaultproxy != None: + self.__proxy = _defaultproxy + else: + self.__proxy = (None, None, None, None, None, None) + self.__proxysockname = None + self.__proxypeername = None + + def __recvall(self, bytes): + """__recvall(bytes) -> data + Receive EXACTLY the number of bytes requested from the socket. + Blocks until the required number of bytes have been received. + """ + data = "" + while len(data) < bytes: + data = data + self.recv(bytes-len(data)) + return data + + def setproxy(self,proxytype=None,addr=None,port=None,rdns=True,username=None,password=None): + """setproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) + Sets the proxy to be used. + proxytype - The type of the proxy to be used. Three types + are supported: PROXY_TYPE_SOCKS4 (including socks4a), + PROXY_TYPE_SOCKS5 and PROXY_TYPE_HTTP + addr - The address of the server (IP or DNS). + port - The port of the server. Defaults to 1080 for SOCKS + servers and 8080 for HTTP proxy servers. + rdns - Should DNS queries be preformed on the remote side + (rather than the local side). The default is True. + Note: This has no effect with SOCKS4 servers. + username - Username to authenticate with to the server. + The default is no authentication. + password - Password to authenticate with to the server. + Only relevant when username is also provided. + """ + self.__proxy = (proxytype,addr,port,rdns,username,password) + + def __negotiatesocks5(self,destaddr,destport): + """__negotiatesocks5(self,destaddr,destport) + Negotiates a connection through a SOCKS5 server. + """ + # First we'll send the authentication packages we support. + if (self.__proxy[4]!=None) and (self.__proxy[5]!=None): + # The username/password details were supplied to the + # setproxy method so we support the USERNAME/PASSWORD + # authentication (in addition to the standard none). + self.sendall("\x05\x02\x00\x02") + else: + # No username/password were entered, therefore we + # only support connections with no authentication. + self.sendall("\x05\x01\x00") + # We'll receive the server's response to determine which + # method was selected + chosenauth = self.__recvall(2) + if chosenauth[0] != "\x05": + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + # Check the chosen authentication method + if chosenauth[1] == "\x00": + # No authentication is required + pass + elif chosenauth[1] == "\x02": + # Okay, we need to perform a basic username/password + # authentication. + self.sendall("\x01" + chr(len(self.__proxy[4])) + self.__proxy[4] + chr(len(self.proxy[5])) + self.__proxy[5]) + authstat = self.__recvall(2) + if authstat[0] != "\x01": + # Bad response + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + if authstat[1] != "\x00": + # Authentication failed + self.close() + raise Socks5AuthError,((3,_socks5autherrors[3])) + # Authentication succeeded + else: + # Reaching here is always bad + self.close() + if chosenauth[1] == "\xFF": + raise Socks5AuthError((2,_socks5autherrors[2])) + else: + raise GeneralProxyError((1,_generalerrors[1])) + # Now we can request the actual connection + req = "\x05\x01\x00" + # If the given destination address is an IP address, we'll + # use the IPv4 address request even if remote resolving was specified. + try: + ipaddr = socket.inet_aton(destaddr) + req = req + "\x01" + ipaddr + except socket.error: + # Well it's not an IP number, so it's probably a DNS name. + if self.__proxy[3]==True: + # Resolve remotely + ipaddr = None + req = req + "\x03" + chr(len(destaddr)) + destaddr + else: + # Resolve locally + ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) + req = req + "\x01" + ipaddr + req = req + struct.pack(">H",destport) + self.sendall(req) + # Get the response + resp = self.__recvall(4) + if resp[0] != "\x05": + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + elif resp[1] != "\x00": + # Connection failed + self.close() + if ord(resp[1])<=8: + raise Socks5Error(ord(resp[1]),_generalerrors[ord(resp[1])]) + else: + raise Socks5Error(9,_generalerrors[9]) + # Get the bound address/port + elif resp[3] == "\x01": + boundaddr = self.__recvall(4) + elif resp[3] == "\x03": + resp = resp + self.recv(1) + boundaddr = self.__recvall(resp[4]) + else: + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + boundport = struct.unpack(">H",self.__recvall(2))[0] + self.__proxysockname = (boundaddr,boundport) + if ipaddr != None: + self.__proxypeername = (socket.inet_ntoa(ipaddr),destport) + else: + self.__proxypeername = (destaddr,destport) + + def getproxysockname(self): + """getsockname() -> address info + Returns the bound IP address and port number at the proxy. + """ + return self.__proxysockname + + def getproxypeername(self): + """getproxypeername() -> address info + Returns the IP and port number of the proxy. + """ + return _orgsocket.getpeername(self) + + def getpeername(self): + """getpeername() -> address info + Returns the IP address and port number of the destination + machine (note: getproxypeername returns the proxy) + """ + return self.__proxypeername + + def __negotiatesocks4(self,destaddr,destport): + """__negotiatesocks4(self,destaddr,destport) + Negotiates a connection through a SOCKS4 server. + """ + # Check if the destination address provided is an IP address + rmtrslv = False + try: + ipaddr = socket.inet_aton(destaddr) + except socket.error: + # It's a DNS name. Check where it should be resolved. + if self.__proxy[3]==True: + ipaddr = "\x00\x00\x00\x01" + rmtrslv = True + else: + ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) + # Construct the request packet + req = "\x04\x01" + struct.pack(">H",destport) + ipaddr + # The username parameter is considered userid for SOCKS4 + if self.__proxy[4] != None: + req = req + self.__proxy[4] + req = req + "\x00" + # DNS name if remote resolving is required + # NOTE: This is actually an extension to the SOCKS4 protocol + # called SOCKS4A and may not be supported in all cases. + if rmtrslv==True: + req = req + destaddr + "\x00" + self.sendall(req) + # Get the response from the server + resp = self.__recvall(8) + if resp[0] != "\x00": + # Bad data + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + if resp[1] != "\x5A": + # Server returned an error + self.close() + if ord(resp[1]) in (91,92,93): + self.close() + raise Socks4Error((ord(resp[1]),_socks4errors[ord(resp[1])-90])) + else: + raise Socks4Error((94,_socks4errors[4])) + # Get the bound address/port + self.__proxysockname = (socket.inet_ntoa(resp[4:]),struct.unpack(">H",resp[2:4])[0]) + if rmtrslv != None: + self.__proxypeername = (socket.inet_ntoa(ipaddr),destport) + else: + self.__proxypeername = (destaddr,destport) + + def __negotiatehttp(self,destaddr,destport): + """__negotiatehttp(self,destaddr,destport) + Negotiates a connection through an HTTP server. + """ + # If we need to resolve locally, we do this now + if self.__proxy[3] == False: + addr = socket.gethostbyname(destaddr) + else: + addr = destaddr + self.sendall("CONNECT " + addr + ":" + str(destport) + " HTTP/1.1\r\n" + "Host: " + destaddr + "\r\n\r\n") + # We read the response until we get the string "\r\n\r\n" + resp = self.recv(1) + while resp.find("\r\n\r\n")==-1: + resp = resp + self.recv(1) + # We just need the first line to check if the connection + # was successful + statusline = resp.splitlines()[0].split(" ",2) + if statusline[0] not in ("HTTP/1.0","HTTP/1.1"): + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + try: + statuscode = int(statusline[1]) + except ValueError: + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + if statuscode != 200: + self.close() + raise HTTPError((statuscode,statusline[2])) + self.__proxysockname = ("0.0.0.0",0) + self.__proxypeername = (addr,destport) + + def connect(self,destpair): + """connect(self,despair) + Connects to the specified destination through a proxy. + destpar - A tuple of the IP/DNS address and the port number. + (identical to socket's connect). + To select the proxy server use setproxy(). + """ + # Do a minimal input check first + if (type(destpair) in (list,tuple)==False) or (len(destpair)<2) or (type(destpair[0])!=str) or (type(destpair[1])!=int): + raise GeneralProxyError((5,_generalerrors[5])) + if self.__proxy[0] == PROXY_TYPE_SOCKS5: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 1080 + _orgsocket.connect(self,(self.__proxy[1],portnum)) + self.__negotiatesocks5(destpair[0],destpair[1]) + elif self.__proxy[0] == PROXY_TYPE_SOCKS4: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 1080 + _orgsocket.connect(self,(self.__proxy[1],portnum)) + self.__negotiatesocks4(destpair[0],destpair[1]) + elif self.__proxy[0] == PROXY_TYPE_HTTP: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 8080 + _orgsocket.connect(self,(self.__proxy[1],portnum)) + self.__negotiatehttp(destpair[0],destpair[1]) + elif self.__proxy[0] == None: + _orgsocket.connect(self,(destpair[0],destpair[1])) + else: + raise GeneralProxyError((4,_generalerrors[4])) diff --git a/wallet-tool.py b/wallet-tool.py index 7be78837..8eeda9d5 100644 --- a/wallet-tool.py +++ b/wallet-tool.py @@ -8,7 +8,7 @@ import bitcoin as btc from common import Wallet, load_program_config, get_addr_vbyte import common -import old_mnemonic +import old_mnemonic, slowaes #structure for cj market wallet # m/0/ root key @@ -90,11 +90,6 @@ total_balance += balance_depth print 'total balance = %.8fbtc' % (total_balance/1e8) elif method == 'generate' or method == 'recover': - try: - import aes - except ImportError: - print 'You must install slowaes\nTry running: sudo pip install slowaes' - sys.exit(0) if method == 'generate': seed = btc.sha256(os.urandom(64))[:32] words = old_mnemonic.mn_encode(seed) @@ -110,7 +105,7 @@ print 'ERROR. Passwords did not match' sys.exit(0) password_key = btc.bin_dbl_sha256(password) - encrypted_seed = aes.encryptData(password_key, seed.decode('hex')) + encrypted_seed = slowaes.encryptData(password_key, seed.decode('hex')) timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S") walletfile = json.dumps({'creator': 'joinmarket project', 'creation_time': timestamp, 'encrypted_seed': encrypted_seed.encode('hex'), 'network': common.get_network()}) From f73aae9e8fb4f0f2edc5cfc3b982a487a79866d9 Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Tue, 26 May 2015 01:07:59 +0100 Subject: [PATCH 290/409] irc lib now authenticates with SASL after redisconnection --- lib/irc.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/irc.py b/lib/irc.py index 172a81ca..ab3b1f19 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -440,7 +440,7 @@ def __init__(self, given_nick, username='username', realname='realname', passwor self.userrealname = (username, realname) if password and len(password) == 0: password = None - self.password = password + self.given_password = password def run(self): self.connect_attempts = 0 @@ -464,7 +464,8 @@ def run(self): self.sock = ssl.wrap_socket(self.sock) self.fd = self.sock.makefile() self.sock.connect(self.serverport) - if self.password: + if self.given_password: + self.password = self.given_password self.send_raw('CAP REQ :sasl') self.send_raw('USER %s b c :%s' % self.userrealname) self.send_raw('NICK ' + self.given_nick) From e7737c19cdc9e01a86b26041fc98d79916d8fda1 Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Tue, 26 May 2015 01:12:44 +0100 Subject: [PATCH 291/409] enabled the irc ping thread, it has been mistakenly disabled --- lib/irc.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/irc.py b/lib/irc.py index ab3b1f19..901c5f8d 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -1,4 +1,4 @@ - +# from common import * from message_channel import MessageChannel from message_channel import CJPeerError @@ -51,13 +51,13 @@ def run(self): if not self.irc.ping_reply: debug('irc ping timed out') try: self.irc.close() - except IOError: pass + except: pass try: self.irc.fd.close() - except IOError: pass + except: pass try: self.irc.sock.shutdown(socket.SHUT_RDWR) self.irc.sock.close() - except IOError: pass + except: pass except IOError as e: debug('ping thread: ' + repr(e)) debug('ended ping thread') @@ -449,7 +449,7 @@ def run(self): self.give_up = False self.ping_reply = True self.lockcond = threading.Condition() - #PingThread(self).start() + PingThread(self).start() while self.connect_attempts < 10 and not self.give_up: try: From dbe4f07e7b32efc43b40486865ae21fd351203bc Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Tue, 26 May 2015 01:18:02 +0100 Subject: [PATCH 292/409] made the irc client ping less often and with a longer timeout, for lower latetncy networks like tor and i2p --- lib/irc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/irc.py b/lib/irc.py index 901c5f8d..97a1f51b 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -9,8 +9,8 @@ MAX_PRIVMSG_LEN = 400 COMMAND_PREFIX = '!' -PING_INTERVAL = 40 -PING_TIMEOUT = 10 +PING_INTERVAL = 180 +PING_TIMEOUT = 30 encrypted_commands = ["auth", "ioauth", "tx", "sig"] plaintext_commands = ["fill", "error", "pubkey", "orderbook", "relorder", "absorder", "push"] From feacc7868e67c6d9c858975134709d728b67b17f Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Tue, 26 May 2015 01:22:54 +0100 Subject: [PATCH 293/409] fixed bug for when password=None --- lib/irc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/irc.py b/lib/irc.py index 97a1f51b..925b62e2 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -464,6 +464,7 @@ def run(self): self.sock = ssl.wrap_socket(self.sock) self.fd = self.sock.makefile() self.sock.connect(self.serverport) + self.password = None if self.given_password: self.password = self.given_password self.send_raw('CAP REQ :sasl') From e57b88e9e5627cec59af9dc825372384d6b35c9a Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Wed, 27 May 2015 01:48:48 +0100 Subject: [PATCH 294/409] fixed subtle bug when unconfirmed utxo was spent --- lib/blockchaininterface.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/blockchaininterface.py b/lib/blockchaininterface.py index 558f00cb..7dc0af11 100644 --- a/lib/blockchaininterface.py +++ b/lib/blockchaininterface.py @@ -293,7 +293,13 @@ def do_HEAD(self): if unconfirmfun == None: common.debug('txid=' + txid + ' not being listened for') else: - txdata = json.loads(self.btcinterface.rpc(['gettxout', txid, '0', 'true'])) + jsonstr = None #on rare occasions people spend their output without waiting for a confirm + for n in range(len(txd['outs'])): + jsonstr = self.btcinterface.rpc(['gettxout', txid, str(n), 'true']) + if jsonstr != '': + break + assert jsonstr != '' + txdata = json.loads(jsonstr) if txdata['confirmations'] == 0: unconfirmfun(txd, txid) #TODO pass the total transfered amount value here somehow From 8767127e775412ca154e5abea5269d9012c0bd10 Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Wed, 27 May 2015 23:21:39 +0100 Subject: [PATCH 295/409] fixed bug where it wouldnt detect nick was already in use if halfway through sasl login --- lib/irc.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/lib/irc.py b/lib/irc.py index 925b62e2..316ed5a8 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -354,6 +354,17 @@ def __handle_line(self, line): return chunks = line.split(' ') + if chunks[1] == 'QUIT': + nick = get_irc_nick(chunks[0]) + if nick == self.nick: + raise IOError('we quit') + else: + if self.on_nick_leave: + self.on_nick_leave(nick) + elif chunks[1] == '433': #nick in use + #self.nick = random_nick() + self.nick += '_' #helps keep identity constant if just _ added + self.send_raw('NICK ' + self.nick) if self.password: if chunks[1] == 'CAP': if chunks[3] != 'ACK': @@ -382,12 +393,8 @@ def __handle_line(self, line): if self.on_connect: self.on_connect() self.send_raw('JOIN ' + self.channel) - self.send_raw('MODE ' + self.given_nick + ' +B') #marks as bots on unreal - self.send_raw('MODE ' + self.given_nick + ' -R') #allows unreg'd private messages - elif chunks[1] == '433': #nick in use - #self.nick = random_nick() - self.nick += '_' #helps keep identity constant if just _ added - self.send_raw('NICK ' + self.nick) + self.send_raw('MODE ' + self.nick + ' +B') #marks as bots on unreal + self.send_raw('MODE ' + self.nick + ' -R') #allows unreg'd private messages elif chunks[1] == '366': #end of names list self.connect_attempts = 0 if self.on_welcome: @@ -396,13 +403,6 @@ def __handle_line(self, line): elif chunks[1] == '332' or chunks[1] == 'TOPIC': #channel topic topic = get_irc_text(line) self.on_set_topic(topic) - elif chunks[1] == 'QUIT': - nick = get_irc_nick(chunks[0]) - if nick == self.nick: - raise IOError('we quit') - else: - if self.on_nick_leave: - self.on_nick_leave(nick) elif chunks[1] == 'KICK': target = chunks[3] nick = get_irc_nick(chunks[0]) @@ -469,7 +469,8 @@ def run(self): self.password = self.given_password self.send_raw('CAP REQ :sasl') self.send_raw('USER %s b c :%s' % self.userrealname) - self.send_raw('NICK ' + self.given_nick) + self.nick = self.given_nick + self.send_raw('NICK ' + self.nick) while 1: try: line = self.fd.readline() From fc238e6a28f3cc3a29c5501433326bef51b1e83f Mon Sep 17 00:00:00 2001 From: Martino Date: Thu, 28 May 2015 12:11:30 +0200 Subject: [PATCH 296/409] Sort the orderbook by cjfee --- ob-watcher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ob-watcher.py b/ob-watcher.py index c2a4c60c..b43cf5e3 100644 --- a/ob-watcher.py +++ b/ob-watcher.py @@ -106,8 +106,8 @@ def __init__(self, request, client_address, base_server): def create_orderbook_table(self): result = '' rows = self.taker.db.execute('SELECT * FROM orderbook;').fetchall() - for o in rows: - result += ' \n' + for o in sorted(rows,key=lambda x: x['cjfee']): + result += ' \n' order_keys_display = (('ordertype', ordertype_display), ('counterparty', do_nothing), ('oid', order_str), ('cjfee', cjfee_display), ('txfee', satoshi_to_unit), ('minsize', satoshi_to_unit), ('maxsize', satoshi_to_unit)) From 117d09893ad6cf3fc0f864d1d5fc7e60439ce2f4 Mon Sep 17 00:00:00 2001 From: Martino Date: Thu, 28 May 2015 12:13:52 +0200 Subject: [PATCH 297/409] Sork the orderbook by cjfee --- ob-watcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ob-watcher.py b/ob-watcher.py index c2a4c60c..3399d63d 100644 --- a/ob-watcher.py +++ b/ob-watcher.py @@ -106,7 +106,7 @@ def __init__(self, request, client_address, base_server): def create_orderbook_table(self): result = '' rows = self.taker.db.execute('SELECT * FROM orderbook;').fetchall() - for o in rows: + for o in sorted(rows,key=lambda x: x['cjfee']): result += ' \n' order_keys_display = (('ordertype', ordertype_display), ('counterparty', do_nothing), ('oid', order_str), ('cjfee', cjfee_display), ('txfee', satoshi_to_unit), From 39531b34d691d38f9fd82b1e180de9cb8f75b9be Mon Sep 17 00:00:00 2001 From: K1773R Date: Thu, 28 May 2015 12:26:14 +0200 Subject: [PATCH 298/409] enable line buffering for the logfile --- lib/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/common.py b/lib/common.py index 6f328bd3..ceac9865 100644 --- a/lib/common.py +++ b/lib/common.py @@ -80,7 +80,7 @@ def debug(msg): global debug_file_handle with debug_file_lock: if nickname and not debug_file_handle: - debug_file_handle = open(os.path.join('logs', nickname+'.log'),'ab') + debug_file_handle = open(os.path.join('logs', nickname+'.log'),'ab',1) outmsg = datetime.datetime.now().strftime("[%Y/%m/%d %H:%M:%S] ") + msg if core_alert: print 'Core Alert Message: ' + core_alert From 3d66769d2aa63d62414dd9629344625067ecb6ce Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Fri, 29 May 2015 00:43:10 +0100 Subject: [PATCH 299/409] created BitcoinCoreWallet for coinjoining by querying bitcoin core instead of the joinmarket internal wallet --- lib/blockchaininterface.py | 4 ++ lib/common.py | 89 ++++++++++++++++++++++++++++---------- patientsendpayment.py | 18 +++++--- sendpayment.py | 18 +++++--- tumbler.py | 1 - yield-generator.py | 1 - 6 files changed, 93 insertions(+), 38 deletions(-) diff --git a/lib/blockchaininterface.py b/lib/blockchaininterface.py index 558f00cb..a920bf6f 100644 --- a/lib/blockchaininterface.py +++ b/lib/blockchaininterface.py @@ -369,6 +369,8 @@ def add_watchonly_addresses(self, addr_list, wallet_name): sys.exit(0) def sync_addresses(self, wallet): + if isinstance(wallet, common.BitcoinCoreWallet): + return common.debug('requesting wallet history') wallet_name = self.get_wallet_name(wallet) addr_req_count = 50 @@ -424,6 +426,8 @@ def sync_addresses(self, wallet): return def sync_unspent(self, wallet): + if isinstance(wallet, common.BitcoinCoreWallet): + return st = time.time() wallet_name = self.get_wallet_name(wallet) wallet.unspent = {} diff --git a/lib/common.py b/lib/common.py index f9156224..d00ff600 100644 --- a/lib/common.py +++ b/lib/common.py @@ -126,8 +126,47 @@ def debug_dump_object(obj, skip_fields=[]): else: debug(str(v)) -class Wallet(object): +class AbstractWallet(object): + ''' + Abstract wallet for use with JoinMarket + Mostly written with Wallet in mind, the default JoinMarket HD wallet + ''' + def __init__(self): + pass + def get_key_from_addr(self, addr): + return None + def get_utxos_by_mixdepth(self): + return None + def get_change_addr(self, mixing_depth): + return None + def update_cache_index(self): + pass + def remove_old_utxos(self, tx): + pass + def add_new_utxos(self, tx, txid): + pass + + def select_utxos(self, mixdepth, amount): + utxo_list = self.get_utxos_by_mixdepth()[mixdepth] + unspent = [{'utxo': utxo, 'value': addrval['value']} + for utxo, addrval in utxo_list.iteritems()] + inputs = btc.select(unspent, amount) + debug('for mixdepth=' + str(mixdepth) + ' amount=' + str(amount) + ' selected:') + debug(pprint.pformat(inputs)) + return dict([(i['utxo'], {'value': i['value'], 'address': + utxo_list[i['utxo']]['address']}) for i in inputs]) + + def get_balance_by_mixdepth(self): + mix_balance = {} + for m in range(self.max_mix_depth): + mix_balance[m] = 0 + for mixdepth, utxos in self.get_utxos_by_mixdepth().iteritems(): + mix_balance[mixdepth] = sum([addrval['value'] for addrval in utxos.values()]) + return mix_balance + +class Wallet(AbstractWallet): def __init__(self, seedarg, max_mix_depth=2, gaplimit=6): + super(Wallet, self).__init__() self.max_mix_depth = max_mix_depth self.gaplimit = gaplimit seed = self.get_seed(seedarg) @@ -263,30 +302,32 @@ def get_utxos_by_mixdepth(self): mix_utxo_list[mixdepth][utxo] = addrvalue return mix_utxo_list - def get_balance_by_mixdepth(self): - mix_balance = {} - for m in range(self.max_mix_depth): - mix_balance[m] = 0 - for mixdepth, utxos in self.get_utxos_by_mixdepth().iteritems(): - mix_balance[mixdepth] = sum([addrval['value'] for addrval in utxos.values()]) - return mix_balance +class BitcoinCoreWallet(AbstractWallet): + def __init__(self, fromaccount): + super(BitcoinCoreWallet, self).__init__() + if not isinstance(bc_interface, blockchaininterface.BitcoinCoreInterface): + raise RuntimeError('Bitcoin Core wallet can only be used when blockchain interface is BitcoinCoreInterface') + self.fromaccount = fromaccount + self.max_mix_depth = 1 - def select_utxos(self, mixdepth, amount): - utxo_list = self.get_utxos_by_mixdepth()[mixdepth] - unspent = [{'utxo': utxo, 'value': self.unspent[utxo]['value']} - for utxo in utxo_list] - inputs = btc.select(unspent, amount) - debug('for mixdepth=' + str(mixdepth) + ' amount=' + str(amount) + ' selected:') - debug(pprint.pformat(inputs)) - return dict([(i['utxo'], {'value': i['value'], 'address': - self.unspent[i['utxo']]['address']}) for i in inputs]) - - def print_debug_wallet_info(self): - debug('printing debug wallet information') - debug('utxos') - debug(pprint.pformat(self.unspent)) - debug('wallet.index') - debug(pprint.pformat(self.index)) + def get_key_from_addr(self, addr): + return bc_interface.rpc(['dumpprivkey', addr]).strip() + + def get_utxos_by_mixdepth(self): + ret = bc_interface.rpc(['listunspent']) + unspent_list = json.loads(ret) + result = {0: {}} + for u in unspent_list: + if not u['spendable']: + continue + if self.fromaccount and 'account' in u and u['account'] != self.fromaccount: + continue + result[0][u['txid'] + ':' + str(u['vout'])] = {'address': u['address'], + 'value': int(Decimal(str(u['amount'])) * Decimal('1e8'))} + return result + + def get_change_addr(self, mixing_depth): + return bc_interface.rpc(['getrawchangeaddress']).strip() def calc_cj_fee(ordertype, cjfee, cj_amount): real_cjfee = None diff --git a/patientsendpayment.py b/patientsendpayment.py index e3bdd5bf..2d01f186 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -110,7 +110,7 @@ def on_tx_confirmed(self, cjorder, confirmations, txid, balance): def main(): - parser = OptionParser(usage='usage: %prog [options] [wallet file / seed] [amount] [destaddr]', + parser = OptionParser(usage='usage: %prog [options] [wallet file / fromaccount] [amount] [destaddr]', description='Sends a payment from your wallet to an given address' + ' using coinjoin. First acts as a maker, announcing an order and ' + 'waiting for someone to fill it. After a set period of time, gives' + @@ -125,12 +125,15 @@ def main(): help='coinjoin fee asked for when being a maker, in satoshis per order filled, default=50000', default=50000) parser.add_option('-m', '--mixdepth', action='store', type='int', dest='mixdepth', help='mixing depth to spend from, default=0', default=0) + parser.add_option('--rpcwallet', action='store_true', dest='userpcwallet', default=False, + help='Use the Bitcoin Core wallet through json rpc, instead of the internal joinmarket ' + + 'wallet. Requires blockchain_source=json-rpc') (options, args) = parser.parse_args() if len(args) < 3: - parser.error('Needs a seed, amount and destination address') + parser.error('Needs a wallet, amount and destination address') sys.exit(0) - seed = args[0] + wallet_name = args[0] amount = int(args[1]) destaddr = args[2] @@ -144,7 +147,12 @@ def main(): print 'txfee=%d cjfee=%d waittime=%s makercount=%d' % (options.txfee, options.cjfee, str(timedelta(hours=options.waittime)), options.makercount) - wallet = Wallet(seed, options.mixdepth + 1) + if not options.userpcwallet: + wallet = Wallet(wallet_name, options.mixdepth + 1) + else: + print 'not implemented yet' + sys.exit(0) + wallet = BitcoinCoreWallet(fromaccount = wallet_name) common.bc_interface.sync_wallet(wallet) available_balance = wallet.get_balance_by_mixdepth()[options.mixdepth] @@ -162,7 +170,7 @@ def main(): irc.run() except: debug('CRASHING, DUMPING EVERYTHING') - debug_dump_object(wallet, ['addr_cache', 'keys', 'seed']) + debug_dump_object(wallet, ['addr_cache', 'keys']) debug_dump_object(taker) import traceback traceback.print_exc() diff --git a/sendpayment.py b/sendpayment.py index 3fffc2da..bf1bbaae 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -93,7 +93,7 @@ def on_welcome(self): PaymentThread(self).start() def main(): - parser = OptionParser(usage='usage: %prog [options] [wallet file / seed] [amount] [destaddr]', + parser = OptionParser(usage='usage: %prog [options] [wallet file / fromaccount] [amount] [destaddr]', description='Sends a single payment from the zero mixing depth of your ' + 'wallet to an given address using coinjoin and then switches off. ' + 'Setting amount to zero will do a sweep, where the entire mix depth is emptied') @@ -111,12 +111,15 @@ def main(): help='mixing depth to spend from, default=0', default=0) parser.add_option('--yes', action='store_true', dest='answeryes', default=False, help='answer yes to everything') + parser.add_option('--rpcwallet', action='store_true', dest='userpcwallet', default=False, + help='Use the Bitcoin Core wallet through json rpc, instead of the internal joinmarket ' + + 'wallet. Requires blockchain_source=json-rpc') (options, args) = parser.parse_args() if len(args) < 3: - parser.error('Needs a seed, amount and destination address') + parser.error('Needs a wallet, amount and destination address') sys.exit(0) - seed = args[0] + wallet_name = args[0] amount = int(args[1]) destaddr = args[2] @@ -129,9 +132,11 @@ def main(): common.nickname = random_nick() debug('starting sendpayment') - wallet = Wallet(seed, options.mixdepth + 1) + if not options.userpcwallet: + wallet = Wallet(wallet_name, options.mixdepth + 1) + else: + wallet = BitcoinCoreWallet(fromaccount = wallet_name) common.bc_interface.sync_wallet(wallet) - wallet.print_debug_wallet_info() irc = IRCMessageChannel(common.nickname) taker = SendPayment(irc, wallet, destaddr, amount, options.makercount, options.txfee, @@ -141,8 +146,7 @@ def main(): irc.run() except: debug('CRASHING, DUMPING EVERYTHING') - debug('wallet seed = ' + seed) - debug_dump_object(wallet, ['addr_cache', 'keys', 'seed']) + debug_dump_object(wallet, ['addr_cache', 'keys', 'wallet_name']) debug_dump_object(taker) import traceback debug(traceback.format_exc()) diff --git a/tumbler.py b/tumbler.py index 9b7ca491..7372818e 100644 --- a/tumbler.py +++ b/tumbler.py @@ -314,7 +314,6 @@ def main(): #python tumbler.py -N 2 1 -c 3 0.001 -l 0.1 -M 3 -a 0 seed 1xxx 1yyy wallet = Wallet(seed, max_mix_depth = options.mixdepthsrc + options.mixdepthcount) common.bc_interface.sync_wallet(wallet) - wallet.print_debug_wallet_info() common.nickname = random_nick() debug('starting tumbler') diff --git a/yield-generator.py b/yield-generator.py index e45e7aff..6407c1f7 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -129,7 +129,6 @@ def main(): wallet = Wallet(seed, max_mix_depth = mix_levels) common.bc_interface.sync_wallet(wallet) - wallet.print_debug_wallet_info() common.nickname = nickname debug('starting yield generator') From 47e737a29cd5a9dabc8c42260d9f3022916c5fa2 Mon Sep 17 00:00:00 2001 From: Martino Salvetti Date: Fri, 29 May 2015 02:04:51 +0200 Subject: [PATCH 300/409] Put absorders first. I confess: I didn't test --- ob-watcher.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ob-watcher.py b/ob-watcher.py index 3399d63d..9feae1bf 100644 --- a/ob-watcher.py +++ b/ob-watcher.py @@ -106,7 +106,8 @@ def __init__(self, request, client_address, base_server): def create_orderbook_table(self): result = '' rows = self.taker.db.execute('SELECT * FROM orderbook;').fetchall() - for o in sorted(rows,key=lambda x: x['cjfee']): + ordersorder = ['absorder','relorder'] + for o in sorted(rows,cmp=lambda x,y: cmp(x['cjfee'],y['cjfee']) if x['ordertype']==y['ordertype'] else cmp(ordersorder.index(x['ordertype']),ordersorder.index(y['ordertype']))): result += ' \n' order_keys_display = (('ordertype', ordertype_display), ('counterparty', do_nothing), ('oid', order_str), ('cjfee', cjfee_display), ('txfee', satoshi_to_unit), From 3bf8fbad7b04a54612334e865668c7942bd92ef8 Mon Sep 17 00:00:00 2001 From: Anduck Date: Sat, 30 May 2015 02:21:13 +0300 Subject: [PATCH 301/409] bugfix for pick_order (-P) --- lib/common.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/common.py b/lib/common.py index 6ac1d268..ed3846de 100644 --- a/lib/common.py +++ b/lib/common.py @@ -381,11 +381,14 @@ def pick_order(orders, n, feekey): print("Only one possible pick, picking it.") return orders[0] while pickedOrderIndex == -1: - pickedOrderIndex = raw_input('Pick an order between 0 and '+str(i)+': ') - if pickedOrderIndex!='' and pickedOrderIndex[0].isdigit(): - pickedOrderIndex = int(pickedOrderIndex[0]) - if pickedOrderIndex>=0 and pickedOrderIndex=0 and pickedOrderIndex Date: Sun, 31 May 2015 20:06:25 +0100 Subject: [PATCH 302/409] removed irc connect_attempts limit --- lib/irc.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/irc.py b/lib/irc.py index 316ed5a8..19bcec91 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -396,7 +396,6 @@ def __handle_line(self, line): self.send_raw('MODE ' + self.nick + ' +B') #marks as bots on unreal self.send_raw('MODE ' + self.nick + ' -R') #allows unreg'd private messages elif chunks[1] == '366': #end of names list - self.connect_attempts = 0 if self.on_welcome: self.on_welcome() debug('Connected to IRC and joined channel') @@ -443,7 +442,6 @@ def __init__(self, given_nick, username='username', realname='realname', passwor self.given_password = password def run(self): - self.connect_attempts = 0 self.waiting = {} self.built_privmsg = {} self.give_up = False @@ -451,7 +449,7 @@ def run(self): self.lockcond = threading.Condition() PingThread(self).start() - while self.connect_attempts < 10 and not self.give_up: + while not self.give_up: try: debug('connecting') if config.get("MESSAGING","socks5").lower() == 'true': @@ -494,8 +492,8 @@ def run(self): if self.on_disconnect: self.on_disconnect() debug('disconnected irc') - time.sleep(10) - self.connect_attempts += 1 + if not self.give_up: + time.sleep(30) debug('ending irc') self.give_up = True From 218b796c45bea9177649fcc625e38dc414231c22 Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Sun, 31 May 2015 20:46:25 +0100 Subject: [PATCH 303/409] added comments to a confusing crypto part --- lib/taker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/taker.py b/lib/taker.py index c64ff3d3..e124f72d 100644 --- a/lib/taker.py +++ b/lib/taker.py @@ -53,6 +53,7 @@ def auth_counterparty(self, nick, btc_sig, cj_pub): '''Validate the counterpartys claim to own the btc address/pubkey that will be used for coinjoining with an ecdsa verification.''' + #crypto_boxes[nick][0] = maker_pubkey if not btc.ecdsa_verify(self.crypto_boxes[nick][0], btc_sig, cj_pub): print 'signature didnt match pubkey and message' return False @@ -246,7 +247,7 @@ def __init__(self, msgchan): #maybe a start_cj_tx() method is needed def get_crypto_box_from_nick(self, nick): - return self.cjtx.crypto_boxes[nick][1] + return self.cjtx.crypto_boxes[nick][1] #libsodium encryption object def start_cj(self, wallet, cj_amount, orders, input_utxos, my_cj_addr, my_change_addr, my_txfee, finishcallback=None): From 3267b228a6fbb6187bd3de42dc6bb9ce9bffe5c6 Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Sun, 31 May 2015 21:03:09 +0100 Subject: [PATCH 304/409] removed some debug statements to make the log less spammy --- lib/irc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/irc.py b/lib/irc.py index 19bcec91..223fdcec 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -167,7 +167,7 @@ def __privmsg(self, nick, cmd, message): def send_raw(self, line): #if not line.startswith('PING LAG'): - debug('sendraw ' + line) + # debug('sendraw ' + line) self.sock.sendall(line + '\r\n') def check_for_orders(self, nick, chunks): @@ -335,20 +335,20 @@ def __handle_privmsg(self, source, target, message): parsed = self.built_privmsg[nick][1] #wipe the message buffer waiting for the next one del self.built_privmsg[nick] - #debug("< Date: Wed, 3 Jun 2015 23:05:20 +0100 Subject: [PATCH 305/409] fixed bugs in tumbler and made error messager clearer --- lib/common.py | 6 +++--- tumbler.py | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/common.py b/lib/common.py index ed3846de..162b6938 100644 --- a/lib/common.py +++ b/lib/common.py @@ -349,7 +349,7 @@ def weighted_order_choose(orders, n, feekey): unless M < orderbook size, then phi goes up to the last order ''' minfee = feekey(orders[0]) - M = n + M = 1.5*n if len(orders) > M: phi = feekey(orders[M]) - minfee else: @@ -398,8 +398,8 @@ def choose_order(db, cj_amount, n, chooseOrdersBy): for o in sqlorders if cj_amount >= o['minsize'] and cj_amount <= o['maxsize']] counterparties = set([o[0] for o in orders]) if n > len(counterparties): - debug('ERROR not enough liquidity in the orderbook n=%d suitable-counterparties=%d' - % (n, len(counterparties))) + debug('ERROR not enough liquidity in the orderbook n=%d suitable-counterparties=%d amount=%d totalorders=%d' + % (n, len(counterparties), cj_amount), len(orders)) return None, 0 #TODO handle not enough liquidity better, maybe an Exception orders = sorted(orders, key=lambda k: k[2]) #sort from smallest to biggest cj fee debug('considered orders = ' + str(orders)) diff --git a/tumbler.py b/tumbler.py index 7372818e..01142d40 100644 --- a/tumbler.py +++ b/tumbler.py @@ -12,7 +12,7 @@ import numpy as np from pprint import pprint -orderwaittime = 5 +orderwaittime = 10 def lower_bounded_int(thelist, lowerbound): return [int(l) if int(l) >= lowerbound else lowerbound for l in thelist] @@ -225,14 +225,14 @@ def main(): parser.add_option('-f', '--txfee', type='int', dest='txfee', default=10000, help='miner fee contribution, in satoshis, default=10000') parser.add_option('-x', '--maxcjfee', type='float', dest='maxcjfee', - default=0.03, help='maximum coinjoin fee the tumbler is willing to pay to a single market maker. default=0.03 (3%)') + default=0.03, help='maximum coinjoin fee the tumbler is willing to pay to a single market maker. default=0.01 (1%)') parser.add_option('-a', '--addrask', type='int', dest='addrask', default=2, help='How many more addresses to ask for in the terminal. Should ' 'be similar to --txcountparams. default=2') parser.add_option('-N', '--makercountrange', type='float', nargs=2, action='store', dest='makercountrange', - help='Input the range of makers to use. e.g. 3-5 will random use between ' - '3 and 5 makers inclusive, default=3 4', default=(3, 1)) + help='Input the mean and spread of number of makers to use. e.g. 3 1.5 will be a normal distribution ' + 'with mean 3 and standard deveation 1.5 inclusive, default=3 1.5', default=(3, 1.5)) parser.add_option('-M', '--mixdepthcount', type='int', dest='mixdepthcount', help='How many mixing depths to mix through', default=4) parser.add_option('-c', '--txcountparams', type='float', nargs=2, dest='txcountparams', default=(5, 1), @@ -247,8 +247,8 @@ def main(): ' events. default=20') parser.add_option('-w', '--wait-time', action='store', type='float', dest='waittime', help='wait time in seconds to allow orders to arrive, default=5', default=5) - parser.add_option('-s', '--mincjamount', type='float', dest='mincjamount', default=0.0001, - help='minimum coinjoin amount in transaction') + parser.add_option('-s', '--mincjamount', type='int', dest='mincjamount', default=100000, + help='minimum coinjoin amount in transaction in satoshi, default 100k') (options, args) = parser.parse_args() #TODO somehow implement a lower limit From fffc09c7a8cc7a841625113e0bd22cf4d4d5b164 Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Thu, 4 Jun 2015 00:24:43 +0100 Subject: [PATCH 306/409] changed prints to debugs so theyre saved to the log file --- tumbler.py | 47 ++++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/tumbler.py b/tumbler.py index 01142d40..f5afeff9 100644 --- a/tumbler.py +++ b/tumbler.py @@ -77,7 +77,7 @@ def __init__(self, taker): self.taker = taker def unconfirm_callback(self, txd, txid): - print 'that was %d tx out of %d' % (self.current_tx+1, len(self.taker.tx_list)) + debug('that was %d tx out of %d' % (self.current_tx+1, len(self.taker.tx_list))) def confirm_callback(self, txd, txid, confirmations): self.taker.wallet.add_new_utxos(txd, txid) @@ -105,20 +105,20 @@ def send_tx(self, tx, balance, sweep): destaddr = tx['destination'] if sweep: - print 'sweeping' + debug('sweeping') all_utxos = self.taker.wallet.get_utxos_by_mixdepth()[tx['srcmixdepth']] total_value = sum([addrval['value'] for addrval in all_utxos.values()]) while True: orders, cjamount = choose_sweep_order(self.taker.db, total_value, self.taker.txfee, tx['makercount'], weighted_order_choose) if orders == None: - print 'waiting for liquidity' - time.sleep(10) + debug('waiting for liquidity 1min, hopefully more orders should come in') + time.sleep(60) continue cj_fee = 1.0*(cjamount - total_value) / tx['makercount'] / cjamount - print 'average fee = ' + str(cj_fee) + debug('average fee = ' + str(cj_fee)) if cj_fee > self.taker.maxcjfee: - print 'cj fee too high at ' + str(cj_fee) + ', waiting 10 seconds' - time.sleep(10) + print 'cj fee higher than maxcjfee at ' + str(cj_fee) + ', waiting 60 seconds' + time.sleep(60) continue break self.taker.start_cj(self.taker.wallet, cjamount, orders, all_utxos, destaddr, @@ -126,26 +126,27 @@ def send_tx(self, tx, balance, sweep): else: amount = int(tx['amount_fraction'] * balance) if amount < self.taker.mincjamount: - print 'cj amount too low, bringing up' + debug('cj amount too low, bringing up') amount = self.taker.mincjamount changeaddr = self.taker.wallet.get_change_addr(tx['srcmixdepth']) - print 'coinjoining ' + str(amount) + debug('coinjoining ' + str(amount) + ' satoshi') while True: orders, total_cj_fee = choose_order(self.taker.db, amount, tx['makercount'], weighted_order_choose) cj_fee = 1.0*total_cj_fee / tx['makercount'] / amount - print 'average fee = ' + str(cj_fee) + debug('average fee = ' + str(cj_fee)) + if cj_fee > self.taker.maxcjfee: - print 'cj fee too high at ' + str(cj_fee) + ', waiting 10 seconds' - time.sleep(10) + debug('cj fee higher than maxcjfee at ' + str(cj_fee) + ', waiting 60 seconds') + time.sleep(60) continue if orders == None: - print 'waiting for liquidity' - time.sleep(10) + debug('waiting for liquidity 1min, hopefully more orders should come in') + time.sleep(60) continue break - print 'chosen orders to fill ' + str(orders) + ' totalcjfee=' + str(total_cj_fee) + debug('chosen orders to fill ' + str(orders) + ' totalcjfee=' + str(total_cj_fee)) total_amount = amount + total_cj_fee + self.taker.txfee - print 'total amount spent = ' + str(total_amount) + debug('total amount spent = ' + str(total_amount)) utxos = self.taker.wallet.select_utxos(tx['srcmixdepth'], amount) self.taker.start_cj(self.taker.wallet, amount, orders, utxos, destaddr, @@ -156,23 +157,23 @@ def send_tx(self, tx, balance, sweep): self.lockcond.release() debug('tx confirmed, waiting for ' + str(tx['wait']) + ' minutes') time.sleep(tx['wait'] * 60) - print 'woken' + debug('woken') def run(self): - print 'waiting for all orders to certainly arrive' + debug('waiting for all orders to certainly arrive') time.sleep(orderwaittime) sqlorders = self.taker.db.execute('SELECT cjfee, ordertype FROM orderbook;').fetchall() orders = [o['cjfee'] for o in sqlorders if o['ordertype'] == 'relorder'] orders = sorted(orders) relorder_fee = float(orders[0]) - print 'relorder fee = ' + str(relorder_fee) + debug('relorder fee = ' + str(relorder_fee)) maker_count = sum([tx['makercount'] for tx in self.taker.tx_list]) - print('uses ' + str(maker_count) + ' makers, at ' + str(relorder_fee*100) + '% per maker, estimated total cost ' + debug('uses ' + str(maker_count) + ' makers, at ' + str(relorder_fee*100) + '% per maker, estimated total cost ' + str(round((1 - (1 - relorder_fee)**maker_count) * 100, 3)) + '%') time.sleep(orderwaittime) - print 'starting' + debug('starting') self.lockcond = threading.Condition() self.balance_by_mixdepth = {} @@ -186,7 +187,7 @@ def run(self): self.current_tx = i self.send_tx(tx, self.balance_by_mixdepth[tx['srcmixdepth']], sweep) - print 'total finished' + debug('total finished') self.taker.msgchan.shutdown() ''' @@ -287,7 +288,7 @@ def main(): dbg_tx_list = [] for srcmixdepth, txlist in tx_dict.iteritems(): dbg_tx_list.append({'srcmixdepth': srcmixdepth, 'tx': txlist}) - print 'tumbler transaction list' + debug('tumbler transaction list') pprint(dbg_tx_list) total_wait = sum([tx['wait'] for tx in tx_list]) From 6940e11e052f9b05c4fb1bfa393eb94fa05eb972 Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Thu, 4 Jun 2015 01:00:53 +0100 Subject: [PATCH 307/409] fixed bug, whoops --- lib/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/common.py b/lib/common.py index 162b6938..c57ed8e6 100644 --- a/lib/common.py +++ b/lib/common.py @@ -399,7 +399,7 @@ def choose_order(db, cj_amount, n, chooseOrdersBy): counterparties = set([o[0] for o in orders]) if n > len(counterparties): debug('ERROR not enough liquidity in the orderbook n=%d suitable-counterparties=%d amount=%d totalorders=%d' - % (n, len(counterparties), cj_amount), len(orders)) + % (n, len(counterparties), cj_amount, len(orders)) return None, 0 #TODO handle not enough liquidity better, maybe an Exception orders = sorted(orders, key=lambda k: k[2]) #sort from smallest to biggest cj fee debug('considered orders = ' + str(orders)) From c931dd6d559a3da8ec02902502861e66af31a0c0 Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Thu, 4 Jun 2015 01:11:40 +0100 Subject: [PATCH 308/409] more stupid typos --- lib/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/common.py b/lib/common.py index c57ed8e6..1eec3954 100644 --- a/lib/common.py +++ b/lib/common.py @@ -399,7 +399,7 @@ def choose_order(db, cj_amount, n, chooseOrdersBy): counterparties = set([o[0] for o in orders]) if n > len(counterparties): debug('ERROR not enough liquidity in the orderbook n=%d suitable-counterparties=%d amount=%d totalorders=%d' - % (n, len(counterparties), cj_amount, len(orders)) + % (n, len(counterparties), cj_amount, len(orders))) return None, 0 #TODO handle not enough liquidity better, maybe an Exception orders = sorted(orders, key=lambda k: k[2]) #sort from smallest to biggest cj fee debug('considered orders = ' + str(orders)) From e641c5c3e21946aae600d99891c4e3a6ffaecde0 Mon Sep 17 00:00:00 2001 From: chris belcher Date: Thu, 4 Jun 2015 19:18:44 +0100 Subject: [PATCH 309/409] another stupid mistake of mine, i blame python although its mostly my fault for not writing any tests --- lib/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/common.py b/lib/common.py index 1eec3954..d89a3db6 100644 --- a/lib/common.py +++ b/lib/common.py @@ -349,7 +349,7 @@ def weighted_order_choose(orders, n, feekey): unless M < orderbook size, then phi goes up to the last order ''' minfee = feekey(orders[0]) - M = 1.5*n + M = int(1.5*n) if len(orders) > M: phi = feekey(orders[M]) - minfee else: From 0a4063b0bf132f3b986f6c4ebcc15ca8bccfbec7 Mon Sep 17 00:00:00 2001 From: chris belcher Date: Thu, 4 Jun 2015 19:31:22 +0100 Subject: [PATCH 310/409] added link to installing on windows in readme --- README.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.txt b/README.txt index 794b2835..32b72367 100644 --- a/README.txt +++ b/README.txt @@ -26,6 +26,9 @@ INSTALLING 2. you will need numpy 1.7 or later installed 3. (optional) matplotlib for displaying the graphs in orderbook-watcher +If installing on Windows, consult this wiki page for helpful instructions +https://github.com/chris-belcher/joinmarket/wiki/Installing-JoinMarket-on-Windows-7-(temporary) + in the joinmarket.cfg configuration file, set network = mainnet for the actual bitcoin mainnet From b7aaa5b70aafa3a2c82456bb507158dd2fa1c285 Mon Sep 17 00:00:00 2001 From: Martino Salvetti Date: Sat, 6 Jun 2015 13:37:58 +0200 Subject: [PATCH 311/409] Addcache support to make_request --- .gitignore | 1 + lib/bitcoin/bci.py | 25 ++++++++++++++++++++++++- lib/bitcoin/cache.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 1 deletion(-) create mode 100755 lib/bitcoin/cache.py diff --git a/.gitignore b/.gitignore index 0c37968c..6be8963c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.pyc *.swp joinmarket.cfg +blockchain.cache diff --git a/lib/bitcoin/bci.py b/lib/bitcoin/bci.py index a9b8d15f..797ec40c 100644 --- a/lib/bitcoin/bci.py +++ b/lib/bitcoin/bci.py @@ -6,15 +6,38 @@ from urllib.request import build_opener except: from urllib2 import build_opener +from cache import Cache # Makes a request to a given URL (first arg) and optional params (second arg) def make_request(*args): + + usecache = all([type(x)==str for x in args]) + if usecache: + c = Cache() + key = '-*-'.join(args) + if c.get(key): + data = c.get(key) + c.close() + return data + + opener = build_opener() opener.addheaders = [('User-agent', 'Mozilla/5.0'+str(random.randrange(1000000)))] try: - return opener.open(*args).read().strip() + data = opener.open(*args).read().strip() + if usecache: + c.put(key,data) + # clean + if 'putcount' not in c: + c['putcount'] = 0 + c['putcount'] += 1 + if c['putcount'] > 1000: + del c['putcount'] + c.clean() + c.close() + return data except Exception as e: try: p = e.read().strip() diff --git a/lib/bitcoin/cache.py b/lib/bitcoin/cache.py new file mode 100755 index 00000000..c6fcd67f --- /dev/null +++ b/lib/bitcoin/cache.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python2 + +from shelve import DbfilenameShelf as Sh +import time + +class Cache(Sh): + ''' + Validity works only with get() and put() methods + ''' + def __init__(self,filename='blockchain.cache',validity=300): + Sh.__init__(self,filename) + self.validity = validity + + def __enter__(self): + return self + + def __exit__(self, exc_t, exc_v, trace): + self.close() + + def clean(): + since = time.time() - self.validity + for k,(t,_) in self.items(): + if t < since: + del self[k] + + def get(self, key, default=None): + since = time.time() - self.validity + if key in self: + t,v = self[key] + if t >= since: + return v + return default + + def put(self, key, value): + self[key] = (time.time(), value) + +if __name__=="__main__": + + with Cache('/tmp/test.cache',1) as c: + print c.get('test') + c.put('test',1) + print c.get('test') + time.sleep(2) + print c.get('test') From 774041b6e92b61f30ac052594ded9f58244f7597 Mon Sep 17 00:00:00 2001 From: Emilian Ursu Date: Fri, 12 Jun 2015 01:32:36 -0400 Subject: [PATCH 312/409] x --- joinmarket.cfg | 2 +- lib/irc.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/joinmarket.cfg b/joinmarket.cfg index 7b34a37d..1bd4c864 100644 --- a/joinmarket.cfg +++ b/joinmarket.cfg @@ -6,7 +6,7 @@ network = mainnet bitcoin_cli_cmd = bitcoin-cli [MESSAGING] -host = irc.snoonet.org +host = irc.freenode.net channel = joinmarket-pit port = 6697 usessl = true diff --git a/lib/irc.py b/lib/irc.py index c94a14e6..2709ddb9 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -179,10 +179,6 @@ def check_for_orders(self, nick, chunks): maxsize = chunks[3] txfee = chunks[4] cjfee = chunks[5] - if any(t < 0 for t in [oid, minsize, maxsize, txfee, cjfee]): - return - if minsize > maxsize: - return if self.on_order_seen: self.on_order_seen(counterparty, oid, ordertype, minsize, maxsize, txfee, cjfee) From 4c82a56fba1debc147b3616f83b57e9d4af576f2 Mon Sep 17 00:00:00 2001 From: Emilian Ursu Date: Fri, 12 Jun 2015 01:35:56 -0400 Subject: [PATCH 313/409] Wrong password handling --- lib/common.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/common.py b/lib/common.py index d89a3db6..ee893211 100644 --- a/lib/common.py +++ b/lib/common.py @@ -210,10 +210,16 @@ def get_seed(self, seedarg): sys.exit(0) if 'index_cache' in walletdata: self.index_cache = walletdata['index_cache'] - password = getpass.getpass('Enter wallet decryption passphrase: ') - password_key = btc.bin_dbl_sha256(password) - decrypted_seed = slowaes.decryptData(password_key, walletdata['encrypted_seed'] - .decode('hex')).encode('hex') + decrypted = 0 + while not decrypted: + try: + password = getpass.getpass('Enter wallet decryption passphrase: ') + password_key = btc.bin_dbl_sha256(password) + decrypted_seed = slowaes.decryptData(password_key, walletdata['encrypted_seed'] + .decode('hex')).encode('hex') + decrypted = 1 + except ValueError: + decrypted = 0 return decrypted_seed def update_cache_index(self): From 009a5827819740b3c5471603c2f00d0604250da0 Mon Sep 17 00:00:00 2001 From: Martino Salvetti Date: Sat, 13 Jun 2015 15:14:21 +0200 Subject: [PATCH 314/409] Implement orderby (ASC only) --- ob-watcher.py | 40 +++++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/ob-watcher.py b/ob-watcher.py index 9feae1bf..43a313ca 100644 --- a/ob-watcher.py +++ b/ob-watcher.py @@ -1,6 +1,8 @@ import BaseHTTPServer, SimpleHTTPServer, threading from decimal import Decimal +import urllib2 +#from urllib2.urlparse import parse_qs import io, base64, time, sys, os data_dir = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(data_dir, 'lib')) @@ -10,16 +12,18 @@ from common import * import common +# ['counterparty', 'oid', 'ordertype', 'minsize', 'maxsize', 'txfee', 'cjfee'] + tableheading = ''' - - - - - - - + + + + + + + ''' @@ -103,11 +107,23 @@ def __init__(self, request, client_address, base_server): self.base_server = base_server SimpleHTTPServer.SimpleHTTPRequestHandler.__init__(self, request, client_address, base_server) - def create_orderbook_table(self): + def create_orderbook_table(self, orderby, desc): result = '' rows = self.taker.db.execute('SELECT * FROM orderbook;').fetchall() + if not rows: + return 0, result + if orderby: + orderby = orderby[0] ordersorder = ['absorder','relorder'] - for o in sorted(rows,cmp=lambda x,y: cmp(x['cjfee'],y['cjfee']) if x['ordertype']==y['ordertype'] else cmp(ordersorder.index(x['ordertype']),ordersorder.index(y['ordertype']))): + if orderby not in rows[0].keys() or orderby == 'cjfee': + orderby = 'cjfee' + orderby_cmp = lambda x,y: cmp(Decimal(x['cjfee']),Decimal(y['cjfee'])) if x['ordertype']==y['ordertype'] \ + else cmp(ordersorder.index(x['ordertype']),ordersorder.index(y['ordertype'])) + else: + orderby_cmp = lambda x,y: cmp(x[orderby],y[orderby]) + if desc: + orderby_cmp = lambda x,y: orderby_cmp(y,x) + for o in sorted(rows,cmp=orderby_cmp): result += ' \n' order_keys_display = (('ordertype', ordertype_display), ('counterparty', do_nothing), ('oid', order_str), ('cjfee', cjfee_display), ('txfee', satoshi_to_unit), @@ -124,6 +140,8 @@ def get_counterparty_count(self): def do_GET(self): #SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self) #print 'httpd received ' + self.path + ' request' + self.path, query = self.path.split('?',1) if '?' in self.path else (self.path,'') + args = urllib2.urlparse.parse_qs(query) pages = ['/', '/ordersize', '/depth'] if self.path not in pages: return @@ -134,7 +152,7 @@ def do_GET(self): if common.joinmarket_alert: alert_msg = '
    JoinMarket Alert Message:
    ' + common.joinmarket_alert if self.path == '/': - ordercount, ordertable = self.create_orderbook_table() + ordercount, ordertable = self.create_orderbook_table(args.get('orderby'),'desc' in args) replacements = { 'PAGETITLE': 'JoinMarket Browser Interface', 'MAINHEADING': 'JoinMarket Orderbook', @@ -193,7 +211,7 @@ def __init__(self, taker): self.daemon = True self.taker = taker def run(self): - hostport = ('localhost', 62601) + hostport = ('localhost', 62602) httpd = BaseHTTPServer.HTTPServer(hostport, OrderbookPageRequestHeader) httpd.taker = self.taker print '\nstarted http server, visit http://{0}:{1}/\n'.format(*hostport) From a86f7eb797cbc7bc803fef6c798e6c734ab5a7ee Mon Sep 17 00:00:00 2001 From: Martino Salvetti Date: Sat, 13 Jun 2015 15:45:27 +0200 Subject: [PATCH 315/409] Implement desc --- ob-watcher.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/ob-watcher.py b/ob-watcher.py index 43a313ca..92b9aba9 100644 --- a/ob-watcher.py +++ b/ob-watcher.py @@ -2,7 +2,6 @@ import BaseHTTPServer, SimpleHTTPServer, threading from decimal import Decimal import urllib2 -#from urllib2.urlparse import parse_qs import io, base64, time, sys, os data_dir = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(data_dir, 'lib')) @@ -13,19 +12,16 @@ import common # ['counterparty', 'oid', 'ordertype', 'minsize', 'maxsize', 'txfee', 'cjfee'] +col = ' \n' # .format(field,label) -tableheading = ''' -
    TypeCounterpartyOrder IDFeeMiner Fee ContributionMinimum SizeMaximum SizeTypeCounterpartyOrder IDFeeMiner Fee ContributionMinimum SizeMaximum Size
    {1}
    (desc)
    - - - - - - - - - -''' +tableheading = '
    TypeCounterpartyOrder IDFeeMiner Fee ContributionMinimum SizeMaximum Size
    \n ' + ''.join( + [col.format('ordertype','Type'), + col.format('counterparty','Counterparty'), + col.format('oid','Order ID'), + col.format('cjfee','Fee'), + col.format('txfee','Miner Fee Contribution'), + col.format('minsize','Minimum Size'), + col.format('maxsize','Maximum Size')]) + ' ' shutdownform = '' shutdownpage = '

    Successfully Shut down

    ' @@ -122,7 +118,8 @@ def create_orderbook_table(self, orderby, desc): else: orderby_cmp = lambda x,y: cmp(x[orderby],y[orderby]) if desc: - orderby_cmp = lambda x,y: orderby_cmp(y,x) + orderby_cmp_wrapper = orderby_cmp + orderby_cmp = lambda x,y: orderby_cmp_wrapper(y,x) for o in sorted(rows,cmp=orderby_cmp): result += ' \n' order_keys_display = (('ordertype', ordertype_display), ('counterparty', do_nothing), From a0046cf9c796dd6c828eb9f65d9e756f836f477b Mon Sep 17 00:00:00 2001 From: Martino Salvetti Date: Sat, 13 Jun 2015 16:00:43 +0200 Subject: [PATCH 316/409] Removed implementation of cache --- lib/bitcoin/bci.py | 25 +------------------------ lib/bitcoin/cache.py | 44 -------------------------------------------- ob-watcher.py | 2 +- 3 files changed, 2 insertions(+), 69 deletions(-) delete mode 100755 lib/bitcoin/cache.py diff --git a/lib/bitcoin/bci.py b/lib/bitcoin/bci.py index 797ec40c..a9b8d15f 100644 --- a/lib/bitcoin/bci.py +++ b/lib/bitcoin/bci.py @@ -6,38 +6,15 @@ from urllib.request import build_opener except: from urllib2 import build_opener -from cache import Cache # Makes a request to a given URL (first arg) and optional params (second arg) def make_request(*args): - - usecache = all([type(x)==str for x in args]) - if usecache: - c = Cache() - key = '-*-'.join(args) - if c.get(key): - data = c.get(key) - c.close() - return data - - opener = build_opener() opener.addheaders = [('User-agent', 'Mozilla/5.0'+str(random.randrange(1000000)))] try: - data = opener.open(*args).read().strip() - if usecache: - c.put(key,data) - # clean - if 'putcount' not in c: - c['putcount'] = 0 - c['putcount'] += 1 - if c['putcount'] > 1000: - del c['putcount'] - c.clean() - c.close() - return data + return opener.open(*args).read().strip() except Exception as e: try: p = e.read().strip() diff --git a/lib/bitcoin/cache.py b/lib/bitcoin/cache.py deleted file mode 100755 index c6fcd67f..00000000 --- a/lib/bitcoin/cache.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python2 - -from shelve import DbfilenameShelf as Sh -import time - -class Cache(Sh): - ''' - Validity works only with get() and put() methods - ''' - def __init__(self,filename='blockchain.cache',validity=300): - Sh.__init__(self,filename) - self.validity = validity - - def __enter__(self): - return self - - def __exit__(self, exc_t, exc_v, trace): - self.close() - - def clean(): - since = time.time() - self.validity - for k,(t,_) in self.items(): - if t < since: - del self[k] - - def get(self, key, default=None): - since = time.time() - self.validity - if key in self: - t,v = self[key] - if t >= since: - return v - return default - - def put(self, key, value): - self[key] = (time.time(), value) - -if __name__=="__main__": - - with Cache('/tmp/test.cache',1) as c: - print c.get('test') - c.put('test',1) - print c.get('test') - time.sleep(2) - print c.get('test') diff --git a/ob-watcher.py b/ob-watcher.py index 92b9aba9..cff90080 100644 --- a/ob-watcher.py +++ b/ob-watcher.py @@ -208,7 +208,7 @@ def __init__(self, taker): self.daemon = True self.taker = taker def run(self): - hostport = ('localhost', 62602) + hostport = ('localhost', 62601) httpd = BaseHTTPServer.HTTPServer(hostport, OrderbookPageRequestHeader) httpd.taker = self.taker print '\nstarted http server, visit http://{0}:{1}/\n'.format(*hostport) From 7385ba4328890358ce542c1b1d3b47c8184a1b32 Mon Sep 17 00:00:00 2001 From: Adlai Chandrasekhar Date: Mon, 15 Jun 2015 00:33:08 +0300 Subject: [PATCH 317/409] cleanup & bugfix orderbook sorting One of the market participants has taken to serializing the fee in exponential notation, presumably in Murphy's honor. String comparison doomed such offers far lower in the order book depths than they deserved to swim, and we thus rescue them thru float coercion. --- ob-watcher.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ob-watcher.py b/ob-watcher.py index 9feae1bf..208634c3 100644 --- a/ob-watcher.py +++ b/ob-watcher.py @@ -97,6 +97,13 @@ def satoshi_to_unit(sat, order): def order_str(s, order): return str(s) +ordersorder = ['absorder','relorder'] +def cmp_orders(first, other): + key = (lambda order: float(order['cjfee']))\ + if first['ordertype'] == other['ordertype'] else \ + lambda order: ordersorder.index(order['ordertype']) + return cmp(key(first), key(other)) + class OrderbookPageRequestHeader(SimpleHTTPServer.SimpleHTTPRequestHandler): def __init__(self, request, client_address, base_server): self.taker = base_server.taker @@ -106,8 +113,7 @@ def __init__(self, request, client_address, base_server): def create_orderbook_table(self): result = '' rows = self.taker.db.execute('SELECT * FROM orderbook;').fetchall() - ordersorder = ['absorder','relorder'] - for o in sorted(rows,cmp=lambda x,y: cmp(x['cjfee'],y['cjfee']) if x['ordertype']==y['ordertype'] else cmp(ordersorder.index(x['ordertype']),ordersorder.index(y['ordertype']))): + for o in sorted(rows,cmp=cmp_orders): result += ' \n' order_keys_display = (('ordertype', ordertype_display), ('counterparty', do_nothing), ('oid', order_str), ('cjfee', cjfee_display), ('txfee', satoshi_to_unit), From 46ada893fbd9347067a8452d2f07dfc968cf75d3 Mon Sep 17 00:00:00 2001 From: Adlai Chandrasekhar Date: Mon, 15 Jun 2015 00:34:39 +0300 Subject: [PATCH 318/409] fix wallet-tool showseed bug by storing wallet seed Although this does increase the amount of sensitive information stored in memory (since storing only child keys leaves one unable to reconstruct the master private key), this additional information is unlikely to secure additional funds (as the receiving addresses are always drawn from the reach of the already-known mixdepth keys). --- lib/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/common.py b/lib/common.py index d89a3db6..76d9b0be 100644 --- a/lib/common.py +++ b/lib/common.py @@ -169,8 +169,8 @@ def __init__(self, seedarg, max_mix_depth=2, gaplimit=6): super(Wallet, self).__init__() self.max_mix_depth = max_mix_depth self.gaplimit = gaplimit - seed = self.get_seed(seedarg) - master = btc.bip32_master_key(seed) + self.seed = self.get_seed(seedarg) + master = btc.bip32_master_key(self.seed) m_0 = btc.bip32_ckd(master, 0) mixing_depth_keys = [btc.bip32_ckd(m_0, c) for c in range(max_mix_depth)] self.keys = [(btc.bip32_ckd(m, 0), btc.bip32_ckd(m, 1)) for m in mixing_depth_keys] From 61e7bcfa357dd9648c8edaa68fe5ceb685d85755 Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Tue, 16 Jun 2015 01:52:31 +0100 Subject: [PATCH 319/409] added incorrect password message --- lib/common.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/common.py b/lib/common.py index ee893211..3bdd940f 100644 --- a/lib/common.py +++ b/lib/common.py @@ -210,16 +210,18 @@ def get_seed(self, seedarg): sys.exit(0) if 'index_cache' in walletdata: self.index_cache = walletdata['index_cache'] - decrypted = 0 + decrypted = False while not decrypted: + password = getpass.getpass('Enter wallet decryption passphrase: ') + password_key = btc.bin_dbl_sha256(password) + encrypted_seed = walletdata['encrypted_seed'] try: - password = getpass.getpass('Enter wallet decryption passphrase: ') - password_key = btc.bin_dbl_sha256(password) - decrypted_seed = slowaes.decryptData(password_key, walletdata['encrypted_seed'] + decrypted_seed = slowaes.decryptData(password_key, encrypted_seed .decode('hex')).encode('hex') - decrypted = 1 + decrypted = True except ValueError: - decrypted = 0 + print 'Incorrect password' + decrypted = False return decrypted_seed def update_cache_index(self): From d691c615d6073648a1b594446061254472b78e7d Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Thu, 25 Jun 2015 02:15:48 +0100 Subject: [PATCH 320/409] replaced space indenting with tab indenting in the latest PR --- ob-watcher.py | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/ob-watcher.py b/ob-watcher.py index cff90080..9313bbfd 100644 --- a/ob-watcher.py +++ b/ob-watcher.py @@ -14,14 +14,14 @@ # ['counterparty', 'oid', 'ordertype', 'minsize', 'maxsize', 'txfee', 'cjfee'] col = ' \n' # .format(field,label) -tableheading = '
    {1}
    (desc)
    \n ' + ''.join( - [col.format('ordertype','Type'), - col.format('counterparty','Counterparty'), - col.format('oid','Order ID'), - col.format('cjfee','Fee'), - col.format('txfee','Miner Fee Contribution'), - col.format('minsize','Minimum Size'), - col.format('maxsize','Maximum Size')]) + ' ' +tableheading = '
    \n ' + ''.join([ + col.format('ordertype','Type'), + col.format('counterparty','Counterparty'), + col.format('oid','Order ID'), + col.format('cjfee','Fee'), + col.format('txfee','Miner Fee Contribution'), + col.format('minsize','Minimum Size'), + col.format('maxsize','Maximum Size')]) + ' ' shutdownform = '' shutdownpage = '

    Successfully Shut down

    ' @@ -106,20 +106,20 @@ def __init__(self, request, client_address, base_server): def create_orderbook_table(self, orderby, desc): result = '' rows = self.taker.db.execute('SELECT * FROM orderbook;').fetchall() - if not rows: - return 0, result - if orderby: - orderby = orderby[0] + if not rows: + return 0, result + if orderby: + orderby = orderby[0] ordersorder = ['absorder','relorder'] - if orderby not in rows[0].keys() or orderby == 'cjfee': - orderby = 'cjfee' - orderby_cmp = lambda x,y: cmp(Decimal(x['cjfee']),Decimal(y['cjfee'])) if x['ordertype']==y['ordertype'] \ - else cmp(ordersorder.index(x['ordertype']),ordersorder.index(y['ordertype'])) - else: - orderby_cmp = lambda x,y: cmp(x[orderby],y[orderby]) - if desc: - orderby_cmp_wrapper = orderby_cmp - orderby_cmp = lambda x,y: orderby_cmp_wrapper(y,x) + if orderby not in rows[0].keys() or orderby == 'cjfee': + orderby = 'cjfee' + orderby_cmp = lambda x,y: cmp(Decimal(x['cjfee']),Decimal(y['cjfee'])) if x['ordertype']==y['ordertype'] \ + else cmp(ordersorder.index(x['ordertype']),ordersorder.index(y['ordertype'])) + else: + orderby_cmp = lambda x,y: cmp(x[orderby],y[orderby]) + if desc: + orderby_cmp_wrapper = orderby_cmp + orderby_cmp = lambda x,y: orderby_cmp_wrapper(y,x) for o in sorted(rows,cmp=orderby_cmp): result += ' \n' order_keys_display = (('ordertype', ordertype_display), ('counterparty', do_nothing), @@ -137,8 +137,8 @@ def get_counterparty_count(self): def do_GET(self): #SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self) #print 'httpd received ' + self.path + ' request' - self.path, query = self.path.split('?',1) if '?' in self.path else (self.path,'') - args = urllib2.urlparse.parse_qs(query) + self.path, query = self.path.split('?',1) if '?' in self.path else (self.path,'') + args = urllib2.urlparse.parse_qs(query) pages = ['/', '/ordersize', '/depth'] if self.path not in pages: return From cd5211008a5652cdcaf0ff82ee889f541193ff63 Mon Sep 17 00:00:00 2001 From: CohibAA Date: Sun, 28 Jun 2015 22:37:27 -0600 Subject: [PATCH 321/409] Update README.txt --- README.txt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.txt b/README.txt index 32b72367..522eb971 100644 --- a/README.txt +++ b/README.txt @@ -17,7 +17,7 @@ Donation address: Wiki page for more detailed articles: https://github.com/chris-belcher/joinmarket/wiki -INSTALLING +REQUIRED INSTALLATION DEPENDENCIES 0. You will need python 2.7 1. You will need libsodium installed Get it here: http://doc.libsodium.org/ @@ -26,6 +26,13 @@ INSTALLING 2. you will need numpy 1.7 or later installed 3. (optional) matplotlib for displaying the graphs in orderbook-watcher +GET DEBIAN / UBUNTU INSTALL DEPENDENCIES: +0. sudo apt-get update -y && sudo apt-get upgrade -y && sudo apt-get install python +1. sudo apt-get install libsodium-dev -y +2. sudo apt-get install python-pip -y && sudo pip install numpy --upgrade +3. (optional) sudo apt-get install python-matplotlib -y + +WINDOWS: If installing on Windows, consult this wiki page for helpful instructions https://github.com/chris-belcher/joinmarket/wiki/Installing-JoinMarket-on-Windows-7-(temporary) From 2f54ced818f542689d8620a528f9a828e5a77280 Mon Sep 17 00:00:00 2001 From: A bit Fan Date: Mon, 29 Jun 2015 03:11:22 -0400 Subject: [PATCH 322/409] Connect to hidden service with ssl --- lib/irc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/irc.py b/lib/irc.py index 223fdcec..ae7c962a 100644 --- a/lib/irc.py +++ b/lib/irc.py @@ -458,10 +458,10 @@ def run(self): self.sock = socks.socksocket() else: self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.connect(self.serverport) if config.get("MESSAGING","usessl").lower() == 'true': self.sock = ssl.wrap_socket(self.sock) self.fd = self.sock.makefile() - self.sock.connect(self.serverport) self.password = None if self.given_password: self.password = self.given_password From b3f3e822e8729c508dd130418d71dd08cd285b10 Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Mon, 29 Jun 2015 09:36:00 +0100 Subject: [PATCH 323/409] do not print out seed --- patientsendpayment.py | 2 +- sendpayment.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/patientsendpayment.py b/patientsendpayment.py index 2d01f186..9468b9f8 100644 --- a/patientsendpayment.py +++ b/patientsendpayment.py @@ -170,7 +170,7 @@ def main(): irc.run() except: debug('CRASHING, DUMPING EVERYTHING') - debug_dump_object(wallet, ['addr_cache', 'keys']) + debug_dump_object(wallet, ['addr_cache', 'keys', 'seed']) debug_dump_object(taker) import traceback traceback.print_exc() diff --git a/sendpayment.py b/sendpayment.py index bf1bbaae..72746f6f 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -146,7 +146,7 @@ def main(): irc.run() except: debug('CRASHING, DUMPING EVERYTHING') - debug_dump_object(wallet, ['addr_cache', 'keys', 'wallet_name']) + debug_dump_object(wallet, ['addr_cache', 'keys', 'wallet_name', 'seed']) debug_dump_object(taker) import traceback debug(traceback.format_exc()) From 220415c7208044ee94e4d7a4c804a676658435ee Mon Sep 17 00:00:00 2001 From: btcspry Date: Tue, 30 Jun 2015 12:16:48 -0600 Subject: [PATCH 324/409] Changed appearance and sortablility of table and website - Converted whole site to bootstrap - Added "GitHub" and "Getting Started" links in the menubar - Included "sortable.js" for sorting the table, client side - Removed server-side table sorting --- ob-watcher.py | 4 +- orderbook.html | 159 ++++++++++++++++++++++++++++++------------------- 2 files changed, 101 insertions(+), 62 deletions(-) diff --git a/ob-watcher.py b/ob-watcher.py index 9313bbfd..cf160e59 100644 --- a/ob-watcher.py +++ b/ob-watcher.py @@ -12,9 +12,9 @@ import common # ['counterparty', 'oid', 'ordertype', 'minsize', 'maxsize', 'txfee', 'cjfee'] -col = ' \n' # .format(field,label) +col = ' \n' # .format(field,label) -tableheading = '
    {1}
    (desc)    		{1}
    \n ' + ''.join([ +tableheading = '
    \n ' + ''.join([ col.format('ordertype','Type'), col.format('counterparty','Counterparty'), col.format('oid','Order ID'), diff --git a/orderbook.html b/orderbook.html index eba54490..c17f909c 100644 --- a/orderbook.html +++ b/orderbook.html @@ -1,62 +1,101 @@ - - - -PAGETITLE - - - - - -
    - - - -

    MAINHEADING

    -

    SECONDHEADING

    - -MAINBODY - -
    - + + + + + + PAGETITLE + + + + + + + + + + +
    +

    MAINHEADING

    +

    SECONDHEADING

    + + MAINBODY +
    +
    + From aad5ea3631f41867be243fb847fa947bead4ba35 Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Wed, 1 Jul 2015 02:44:01 +0100 Subject: [PATCH 325/409] added debug information to attempt to fix weight not adding to 1 bug --- lib/common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/common.py b/lib/common.py index bdbe785b..c22ba866 100644 --- a/lib/common.py +++ b/lib/common.py @@ -358,11 +358,12 @@ def weighted_order_choose(orders, n, feekey): ''' minfee = feekey(orders[0]) M = int(1.5*n) - if len(orders) > M: + if len(orders) >= M: phi = feekey(orders[M]) - minfee else: phi = feekey(orders[-1]) - minfee fee = np.array([feekey(o) for o in orders]) + debug('phi=' + str(phi) + ' fee=' + str(fee)) if phi > 0: weight = np.exp(-(1.0*fee - minfee) / phi) else: From 77c11bede6ddcb01353810db1674dd4f48fa34f6 Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Wed, 1 Jul 2015 13:33:03 +0100 Subject: [PATCH 326/409] it was bugged Revert "added debug information to attempt to fix weight not adding to 1 bug" This reverts commit aad5ea3631f41867be243fb847fa947bead4ba35. --- lib/common.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/common.py b/lib/common.py index c22ba866..bdbe785b 100644 --- a/lib/common.py +++ b/lib/common.py @@ -358,12 +358,11 @@ def weighted_order_choose(orders, n, feekey): ''' minfee = feekey(orders[0]) M = int(1.5*n) - if len(orders) >= M: + if len(orders) > M: phi = feekey(orders[M]) - minfee else: phi = feekey(orders[-1]) - minfee fee = np.array([feekey(o) for o in orders]) - debug('phi=' + str(phi) + ' fee=' + str(fee)) if phi > 0: weight = np.exp(-(1.0*fee - minfee) / phi) else: From 0cc6914ab6ae450057048852d4924c10863c21c5 Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Wed, 1 Jul 2015 13:34:24 +0100 Subject: [PATCH 327/409] properly added debug information to attempt to fix weight not adding to 1 bug --- lib/common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/common.py b/lib/common.py index bdbe785b..7b5b91b4 100644 --- a/lib/common.py +++ b/lib/common.py @@ -363,6 +363,7 @@ def weighted_order_choose(orders, n, feekey): else: phi = feekey(orders[-1]) - minfee fee = np.array([feekey(o) for o in orders]) + debug('phi=' + str(phi) + ' fee=' + str(fee)) if phi > 0: weight = np.exp(-(1.0*fee - minfee) / phi) else: From c7572a29e46e51aa171998105bef0cebec65a279 Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Wed, 1 Jul 2015 23:49:28 +0100 Subject: [PATCH 328/409] added shuffling of the inputs to improve privacy --- lib/taker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/taker.py b/lib/taker.py index e124f72d..e1f65664 100644 --- a/lib/taker.py +++ b/lib/taker.py @@ -103,6 +103,7 @@ def recv_txio(self, nick, utxo_list, cj_pub, change_addr): else: self.outputs.append({'address': self.my_change_addr, 'value': my_change_value}) utxo_tx = [dict([('output', u)]) for u in sum(self.utxos.values(), [])] + random.shuffle(utxo_tx) random.shuffle(self.outputs) tx = btc.mktx(utxo_tx, self.outputs) debug('obtained tx\n' + pprint.pformat(btc.deserialize(tx))) From d0efc4cebec43c3f44f729fbeb641d0a262132b6 Mon Sep 17 00:00:00 2001 From: btcspry Date: Wed, 1 Jul 2015 23:31:34 +0000 Subject: [PATCH 329/409] Removed commented out lines in the style tag --- orderbook.html | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/orderbook.html b/orderbook.html index c17f909c..3df46022 100644 --- a/orderbook.html +++ b/orderbook.html @@ -6,42 +6,6 @@ PAGETITLE - + + + + + From c11229a3614b2c94b612e0d59a93481841f99c5a Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Tue, 29 Sep 2015 23:41:58 +0100 Subject: [PATCH 406/409] made an exiting procedure slightly cleaner --- sendpayment.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sendpayment.py b/sendpayment.py index d26880a0..bb75720c 100644 --- a/sendpayment.py +++ b/sendpayment.py @@ -102,8 +102,9 @@ def sendpayment_choose_orders(self, cj_amount, makercount, nonrespondants=[], ac debug(noun + ' coinjoin fee = ' + str(float('%.3g' % (100.0 * total_fee_pc))) + '%') check_high_fee(total_fee_pc) if raw_input('send with these orders? (y/n):')[0] != 'y': + debug('ending') self.taker.msgchan.shutdown() - return + return None, -1 return orders, total_cj_fee def run(self): From a436ebff7cf71b7f7d65050358e623a0b0ca272b Mon Sep 17 00:00:00 2001 From: A bit Fan Date: Wed, 30 Sep 2015 01:36:33 -0400 Subject: [PATCH 407/409] Move yieldgen configuration to joinmarket.cfg --- lib/common.py | 10 +++++++++- yield-generator.py | 17 +++++++++++------ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/lib/common.py b/lib/common.py index 410b566d..38b3c409 100644 --- a/lib/common.py +++ b/lib/common.py @@ -21,7 +21,7 @@ joinmarket_alert = None debug_silence = False -config = SafeConfigParser() +config = SafeConfigParser({'txfee' : '1000', 'cjfee' : '0.002', 'mix_levels' : '5', 'minsize' : '0'}) config_location = 'joinmarket.cfg' # FIXME: Add rpc_* options here in the future! required_options = {'BLOCKCHAIN':['blockchain_source', 'network'], @@ -57,6 +57,14 @@ [POLICY] #for dust sweeping, try merge_algorithm = gradual merge_algorithm = default + +[YIELDGEN] +txfee = 1000 +cjfee = 0.002 +nickname = '' +nickserv_password = '' +mix_levels = 5 +minsize = 0 """ def load_program_config(): diff --git a/yield-generator.py b/yield-generator.py index caf65843..cbb0d16c 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -12,12 +12,17 @@ from socket import gethostname -txfee = 1000 -cjfee = '0.002' # 0.2% fee -nickname = random_nick() -nickserv_password = '' -minsize = int(1.2 * txfee / float(cjfee)) #minimum size is such that you always net profit at least 20% of the miner fee -mix_levels = 5 +txfee = config.getint('YIELDGEN', 'txfee') +cjfee = config.getfloat('YIELDGEN', 'cjfee') # 0.2% fee +nickname = config.get('YIELDGEN', 'nickname'); +nickserv_password = config.get('YIELDGEN', 'nickserv_password'); +minsize = config.getint('YIELDGEN', 'minsize'); +mix_levels = config.getint('YIELDGEN', 'mix_levels'); + +if not minsize: + minsize = int(1.2 * txfee / float(cjfee)) #minimum size is such that you always net profit at least 20% of the miner fee +if not nickname: + nickname = random_nick() From 06baba442c7648dd484892ac128d7a21599358d6 Mon Sep 17 00:00:00 2001 From: A bit Fan Date: Wed, 30 Sep 2015 02:05:22 -0400 Subject: [PATCH 408/409] Move yieldgen conf to joinmarket.cfg --- lib/common.py | 6 +++--- yield-generator.py | 20 +++++++++----------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/lib/common.py b/lib/common.py index 38b3c409..2f05e298 100644 --- a/lib/common.py +++ b/lib/common.py @@ -21,7 +21,7 @@ joinmarket_alert = None debug_silence = False -config = SafeConfigParser({'txfee' : '1000', 'cjfee' : '0.002', 'mix_levels' : '5', 'minsize' : '0'}) +config = SafeConfigParser({'txfee' : '1000', 'cjfee' : '0.002', 'mix_levels' : '5', 'minsize' : '0', 'nickname' : '', 'nickserv_password': ''}) config_location = 'joinmarket.cfg' # FIXME: Add rpc_* options here in the future! required_options = {'BLOCKCHAIN':['blockchain_source', 'network'], @@ -61,8 +61,8 @@ [YIELDGEN] txfee = 1000 cjfee = 0.002 -nickname = '' -nickserv_password = '' +nickname = +nickserv_password = mix_levels = 5 minsize = 0 """ diff --git a/yield-generator.py b/yield-generator.py index cbb0d16c..cbafee91 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -12,21 +12,19 @@ from socket import gethostname -txfee = config.getint('YIELDGEN', 'txfee') -cjfee = config.getfloat('YIELDGEN', 'cjfee') # 0.2% fee -nickname = config.get('YIELDGEN', 'nickname'); -nickserv_password = config.get('YIELDGEN', 'nickserv_password'); -minsize = config.getint('YIELDGEN', 'minsize'); -mix_levels = config.getint('YIELDGEN', 'mix_levels'); +load_program_config() +if not config.has_section('YIELDGEN'): + config.add_section('YIELDGEN') -if not minsize: - minsize = int(1.2 * txfee / float(cjfee)) #minimum size is such that you always net profit at least 20% of the miner fee -if not nickname: - nickname = random_nick() +txfee = config.getint('YIELDGEN', 'txfee') +cjfee = config.getfloat('YIELDGEN', 'cjfee') +nickname = config.get('YIELDGEN', 'nickname') or random_nick() +nickserv_password = config.get('YIELDGEN', 'nickserv_password') +minsize = config.getint('YIELDGEN', 'minsize') or int(1.2 * txfee / float(cjfee)) +mix_levels = config.getint('YIELDGEN', 'mix_levels') -#is a maker for the purposes of generating a yield from held # bitcoins without ruining privacy for the taker, the taker could easily check # the history of the utxos this bot sends, so theres not much incentive # to ruin the privacy for barely any more yield From a0d305d08948634b96060020259cbe17160e1b75 Mon Sep 17 00:00:00 2001 From: A bit Fan Date: Wed, 30 Sep 2015 02:07:44 -0400 Subject: [PATCH 409/409] fixes --- yield-generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yield-generator.py b/yield-generator.py index cbafee91..719803c6 100644 --- a/yield-generator.py +++ b/yield-generator.py @@ -24,7 +24,7 @@ mix_levels = config.getint('YIELDGEN', 'mix_levels') - +#is a maker for the purposes of generating a yield from held # bitcoins without ruining privacy for the taker, the taker could easily check # the history of the utxos this bot sends, so theres not much incentive # to ruin the privacy for barely any more yield