From 30d69a099db27d136c828282abe1adcce3e0ba22 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Tue, 9 Aug 2016 20:29:32 +0300 Subject: [PATCH] refactor yieldgenerator module into joinmarket, add yg-pe (privacy enhancing) --- joinmarket/__init__.py | 1 + joinmarket/yieldgenerator.py | 161 +++++++++++++++++++++++++++++++ yg-pe.py | 179 +++++++++++++++++++++++++++++++++++ 3 files changed, 341 insertions(+) create mode 100644 joinmarket/yieldgenerator.py create mode 100644 yg-pe.py diff --git a/joinmarket/__init__.py b/joinmarket/__init__.py index ed308c47..c8232e21 100644 --- a/joinmarket/__init__.py +++ b/joinmarket/__init__.py @@ -21,6 +21,7 @@ get_network, jm_single, get_network, validate_address, get_irc_mchannels, \ check_utxo_blacklist from .blockchaininterface import BlockrInterface, BlockchainInterface +from .yieldgenerator import YieldGenerator, ygmain # Set default logging handler to avoid "No handler found" warnings. try: diff --git a/joinmarket/yieldgenerator.py b/joinmarket/yieldgenerator.py new file mode 100644 index 00000000..2a915366 --- /dev/null +++ b/joinmarket/yieldgenerator.py @@ -0,0 +1,161 @@ +#! /usr/bin/env python +from __future__ import absolute_import, print_function + +import datetime +import os +import time +import abc +from optparse import OptionParser + +from joinmarket import Maker, IRCMessageChannel, MessageChannelCollection +from joinmarket import BlockrInterface +from joinmarket import jm_single, get_network, load_program_config +from joinmarket import get_log, calc_cj_fee, debug_dump_object +from joinmarket import Wallet +from joinmarket import get_irc_mchannels + +log = get_log() + +# is a maker for the purposes of generating a yield from held +# bitcoins +class YieldGenerator(Maker): + __metaclass__ = abc.ABCMeta + statement_file = os.path.join('logs', 'yigen-statement.csv') + + def __init__(self, msgchan, wallet): + Maker.__init__(self, msgchan, wallet) + self.msgchan.register_channel_callbacks(self.on_welcome, + self.on_set_topic, None, None, + self.on_nick_leave, None) + self.tx_unconfirm_timestamp = {} + + def log_statement(self, data): + if get_network() == 'testnet': + return + + data = [str(d) for d in data] + self.income_statement = open(self.statement_file, 'a') + self.income_statement.write(','.join(data) + '\n') + self.income_statement.close() + + def on_welcome(self): + Maker.on_welcome(self) + if not os.path.isfile(self.statement_file): + self.log_statement( + ['timestamp', 'cj amount/satoshi', 'my input count', + 'my input value/satoshi', 'cjfee/satoshi', 'earned/satoshi', + 'confirm time/min', 'notes']) + + timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S") + self.log_statement([timestamp, '', '', '', '', '', '', 'Connected']) + + @abc.abstractmethod + def create_my_orders(self): + """Must generate a set of orders to be displayed + according to the contents of the wallet + some algo. + (Note: should be called "create_my_offers") + """ + + @abc.abstractmethod + def oid_to_order(self, cjorder, oid, amount): + """Must convert an order with an offer/order id + into a set of utxos to fill the order. + Also provides the output addresses for the Taker. + """ + + @abc.abstractmethod + def on_tx_unconfirmed(self, cjorder, txid, removed_utxos): + """Performs action on receipt of transaction into the + mempool in the blockchain instance (e.g. announcing orders) + """ + + @abc.abstractmethod + def on_tx_confirmed(self, cjorder, confirmations, txid): + """Performs actions on receipt of 1st confirmation of + a transaction into a block (e.g. announce orders) + """ + + +def ygmain(ygclass, txfee=1000, cjfee_a=200, cjfee_r=0.002, ordertype='reloffer', + nickserv_password='', minsize=100000, mix_levels=5): + import sys + + parser = OptionParser(usage='usage: %prog [options] [wallet file]') + parser.add_option('-o', '--ordertype', action='store', type='string', + dest='ordertype', default=ordertype, + help='type of order; can be either reloffer or absoffer') + parser.add_option('-t', '--txfee', action='store', type='int', + dest='txfee', default=txfee, + help='minimum miner fee in satoshis') + parser.add_option('-c', '--cjfee', action='store', type='string', + dest='cjfee', default='', + help='requested coinjoin fee in satoshis or proportion') + parser.add_option('-p', '--password', action='store', type='string', + dest='password', default=nickserv_password, + help='irc nickserv password') + parser.add_option('-s', '--minsize', action='store', type='int', + dest='minsize', default=minsize, + help='minimum coinjoin size in satoshis') + parser.add_option('-m', '--mixlevels', action='store', type='int', + dest='mixlevels', default=mix_levels, + help='number of mixdepths to use') + (options, args) = parser.parse_args() + if len(args) < 1: + parser.error('Needs a wallet') + sys.exit(0) + seed = args[0] + ordertype = options.ordertype + txfee = options.txfee + if ordertype == 'reloffer': + if options.cjfee != '': + cjfee_r = options.cjfee + # minimum size is such that you always net profit at least 20% + #of the miner fee + minsize = max(int(1.2 * txfee / float(cjfee_r)), options.minsize) + elif ordertype == 'absoffer': + if options.cjfee != '': + cjfee_a = int(options.cjfee) + minsize = options.minsize + else: + parser.error('You specified an incorrect order type which ' +\ + 'can be either reloffer or absoffer') + sys.exit(0) + nickserv_password = options.password + mix_levels = options.mixlevels + + load_program_config() + if isinstance(jm_single().bc_interface, BlockrInterface): + c = ('\nYou are running a yield generator by polling the blockr.io ' + 'website. This is quite bad for privacy. That site is owned by ' + 'coinbase.com Also your bot will run faster and more efficently, ' + 'you can be immediately notified of new bitcoin network ' + 'information so your money will be working for you as hard as ' + 'possibleLearn how to setup JoinMarket with Bitcoin Core: ' + 'https://github.com/chris-belcher/joinmarket/wiki/Running' + '-JoinMarket-with-Bitcoin-Core-full-node') + print(c) + ret = raw_input('\nContinue? (y/n):') + if ret[0] != 'y': + return + + wallet = Wallet(seed, max_mix_depth=mix_levels) + jm_single().bc_interface.sync_wallet(wallet) + + log.debug('starting yield generator') + mcs = [IRCMessageChannel(c, realname='btcint=' + jm_single().config.get( + "BLOCKCHAIN", "blockchain_source"), + password=nickserv_password) for c in get_irc_mchannels()] + mcc = MessageChannelCollection(mcs) + maker = ygclass(mcc, wallet, [options.txfee, cjfee_a, cjfee_r, + options.ordertype, options.minsize, mix_levels]) + try: + log.debug('connecting to message channels') + mcc.run() + except: + log.debug('CRASHING, DUMPING EVERYTHING') + debug_dump_object(wallet, ['addr_cache', 'keys', 'seed']) + debug_dump_object(maker) + debug_dump_object(mcc) + import traceback + log.debug(traceback.format_exc()) + diff --git a/yg-pe.py b/yg-pe.py new file mode 100644 index 00000000..559b4fd4 --- /dev/null +++ b/yg-pe.py @@ -0,0 +1,179 @@ +#! /usr/bin/env python +from __future__ import print_function + +import datetime +import os +import time + +from joinmarket import jm_single, get_network, load_program_config +from joinmarket import get_log, calc_cj_fee, debug_dump_object +from joinmarket import Wallet +from joinmarket import get_irc_mchannels +from joinmarket import YieldGenerator, ygmain + +txfee = 1000 +cjfee_a = 200 +cjfee_r = '0.002' +ordertype = 'reloffer' +nickserv_password = '' +minsize = 100000 +mix_levels = 5 + + +log = get_log() + +# is a maker for the purposes of generating a yield from held +# bitcoins without ruining privacy for the taker, the taker could easily check +# the history of the utxos this bot sends, so theres not much incentive +# to ruin the privacy for barely any more yield +# sell-side algorithm: +# add up the value of each utxo for each mixing depth, +# announce a relative-fee order of the highest balance +# spent from utxos that try to make the highest balance even higher +# so try to keep coins concentrated in one mixing depth +class YieldGeneratorPrivEnhance(YieldGenerator): + + + def __init__(self, msgchan, wallet, offerconfig): + self.txfee, self.cjfee_a, self.cjfee_r, self.ordertype, self.minsize, \ + self.mix_levels = offerconfig + super(YieldGeneratorPrivEnhance,self).__init__(msgchan, wallet) + + def create_my_orders(self): + mix_balance = self.wallet.get_balance_by_mixdepth() + #We publish ONLY the maximum amount and use minsize for lower bound; + #leave it to oid_to_order to figure out the right depth to use. + f = '0' + if ordertype == 'reloffer': + f = self.cjfee_r + #minimum size bumped if necessary such that you always profit + #least 50% of the miner fee + self.minsize = int(1.5 * self.txfee / float(self.cjfee_r)) + elif ordertype == 'absoffer': + f = str(self.txfee + self.cjfee_a) + mix_balance = dict([(m, b) for m, b in mix_balance.iteritems() + if b > self.minsize]) + if len(mix_balance) == 0: + log.debug('do not have any coins left') + return [] + max_mix = max(mix_balance, key=mix_balance.get) + order = {'oid': 0, + 'ordertype': self.ordertype, + 'minsize': self.minsize, + 'maxsize': mix_balance[max_mix] - max( + jm_single().DUST_THRESHOLD, self.txfee), + 'txfee': self.txfee, + 'cjfee': f} + + # sanity check + assert order['minsize'] >= 0 + assert order['maxsize'] > 0 + assert order['minsize'] <= order['maxsize'] + + return [order] + + def oid_to_order(self, cjorder, oid, amount): + """The only change from *basic here (for now) is that + we choose outputs to avoid increasing the max_mixdepth + as much as possible, thus avoiding reannouncement as + much as possible. + """ + total_amount = amount + cjorder.txfee + mix_balance = self.wallet.get_balance_by_mixdepth() + max_mix = max(mix_balance, key=mix_balance.get) + min_mix = min(mix_balance, key=mix_balance.get) + + filtered_mix_balance = [m + for m in mix_balance.iteritems() + if m[1] >= total_amount] + if not filtered_mix_balance: + return None, None, None + + log.debug('mix depths that have enough = ' + str(filtered_mix_balance)) + + #Avoid the max mixdepth wherever possible, to avoid changing the + #offer. Algo: + #"mixdepth" is the mixdepth we are spending FROM, so it is also + #the destination of change. + #"cjoutdepth" is the mixdepth we are sending coinjoin out to. + # + #Find a mixdepth, in the set that have enough, which is + #not the maximum, and choose any from that set as "mixdepth". + #If not possible, it means only the max_mix depth has enough, + #so must choose "mixdepth" to be that. + #To find the cjoutdepth: ensure that max != min, if so it means + #we had only one depth; in that case, just set "cjoutdepth" + #to the next mixdepth. Otherwise, we set "cjoutdepth" to the minimum. + + nonmax_mix_balance = [m for m in filtered_mix_balance if m[0] != max_mix] + mixdepth = None + for m, bal in nonmax_mix_balance: + if m != max_mix: + mixdepth = m + break + if not mixdepth: + log.debug("Could not spend from a mixdepth which is not max") + mixdepth = max_mix + log.debug('filling offer, mixdepth=' + str(mixdepth)) + + # mixdepth is the chosen depth we'll be spending from + # min_mixdepth is the one we want to send our cjout TO, + # to minimize chance of it becoming the largest, and reannouncing offer. + if mixdepth == min_mix: + cjoutmix = (mixdepth + 1) % self.wallet.max_mix_depth + else: + cjoutmix = min_mix + cj_addr = self.wallet.get_internal_addr(cjoutmix) + change_addr = self.wallet.get_internal_addr(mixdepth) + + utxos = self.wallet.select_utxos(mixdepth, total_amount) + my_total_in = sum([va['value'] for va in utxos.values()]) + real_cjfee = calc_cj_fee(cjorder.ordertype, cjorder.cjfee, amount) + change_value = my_total_in - amount - cjorder.txfee + real_cjfee + if change_value <= jm_single().DUST_THRESHOLD: + log.debug(('change value={} below dust threshold, ' + 'finding new utxos').format(change_value)) + try: + utxos = self.wallet.select_utxos( + mixdepth, total_amount + jm_single().DUST_THRESHOLD) + except Exception: + log.debug('dont have the required UTXOs to make a ' + 'output above the dust threshold, quitting') + return None, None, None + + return utxos, cj_addr, change_addr + + def on_tx_unconfirmed(self, cjorder, txid, removed_utxos): + self.tx_unconfirm_timestamp[cjorder.cj_addr] = int(time.time()) + # if the balance of the highest-balance mixing depth change then + # reannounce it + oldorder = self.orderlist[0] if len(self.orderlist) > 0 else None + neworders = self.create_my_orders() + if len(neworders) == 0: + return [0], [] # cancel old order + # oldorder may not exist when this is called from on_tx_confirmed + # (this happens when we just spent from the max mixdepth and so had + # to cancel the order). + if oldorder: + if oldorder['maxsize'] == neworders[0]['maxsize']: + return [], [] # change nothing + # announce new order, replacing the old order + return [], [neworders[0]] + + def on_tx_confirmed(self, cjorder, confirmations, txid): + if cjorder.cj_addr in self.tx_unconfirm_timestamp: + confirm_time = int(time.time()) - self.tx_unconfirm_timestamp[ + cjorder.cj_addr] + else: + confirm_time = 0 + timestamp = datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S") + self.log_statement([timestamp, cjorder.cj_amount, len( + cjorder.utxos), sum([av['value'] for av in cjorder.utxos.values( + )]), cjorder.real_cjfee, cjorder.real_cjfee - cjorder.txfee, round( + confirm_time / 60.0, 2), '']) + return self.on_tx_unconfirmed(cjorder, txid, None) + + +if __name__ == "__main__": + ygmain(YieldGeneratorPrivEnhance) + print('done')