Merge remote-tracking branch 'upstream/master'

contrib/requirements/requirements-hw.txt

@@ -11,8 +11,10 @@ keepkey>=6.3.1
 
 # device plugin: ledger
 # note: btchip-python only needed for "legacy" protocol and HW.1 support
+# note: ledger-bitcoin==0.2.2 added new deps we don't want to bundle. otherwise it should be ok to use.
+#       see https://github.com/LedgerHQ/app-bitcoin-new/issues/192
 btchip-python>=0.1.32
-ledger-bitcoin>=0.2.0,<0.3.0
+ledger-bitcoin>=0.2.0,<0.2.2
 hidapi
 
 # device plugin: coldcard

electrum/plugins/swapserver/server.py

@@ -3,13 +3,11 @@
 from collections import defaultdict
 
 from aiohttp import web
-from aiorpcx import NetAddress
-
 
 from electrum_grs.util import log_exceptions, ignore_exceptions
 from electrum_grs.logging import Logger
 from electrum_grs.util import EventListener
-
+from electrum_grs.lnaddr import lndecode
 
 class SwapServer(Logger, EventListener):
     """
@@ -24,7 +22,8 @@ def __init__(self, config, wallet):
         Logger.__init__(self)
         self.config = config
         self.wallet = wallet
-        self.addr = NetAddress.from_string(self.config.SWAPSERVER_ADDRESS)
+        self.sm = self.wallet.lnworker.swap_manager
+        self.port = self.config.SWAPSERVER_PORT
         self.register_callbacks() # eventlistener
 
         self.pending = defaultdict(asyncio.Event)
@@ -34,21 +33,24 @@ def __init__(self, config, wallet):
     @log_exceptions
     async def run(self):
         app = web.Application()
-        app.add_routes([web.get('/api/getpairs', self.get_pairs)])
-        app.add_routes([web.post('/api/createswap', self.create_swap)])
+        app.add_routes([web.get('/getpairs', self.get_pairs)])
+        app.add_routes([web.post('/createswap', self.create_swap)])
+        app.add_routes([web.post('/createnormalswap', self.create_normal_swap)])
+        app.add_routes([web.post('/addswapinvoice', self.add_swap_invoice)])
 
         runner = web.AppRunner(app)
         await runner.setup()
-        site = web.TCPSite(runner, host=str(self.addr.host), port=self.addr.port, ssl_context=self.config.get_ssl_context())
+        site = web.TCPSite(runner, host='localhost', port=self.port)
         await site.start()
-        self.logger.info(f"now running and listening. addr={self.addr}")
+        self.logger.info(f"running and listening on port {self.port}")
 
     async def get_pairs(self, r):
-        sm = self.wallet.lnworker.swap_manager
+        sm = self.sm
         sm.init_pairs()
         pairs = {
             "info": [],
             "warnings": [],
+            "htlcFirst": True,
             "pairs": {
                 "GRS/GRS": {
                     "rate": 1,
@@ -84,9 +86,36 @@ async def get_pairs(self, r):
         }
         return web.json_response(pairs)
 
+    async def add_swap_invoice(self, r):
+        request = await r.json()
+        invoice = request['invoice']
+        self.sm.add_invoice(invoice, pay_now=True)
+        return web.json_response({})
+
+    async def create_normal_swap(self, r):
+        # normal for client, reverse for server
+        request = await r.json()
+        lightning_amount_sat = request['invoiceAmount']
+        their_pubkey = bytes.fromhex(request['refundPublicKey'])
+        assert len(their_pubkey) == 33
+        swap = self.sm.create_reverse_swap(
+            payment_hash=None,
+            lightning_amount_sat=lightning_amount_sat,
+            their_pubkey=their_pubkey
+        )
+        response = {
+            "id": swap.payment_hash.hex(),
+            'preimageHash': swap.payment_hash.hex(),
+            "acceptZeroConf": False,
+            "expectedAmount": swap.onchain_amount,
+            "timeoutBlockHeight": swap.locktime,
+            "address": swap.lockup_address,
+            "redeemScript": swap.redeem_script.hex(),
+        }
+        return web.json_response(response)
+
     async def create_swap(self, r):
-        sm = self.wallet.lnworker.swap_manager
-        sm.init_pairs()
+        self.sm.init_pairs()
         request = await r.json()
         req_type = request['type']
         assert request['pairId'] == 'GRS/GRS'
@@ -96,7 +125,7 @@ async def create_swap(self, r):
             their_pubkey=bytes.fromhex(request['claimPublicKey'])
             assert len(payment_hash) == 32
             assert len(their_pubkey) == 33
-            swap, payment_hash, invoice, prepay_invoice = sm.add_server_swap(
+            swap, invoice, prepay_invoice = self.sm.create_normal_swap(
                 lightning_amount_sat=lightning_amount_sat,
                 payment_hash=payment_hash,
                 their_pubkey=their_pubkey
@@ -111,13 +140,19 @@ async def create_swap(self, r):
                 "onchainAmount": swap.onchain_amount,
             }
         elif req_type == 'submarine':
+            # old protocol
             their_invoice=request['invoice']
             their_pubkey=bytes.fromhex(request['refundPublicKey'])
             assert len(their_pubkey) == 33
-            swap, payment_hash, invoice, prepay_invoice = sm.add_server_swap(
-                invoice=their_invoice,
+            lnaddr = lndecode(their_invoice)
+            payment_hash = lnaddr.paymenthash
+            lightning_amount_sat = int(lnaddr.get_amount_sat()) # should return int
+            swap = self.sm.create_reverse_swap(
+                lightning_amount_sat=lightning_amount_sat,
+                payment_hash=payment_hash,
                 their_pubkey=their_pubkey
             )
+            self.sm.add_invoice(their_invoice, pay_now=False)
             response = {
                 "id": payment_hash.hex(),
                 "acceptZeroConf": False,

electrum/plugins/swapserver/swapserver.py

@@ -55,7 +55,6 @@ def daemon_wallet_loaded(self, daemon: 'Daemon', wallet: 'Abstract_Wallet'):
         self.server = SwapServer(self.config, wallet)
         sm = wallet.lnworker.swap_manager
         for coro in [
-            sm.pay_pending_invoices(),  # FIXME this method can raise, which is not properly handled...?
             self.server.run(),
         ]:
             asyncio.run_coroutine_threadsafe(daemon.taskgroup.spawn(coro), daemon.asyncio_loop)

electrum_grs/base_wizard.py

@@ -681,12 +681,12 @@ def create_storage(self, path) -> Tuple[WalletStorage, WalletDB]:
         storage = WalletStorage(path)
         if pw_args.encrypt_storage:
             storage.set_password(pw_args.password, enc_version=pw_args.storage_enc_version)
-        db = WalletDB('', manual_upgrades=False)
+        db = WalletDB('', storage=storage, manual_upgrades=False)
         db.set_keystore_encryption(bool(pw_args.password) and pw_args.encrypt_keystore)
         for key, value in self.data.items():
             db.put(key, value)
         db.load_plugins()
-        db.write(storage)
+        db.write()
         return storage, db
 
     def terminate(self, *, storage: WalletStorage = None,

electrum_grs/channel_db.py

@@ -307,14 +307,19 @@ def get_mychannel_policy(short_channel_id: bytes, node_id: bytes,
 class ChannelDB(SqlDB):
 
     NUM_MAX_RECENT_PEERS = 20
+    PRIVATE_CHAN_UPD_CACHE_TTL_NORMAL = 600
+    PRIVATE_CHAN_UPD_CACHE_TTL_SHORT = 120
 
     def __init__(self, network: 'Network'):
         path = self.get_file_path(network.config)
         super().__init__(network.asyncio_loop, path, commit_interval=100)
         self.lock = threading.RLock()
         self.num_nodes = 0
         self.num_channels = 0
-        self._channel_updates_for_private_channels = {}  # type: Dict[Tuple[bytes, bytes], dict]
+        self._channel_updates_for_private_channels = {}  # type: Dict[Tuple[bytes, bytes], Tuple[dict, int]]
+        # note: ^ we could maybe move this cache into PaySession instead of being global.
+        #       That would only make sense though if PaySessions were never too short
+        #       (e.g. consider trampoline forwarding).
         self.ca_verifier = LNChannelVerifier(network, self)
 
         # initialized in load_data
@@ -667,19 +672,46 @@ def prune_orphaned_channels(self):
             self.update_counts()
             self.logger.info(f'Deleting {len(orphaned_chans)} orphaned channels')
 
-    def add_channel_update_for_private_channel(self, msg_payload: dict, start_node_id: bytes) -> bool:
+    def _get_channel_update_for_private_channel(
+        self,
+        start_node_id: bytes,
+        short_channel_id: ShortChannelID,
+        *,
+        now: int = None,  # unix ts
+    ) -> Optional[dict]:
+        if now is None:
+            now = int(time.time())
+        key = (start_node_id, short_channel_id)
+        chan_upd_dict, cache_expiration = self._channel_updates_for_private_channels.get(key, (None, 0))
+        if cache_expiration < now:
+            chan_upd_dict = None  # already expired
+            # TODO rm expired entries from cache (note: perf vs thread-safety)
+        return chan_upd_dict
+
+    def add_channel_update_for_private_channel(
+        self,
+        msg_payload: dict,
+        start_node_id: bytes,
+        *,
+        cache_ttl: int = None,  # seconds
+    ) -> bool:
         """Returns True iff the channel update was successfully added and it was different than
         what we had before (if any).
         """
         if not verify_sig_for_channel_update(msg_payload, start_node_id):
             return False  # ignore
+        now = int(time.time())
         short_channel_id = ShortChannelID(msg_payload['short_channel_id'])
         msg_payload['start_node'] = start_node_id
-        key = (start_node_id, short_channel_id)
-        prev_chanupd = self._channel_updates_for_private_channels.get(key)
+        prev_chanupd = self._get_channel_update_for_private_channel(start_node_id, short_channel_id, now=now)
         if prev_chanupd == msg_payload:
             return False
-        self._channel_updates_for_private_channels[key] = msg_payload
+        if cache_ttl is None:
+            cache_ttl = self.PRIVATE_CHAN_UPD_CACHE_TTL_NORMAL
+        cache_expiration = now + cache_ttl
+        key = (start_node_id, short_channel_id)
+        with self.lock:
+            self._channel_updates_for_private_channels[key] = msg_payload, cache_expiration
         return True
 
     def remove_channel(self, short_channel_id: ShortChannelID):
@@ -791,21 +823,20 @@ def get_num_channels_partitioned_by_policy_count(self) -> Tuple[int, int, int]:
 
     def get_policy_for_node(
             self,
-            short_channel_id: bytes,
+            short_channel_id: ShortChannelID,
             node_id: bytes,
             *,
             my_channels: Dict[ShortChannelID, 'Channel'] = None,
             private_route_edges: Dict[ShortChannelID, 'RouteEdge'] = None,
+            now: int = None,  # unix ts
     ) -> Optional['Policy']:
         channel_info = self.get_channel_info(short_channel_id)
         if channel_info is not None:  # publicly announced channel
             policy = self._policies.get((node_id, short_channel_id))
             if policy:
                 return policy
-        else:  # private channel
-            chan_upd_dict = self._channel_updates_for_private_channels.get((node_id, short_channel_id))
-            if chan_upd_dict:
-                return Policy.from_msg(chan_upd_dict)
+        elif chan_upd_dict := self._get_channel_update_for_private_channel(node_id, short_channel_id, now=now):
+            return Policy.from_msg(chan_upd_dict)
         # check if it's one of our own channels
         if my_channels:
             policy = get_mychannel_policy(short_channel_id, node_id, my_channels)

electrum_grs/commands.py

@@ -1204,12 +1204,13 @@ async def enable_htlc_settle(self, b: bool, wallet: Abstract_Wallet = None):
     @command('n')
     async def clear_ln_blacklist(self):
         if self.network.path_finder:
-            self.network.path_finder.liquidity_hints.clear_blacklist()
+            self.network.path_finder.clear_blacklist()
 
     @command('n')
     async def reset_liquidity_hints(self):
         if self.network.path_finder:
             self.network.path_finder.liquidity_hints.reset_liquidity_hints()
+            self.network.path_finder.clear_blacklist()
 
     @command('wnl')
     async def close_channel(self, channel_point, force=False, wallet: Abstract_Wallet = None):

electrum_grs/daemon.py

@@ -357,8 +357,8 @@ async def run_cmdline(self, config_options):
 
 class WatchTowerServer(AuthenticatedServer):
 
-    def __init__(self, network: 'Network', netaddress):
-        self.addr = netaddress
+    def __init__(self, network: 'Network', port:int):
+        self.port = port
         self.config = network.config
         self.network = network
         watchtower_user = self.config.WATCHTOWER_SERVER_USER or ""
@@ -373,9 +373,9 @@ def __init__(self, network: 'Network', netaddress):
     async def run(self):
         self.runner = web.AppRunner(self.app)
         await self.runner.setup()
-        site = web.TCPSite(self.runner, host=str(self.addr.host), port=self.addr.port, ssl_context=self.config.get_ssl_context())
+        site = web.TCPSite(self.runner, host='localhost', port=self.port)
         await site.start()
-        self.logger.info(f"now running and listening. addr={self.addr}")
+        self.logger.info(f"running and listening on port {self.port}")
 
     async def get_ctn(self, *args):
         return await self.lnwatcher.get_ctn(*args)
@@ -453,8 +453,8 @@ def start_network(self):
         assert not self.config.NETWORK_OFFLINE
         assert self.network
         # server-side watchtower
-        if watchtower_address := self.config.get_netaddress(self.config.cv.WATCHTOWER_SERVER_ADDRESS):
-            self.watchtower = WatchTowerServer(self.network, watchtower_address)
+        if watchtower_port := self.config.WATCHTOWER_SERVER_PORT:
+            self.watchtower = WatchTowerServer(self.network, watchtower_port)
             asyncio.run_coroutine_threadsafe(self.taskgroup.spawn(self.watchtower.run), self.asyncio_loop)
 
         self.network.start(jobs=[self.fx.run])
@@ -511,14 +511,14 @@ def _load_wallet(
                 return
             storage.decrypt(password)
         # read data, pass it to db
-        db = WalletDB(storage.read(), manual_upgrades=manual_upgrades)
+        db = WalletDB(storage.read(), storage=storage, manual_upgrades=manual_upgrades)
         if db.requires_split():
             return
         if db.requires_upgrade():
             return
         if db.get_action():
             return
-        wallet = Wallet(db, storage, config=config)
+        wallet = Wallet(db, config=config)
         return wallet
 
     @with_wallet_lock

electrum_grs/gui/kivy/main_window.py

@@ -673,13 +673,13 @@ def get_wallet_path(self):
         else:
             return ''
 
-    def on_wizard_success(self, storage, db, password):
+    def on_wizard_success(self, db, password):
         self.password = password
         if self.electrum_config.WALLET_USE_SINGLE_PASSWORD:
             self._use_single_password = self.daemon.update_password_for_directory(
                 old_password=password, new_password=password)
         self.logger.info(f'use single password: {self._use_single_password}')
-        wallet = Wallet(db, storage, config=self.electrum_config)
+        wallet = Wallet(db, config=self.electrum_config)
         wallet.start_network(self.daemon.network)
         self.daemon.add_wallet(wallet)
         self.load_wallet(wallet)
@@ -714,7 +714,7 @@ def on_open_wallet(self, password, storage):
             wizard.run('new')
         else:
             assert storage.is_past_initial_decryption()
-            db = WalletDB(storage.read(), manual_upgrades=False)
+            db = WalletDB(storage.read(), storage=storage, manual_upgrades=False)
             assert not db.requires_upgrade()
             self.on_wizard_success(storage, db, password)
 

electrum_grs/gui/kivy/uix/dialogs/password_dialog.py

@@ -359,8 +359,8 @@ def init_storage_from_path(self, path):
         else:
             # it is a bit wasteful load the wallet here and load it again in main_window,
             # but that is fine, because we are progressively enforcing storage encryption.
-            db = WalletDB(self.storage.read(), manual_upgrades=False)
-            wallet = Wallet(db, self.storage, config=self.app.electrum_config)
+            db = WalletDB(self.storage.read(), storage=self.storage, manual_upgrades=False)
+            wallet = Wallet(db, config=self.app.electrum_config)
             self.require_password = wallet.has_password()
             self.pw_check = wallet.check_password
             self.message = self.enter_pw_message if self.require_password else _('Wallet not encrypted')

electrum_grs/gui/qml/components/WalletMainView.qml

@@ -312,6 +312,10 @@ Item {
             })
             dialog.open()
         }
+        function onImportChannelBackupFailed(message) {
+            var dialog = app.messageDialog.createObject(app, { title: qsTr('Error'), text: message })
+            dialog.open()
+        }
     }
 
     Component {

electrum_grs/gui/qml/components/controls/ElListView.qml

@@ -26,13 +26,15 @@ ListView {
     // android back gesture is used
     function layoutExclusionZones() {
         var reserve = constants.fingerWidth / 2
-        var p = root.mapToGlobal(0, 0)
+        var p = root.mapToGlobal(0, 0)  // note: coords on whole *screen*, not just window
         width_left_exclusion_zone = Math.max(0, reserve - p.x)
         p = root.mapToGlobal(width, 0)
         width_right_exclusion_zone = Math.max(0, reserve - (app.width - p.x))
     }
 
     Component.onCompleted: {
-        layoutExclusionZones()
+        if (AppController.isAndroid()) {
+            layoutExclusionZones()
+        }
     }
 }

electrum_grs/gui/qml/qechannelopener.py

@@ -141,7 +141,7 @@ def validateConnectString(self, connect_str):
             return False
         return True
 
-    # FIXME "max" button in amount_dialog should enforce LN_MAX_FUNDING_SAT
+    # FIXME "max" button in amount_dialog should enforce LIGHTNING_MAX_FUNDING_SAT
     @pyqtSlot()
     @pyqtSlot(bool)
     def openChannel(self, confirm_backup_conflict=False):