Various bug fixes and improvements for the market

Tribler/Core/APIImplementation/LaunchManyCore.py

@@ -878,6 +878,11 @@ def early_shutdown(self):
             else:
                 self._logger.info("lmc: Dispersy failed to shutdown in %.2f seconds", diff)
 
+        if self.tunnel_community and self.triblerchain_community:
+            # We unload these overlays manually since the triblerchain has to be unloaded after the tunnel overlay.
+            yield self.ipv8.unload_overlay(self.tunnel_community)
+            yield self.ipv8.unload_overlay(self.triblerchain_community)
+
         if self.ipv8:
             yield self.ipv8.stop(stop_reactor=False)
             self.ipv8.endpoint.close()

Tribler/Core/Modules/restapi/market/matchmakers_endpoint.py

@@ -0,0 +1,35 @@
+from Tribler.Core.Modules.restapi.market import BaseMarketEndpoint
+from Tribler.Core.Utilities.json_util import dumps
+
+
+class MatchmakersEndpoint(BaseMarketEndpoint):
+    """
+    This class handles requests regarding your known matchmakers in the market community.
+    """
+
+    def render_GET(self, request):
+        """
+        .. http:get:: /market/matchmakers
+
+        A GET request to this endpoint will return all known matchmakers.
+
+            **Example request**:
+
+            .. sourcecode:: none
+
+                curl -X GET http://localhost:8085/market/matchmakers
+
+            **Example response**:
+
+            .. sourcecode:: javascript
+
+                {
+                    "matchmakers": [{
+                        "ip": "131.249.48.3",
+                        "port": 7008
+                    }]
+                }
+        """
+        matchmakers = self.session.lm.market_community.matchmakers
+        matchmakers_json = [{"ip": mm.address[0], "port": mm.address[1]} for mm in matchmakers]
+        return dumps({"matchmakers": matchmakers_json})

Tribler/Core/Modules/restapi/market_endpoint.py

@@ -1,3 +1,4 @@
+from Tribler.Core.Modules.restapi.market.matchmakers_endpoint import MatchmakersEndpoint
 from twisted.web import resource
 
 from Tribler.Core.Modules.restapi.market.asks_bids_endpoint import AsksEndpoint, BidsEndpoint
@@ -15,6 +16,6 @@ def __init__(self, session):
         self.session = session
 
         child_handler_dict = {"asks": AsksEndpoint, "bids": BidsEndpoint, "transactions": TransactionsEndpoint,
-                              "orders": OrdersEndpoint}
+                              "orders": OrdersEndpoint, "matchmakers": MatchmakersEndpoint}
         for path, child_cls in child_handler_dict.iteritems():
             self.putChild(path, child_cls(self.session))

Tribler/Core/Modules/restapi/wallets_endpoint.py

@@ -232,15 +232,15 @@ def __init__(self, session, identifier):
 
     def render_POST(self, request):
         """
-        .. http:get:: /wallets/(string:wallet identifier)/transfer
+        .. http:post:: /wallets/(string:wallet identifier)/transfer
 
-        A GET request to this endpoint will return past transactions of a specific wallet.
+        A POST request to this endpoint will transfer some units from a wallet to another address.
 
             **Example request**:
 
             .. sourcecode:: none
 
-                curl -X GET http://localhost:8085/wallets/BTC/transfer
+                curl -X POST http://localhost:8085/wallets/BTC/transfer
                 --data "amount=0.3&destination=mpC1DDgSP4PKc5HxJzQ5w9q6CGLBEQuLsN"
 
             **Example response**:

Tribler/Test/Community/Market/Wallet/test_btc_wallet.py

@@ -1,3 +1,4 @@
+from jsonrpclib import ProtocolError
 from twisted.internet.defer import inlineCallbacks, succeed, Deferred
 
 from Tribler.Test.Core.base_test import MockObject
@@ -143,8 +144,43 @@ def test_get_transactions(self):
         wallet = BitcoinWallet(self.session_base_dir)
         mock_daemon = MockObject()
         mock_server = MockObject()
-        transactions = [{'value': -1, 'txid': 'a', 'timestamp': 1}, {'value': 1, 'txid': 'a', 'timestamp': 1}]
+        transactions = [{
+            'value': -1,
+            'txid': 'a',
+            'timestamp': 1,
+            'input_addresses': ['a', 'b'],
+            'output_addresses': ['c', 'd'],
+            'confirmations': 3
+        }, {
+            'value': 1,
+            'txid': 'b',
+            'timestamp': False,  # In Electrum, this means that the transaction has not been confirmed yet
+            'input_addresses': ['a', 'b'],
+            'output_addresses': ['c', 'd'],
+            'confirmations': 0
+        }]
         mock_server.run_cmdline = lambda _: transactions
         mock_daemon.get_server = lambda _: mock_server
         wallet.get_daemon = lambda: mock_daemon
         return wallet.get_transactions()
+
+    @deferred(timeout=10)
+    def test_get_transactions_error(self):
+        """
+        Test whether no transactions are returned when there's a protocol in the JSON RPC protocol
+        """
+        wallet = BitcoinWallet(self.session_base_dir)
+        mock_daemon = MockObject()
+        mock_server = MockObject()
+
+        def failing_run_cmdline(*_):
+            raise ProtocolError()
+
+        mock_server.run_cmdline = failing_run_cmdline
+        mock_daemon.get_server = lambda _: mock_server
+        wallet.get_daemon = lambda: mock_daemon
+
+        def verify_transactions(transactions):
+            self.assertFalse(transactions)
+
+        return wallet.get_transactions().addCallback(verify_transactions)

Tribler/Test/Community/Market/Wallet/test_trustchain_wallet.py

@@ -14,6 +14,7 @@ def setUp(self):
         self.initialize(TrustChainCommunity, 2)
         self.tc_wallet = TrustchainWallet(self.nodes[0].overlay)
         self.tc_wallet.MONITOR_DELAY = 0.01
+        self.tc_wallet.check_negative_balance = True
 
     def create_node(self):
         return MockIPv8(u"curve25519", TrustChainCommunity, working_directory=u":memory:")

Tribler/Test/Community/Market/test_community.py

@@ -184,13 +184,38 @@ def test_failing_payment(self):
             self.nodes[node_nr].overlay.wallets['DUM2'].transfer = lambda *_: fail(RuntimeError("oops"))
 
         yield self.nodes[0].overlay.create_ask(1, 'DUM1', 1, 'DUM2', 3600)
-        yield self.deliver_messages()
         yield self.nodes[1].overlay.create_bid(1, 'DUM1', 1, 'DUM2', 3600)
+
         yield self.deliver_messages()
 
         self.assertEqual(self.nodes[0].overlay.transaction_manager.find_all()[0].status, "error")
         self.assertEqual(self.nodes[1].overlay.transaction_manager.find_all()[0].status, "error")
 
+    @twisted_wrapper
+    def test_proposed_trade_timeout(self):
+        """
+        Test whether we unreserve the quantity if a proposed trade timeouts
+        """
+        yield self.introduce_nodes()
+
+        self.nodes[0].overlay.decode_map[chr(10)] = lambda *_: None
+
+        ask_order = yield self.nodes[0].overlay.create_ask(1, 'DUM1', 1, 'DUM2', 3600)
+        bid_order = yield self.nodes[1].overlay.create_bid(1, 'DUM1', 1, 'DUM2', 3600)
+
+        yield self.deliver_messages(timeout=.5)
+
+        outstanding = self.nodes[1].overlay.get_outstanding_proposals(bid_order.order_id, ask_order.order_id)
+        self.assertTrue(outstanding)
+        outstanding[0][1].on_timeout()
+
+        yield self.deliver_messages(timeout=.5)
+
+        ask_tick_entry = self.nodes[2].overlay.order_book.get_tick(ask_order.order_id)
+        bid_tick_entry = self.nodes[2].overlay.order_book.get_tick(bid_order.order_id)
+        self.assertEqual(float(bid_tick_entry.reserved_for_matching), 0)
+        self.assertEqual(float(ask_tick_entry.reserved_for_matching), 0)
+
 
 class TestMarketCommunityTwoNodes(TestMarketCommunityBase):
     __testing__ = True
@@ -220,3 +245,36 @@ def test_e2e_trade(self):
         balance2 = yield self.nodes[1].overlay.wallets['DUM2'].get_balance()
         self.assertEqual(balance1['available'], 999)
         self.assertEqual(balance2['available'], 1001)
+
+    @twisted_wrapper
+    def test_partial_trade(self):
+        """
+        Test a partial trade
+        """
+        yield self.introduce_nodes()
+
+        yield self.nodes[0].overlay.create_ask(1, 'DUM1', 2, 'DUM2', 3600)
+        bid_order = yield self.nodes[1].overlay.create_bid(1, 'DUM1', 10, 'DUM2', 3600)
+
+        yield self.deliver_messages(timeout=.5)
+
+        # Verify that the trade has been made
+        self.assertEqual(len(self.nodes[0].overlay.transaction_manager.find_all()), 1)
+        self.assertEqual(len(self.nodes[1].overlay.transaction_manager.find_all()), 1)
+
+        # There should be no reserved quantity for the bid tick
+        for node_nr in [0, 1]:
+            bid_tick_entry = self.nodes[node_nr].overlay.order_book.get_tick(bid_order.order_id)
+            self.assertEqual(float(bid_tick_entry.reserved_for_matching), 0)
+
+        yield self.nodes[0].overlay.create_ask(1, 'DUM1', 8, 'DUM2', 3600)
+
+        yield self.deliver_messages(timeout=.5)
+
+        # Verify that the trade has been made
+        self.assertEqual(len(self.nodes[0].overlay.transaction_manager.find_all()), 2)
+        self.assertEqual(len(self.nodes[1].overlay.transaction_manager.find_all()), 2)
+
+        for node_nr in [0, 1]:
+            self.assertEqual(len(self.nodes[node_nr].overlay.order_book.asks), 0)
+            self.assertEqual(len(self.nodes[node_nr].overlay.order_book.bids), 0)

Tribler/Test/Community/Market/test_orderbook.py

@@ -187,13 +187,15 @@ def test_update_ticks(self):
             "trader_id": str(self.ask.order_id.trader_id),
             "order_number": int(self.ask.order_id.order_number),
             "quantity": 3,
-            "quantity_type": self.ask.quantity.wallet_id
+            "quantity_type": self.ask.quantity.wallet_id,
+            "traded_quantity": 3
         }
         bid_dict = {
             "trader_id": str(self.bid.order_id.trader_id),
             "order_number": int(self.bid.order_id.order_number),
             "quantity": 3,
-            "quantity_type": self.bid.quantity.wallet_id
+            "quantity_type": self.bid.quantity.wallet_id,
+            "traded_quantity": 3
         }
 
         self.order_book.get_tick(self.ask.order_id).reserve_for_matching(Quantity(3, self.ask.quantity.wallet_id))

Tribler/Test/Community/Market/test_trade.py

@@ -22,7 +22,6 @@ def test_to_network(self):
         # Test for to network
         self.assertEquals(NotImplemented, self.trade.to_network())
 
-
 class ProposedTradeTestSuite(unittest.TestCase):
     """Proposed trade test cases."""
 
@@ -61,6 +60,19 @@ def test_from_network(self):
         self.assertEquals(Quantity(30, 'MC'), data.quantity)
         self.assertEquals(Timestamp(1462224447.117), data.timestamp)
 
+    def test_has_acceptable_price(self):
+        """
+        Test the acceptable price method
+        """
+        self.assertTrue(self.proposed_trade.has_acceptable_price(True, Price(63000, 'BTC')))
+        self.assertFalse(self.proposed_trade.has_acceptable_price(False, Price(63000, 'BTC')))
+        self.assertTrue(self.proposed_trade.has_acceptable_price(False, Price(64000, 'BTC')))
+        self.assertFalse(self.proposed_trade.has_acceptable_price(True, Price(64000, 'BTC')))
+
+        # Test a price close to the proposed price
+        self.assertTrue(self.proposed_trade.has_acceptable_price(False, Price(63400.0 - 1e-07, 'BTC')))
+        self.assertTrue(self.proposed_trade.has_acceptable_price(True, Price(63400.0 - 1e-07, 'BTC')))
+
 
 class DeclinedTradeTestSuite(unittest.TestCase):
     """Declined trade test cases."""

Tribler/Test/Core/Modules/RestApi/test_market_endpoint.py

@@ -261,3 +261,16 @@ def on_response(response):
         self.should_check_equality = False
         return self.do_request('market/orders/1/cancel', request_type='POST', expected_code=200)\
             .addCallback(on_response)
+
+    @deferred(timeout=10)
+    def test_get_matchmakers(self):
+        """
+        Test the request to fetch known matchmakers
+        """
+        def on_response(response):
+            json_response = json.loads(response)
+            self.assertGreaterEqual(len(json_response['matchmakers']), 1)
+
+        self.session.lm.market_community.matchmakers.add(self.session.lm.market_community.my_peer)
+        self.should_check_equality = False
+        return self.do_request('market/matchmakers', expected_code=200).addCallback(on_response)

Tribler/community/market/community.py

@@ -67,7 +67,7 @@ def on_timeout(self):
         if self.match_id:
             # Inform the matchmaker about the failed trade
             match_message = self.community.incoming_match_messages[self.match_id]
-            self.community.send_decline_match_message(self.match_id, match_message.payload.matchmaker_trader_id,
+            self.community.send_decline_match_message(self.match_id, match_message.matchmaker_trader_id,
                                                       DeclineMatchReason.OTHER)
 
 
@@ -114,7 +114,7 @@ def __init__(self, *args, **kwargs):
         self.use_local_address = False
         self.matching_enabled = True
         self.message_repository = MemoryMessageRepository(self.mid)
-        self.use_incremental_payments = True
+        self.use_incremental_payments = False
         self.matchmakers = set()
         self.pending_matchmaker_deferreds = []
         self.request_cache = RequestCache()
@@ -529,6 +529,7 @@ def received_half_block(self, source_address, data):
                                            TransactionNumber(block.transaction["tx"]["transaction_number"]))
             transaction = self.transaction_manager.find_by_id(transaction_id)
             if transaction and self.market_database.get_linked(block):
+                self.notify_transaction_complete(transaction.to_dictionary(), mine=True)
                 self.send_transaction_completed(transaction, block)
 
         self.process_market_block(block)
@@ -798,8 +799,8 @@ def received_match(self, source_address, data):
         auth, _, payload = self._ez_unpack_auth(MatchPayload, data)
         peer = Peer(auth.public_key_bin, source_address)
 
-        self.logger.debug("We received a match message for order %s.%s",
-                          TraderId(self.mid), payload.recipient_order_number)
+        self.logger.debug("We received a match message for order %s.%s (matched quantity: %s)",
+                          TraderId(self.mid), payload.recipient_order_number, payload.match_quantity)
 
         # We got a match, check whether we can respond to this match
         self.update_ip(payload.matchmaker_trader_id, source_address)
@@ -1032,10 +1033,10 @@ def received_proposed_trade(self, _, data):
             declined_trade = Trade.decline(self.message_repository.next_identity(),
                                            Timestamp.now(), proposed_trade, decline_reason)
             self.logger.debug("Declined trade made with id: %s for proposed trade with id: %s "
-                              "(valid? %s, available quantity of order: %s, reserved: %s, traded: %s)",
+                              "(valid? %s, available quantity of order: %s, reserved: %s, traded: %s), reason: %s",
                               str(declined_trade.message_id), str(proposed_trade.message_id),
                               order.is_valid(), order.available_quantity, order.reserved_quantity,
-                              order.traded_quantity)
+                              order.traded_quantity, decline_reason)
             self.send_declined_trade(declined_trade)
         else:
             self.logger.debug("Proposed trade received with id: %s for order with id: %s",
@@ -1356,7 +1357,7 @@ def on_payment_error(failure):
             When a payment fails, log the error and still send a payment message to inform the other party that the
             payment has failed.
             """
-            self.logger.error("Payment of %f to %s failed: %s", transfer_amount,
+            self.logger.error("Payment of %s to %s failed: %s", transfer_amount,
                               str(transaction.partner_incoming_address), failure.value)
             self.send_payment_message(PaymentId(''), transaction, payment_tup, False)
 
@@ -1449,7 +1450,7 @@ def on_tx_done_signed(block):
             """
             We received the signed block from the counterparty, wrap everything up
             """
-            self.notify_transaction_complete(transaction)
+            self.notify_transaction_complete(transaction.to_dictionary(), mine=True)
             self.send_transaction_completed(transaction, block)
 
         def build_tx_done_block(other_order_dict):
@@ -1481,14 +1482,16 @@ def abort_transaction(self, transaction):
         """
         self.logger.error("Aborting transaction %s", transaction.transaction_id)
         order = self.order_manager.order_repository.find_by_id(transaction.order_id)
-        order.release_quantity_for_tick(transaction.partner_order_id,
-                                        transaction.total_quantity - transaction.transferred_quantity)
-        self.order_manager.order_repository.update(order)
+        if (transaction.total_quantity - transaction.transferred_quantity) > \
+                Quantity(0, transaction.total_quantity.wallet_id):
+            order.release_quantity_for_tick(transaction.partner_order_id,
+                                            transaction.total_quantity - transaction.transferred_quantity)
+            self.order_manager.order_repository.update(order)
 
-    def notify_transaction_complete(self, transaction):
+    def notify_transaction_complete(self, tx_dict, mine=False):
         if self.tribler_session:
             self.tribler_session.notifier.notify(NTFY_MARKET_ON_TRANSACTION_COMPLETE, NTFY_UPDATE, None,
-                                                 transaction.to_dictionary())
+                                                 {"tx": tx_dict, "mine": mine})
 
     def send_transaction_completed(self, transaction, block):
         """
@@ -1535,6 +1538,8 @@ def on_transaction_completed_bc_message(self, block1, _):
         if not self.is_matchmaker:
             return
 
+        self.notify_transaction_complete(tx_dict["tx"])
+
         # Update ticks in order book, release the reserved quantity
         quantity = Quantity(tx_dict["tx"]["quantity"], tx_dict["tx"]["quantity_type"])
         self.order_book.update_ticks(tx_dict["ask"], tx_dict["bid"], quantity, unreserve=False)

Tribler/community/market/core/matching_engine.py

@@ -237,8 +237,8 @@ def _search_for_quantity_in_price_level(self, order_id, tick_entry, quantity_to_
     def _search_for_quantity_in_price_level_total(self, tick_entry, quantity_to_trade):
         trading_quantity = quantity_to_trade
 
-        self._logger.debug("Match with the id (%s) was found: price %i, quantity %i",
-                           str(tick_entry.order_id), float(tick_entry.price), int(trading_quantity))
+        self._logger.debug("Match with the id (%s) was found: price %f, quantity %f",
+                           str(tick_entry.order_id), float(tick_entry.price), float(trading_quantity))
 
         return [(self.get_unique_match_id(), tick_entry, trading_quantity)]
 
@@ -250,8 +250,8 @@ def _search_for_quantity_in_price_level_partial(self, order_id, tick_entry, quan
                 tick_entry.is_blocked_for_matching(order_id):
             quantity_to_trade -= matched_quantity
 
-            self._logger.debug("Match with the id (%s) was found: price %i, quantity %i",
-                               str(tick_entry.order_id), float(tick_entry.price), int(matched_quantity))
+            self._logger.debug("Match with the id (%s) was found: price %f, quantity %f",
+                               str(tick_entry.order_id), float(tick_entry.price), float(matched_quantity))
 
             matching_ticks = [(self.get_unique_match_id(), tick_entry, matched_quantity)]