feat: bigint units & decimalplaces from db

This commit makes two significant changes.

1. It uses the new `bigint` type - supported as of node LTS v14 - for
internal representations of the base "units" of any currencies/tokens.
ETH wei, having 10^18 units per ether, can exceed the capacity of
javascript's `number` type and lead to unexpected rounding or scientific
notation when converting to string that has required hacky workarounds.
Using `bigint` gives us native integer support for base tokens for all
tokens, regardless of how many decimal places they can be split into.

2. It reads the aforementioned decimal places from the database for
every token. Previously, we hardcoded the units per tokens for a handful
of known ERC20 tokens. Adding other currencies to be supported by xud
would have required a code change. Now, new tokens can be supported
with changes to the database via rpc calls, without requiring code
updates.

Closes #1054. Closes #1888.

lib/Xud.ts

@@ -44,7 +44,6 @@ class Xud extends EventEmitter {
   private swaps!: Swaps;
   private shuttingDown = false;
   private swapClientManager?: SwapClientManager;
-  private unitConverter?: UnitConverter;
   private simnetChannels$?: Subscription;
 
   /**
@@ -118,16 +117,16 @@ class Xud extends EventEmitter {
       this.db = new DB(loggers.db, this.config.dbpath);
       await this.db.init(this.config.network, this.config.initdb);
 
-      this.unitConverter = new UnitConverter();
-      this.unitConverter.init();
+      const currencies = await this.db.models.Currency.findAll();
+      const unitConverter = new UnitConverter(currencies);
 
       const nodeKeyPath = NodeKey.getPath(this.config.xudir, this.config.instanceid);
       const nodeKeyExists = await fs
         .access(nodeKeyPath)
         .then(() => true)
         .catch(() => false);
 
-      this.swapClientManager = new SwapClientManager(this.config, loggers, this.unitConverter, this.db.models);
+      this.swapClientManager = new SwapClientManager(this.config, loggers, unitConverter, this.db.models);
       await this.swapClientManager.init();
 
       let nodeKey: NodeKey | undefined;
@@ -182,6 +181,7 @@ class Xud extends EventEmitter {
       const initPromises: Promise<any>[] = [];
 
       this.swaps = new Swaps({
+        unitConverter,
         logger: loggers.swaps,
         models: this.db.models,
         pool: this.pool,
@@ -191,6 +191,7 @@ class Xud extends EventEmitter {
       initPromises.push(this.swaps.init());
 
       this.orderBook = new OrderBook({
+        unitConverter,
         logger: loggers.orderbook,
         models: this.db.models,
         thresholds: this.config.orderthresholds,

lib/connextclient/ConnextClient.ts

@@ -47,11 +47,11 @@ import {
 interface ConnextClient {
   on(event: 'preimage', listener: (preimageRequest: ProvidePreimageEvent) => void): void;
   on(event: 'transferReceived', listener: (transferReceivedRequest: TransferReceivedEvent) => void): void;
-  on(event: 'htlcAccepted', listener: (rHash: string, amount: number, currency: string) => void): this;
+  on(event: 'htlcAccepted', listener: (rHash: string, units: bigint, currency: string) => void): this;
   on(event: 'connectionVerified', listener: (swapClientInfo: SwapClientInfo) => void): this;
   on(event: 'depositConfirmed', listener: (hash: string) => void): this;
   once(event: 'initialized', listener: () => void): this;
-  emit(event: 'htlcAccepted', rHash: string, amount: number, currency: string): boolean;
+  emit(event: 'htlcAccepted', rHash: string, units: bigint, currency: string): boolean;
   emit(event: 'connectionVerified', swapClientInfo: SwapClientInfo): boolean;
   emit(event: 'initialized'): boolean;
   emit(event: 'preimage', preimageRequest: ProvidePreimageEvent): void;
@@ -249,15 +249,15 @@ class ConnextClient extends SwapClient {
    * if one doesn't exist, starts a new request for the specified amount. Then
    * calls channelBalance to refresh the inbound capacity for the currency.
    */
-  private requestCollateralInBackground = (currency: string, units: number) => {
+  private requestCollateralInBackground = (currency: string, units: bigint) => {
     // first check whether we already have a pending collateral request for this currency
     // if not start a new request, and when it completes call channelBalance to refresh our inbound capacity
     const requestCollateralPromise =
       this.requestCollateralPromises.get(currency) ??
       this.sendRequest('/request-collateral', 'POST', {
         channelAddress: this.channelAddress,
         assetId: this.tokenAddresses.get(currency),
-        amount: units.toLocaleString('fullwide', { useGrouping: false }),
+        amount: units.toString(),
         publicIdentifier: this.publicIdentifier,
       })
         .then(() => {
@@ -452,7 +452,7 @@ class ConnextClient extends SwapClient {
       let secret;
       if (deal.role === SwapRole.Maker) {
         // we are the maker paying the taker
-        amount = deal.takerUnits.toLocaleString('fullwide', { useGrouping: false });
+        amount = deal.takerUnits.toString();
         tokenAddress = this.tokenAddresses.get(deal.takerCurrency)!;
         const expiry = await this.getExpiry(this.finalLock);
         const executeTransfer = this.executeHashLockTransfer({
@@ -477,7 +477,7 @@ class ConnextClient extends SwapClient {
         secret = preimage;
       } else {
         // we are the taker paying the maker
-        amount = deal.makerUnits.toLocaleString('fullwide', { useGrouping: false });
+        amount = deal.makerUnits.toString();
         tokenAddress = this.tokenAddresses.get(deal.makerCurrency)!;
         secret = deal.rPreimage!;
         assert(deal.makerCltvDelta, 'cannot send transfer without deal.makerCltvDelta');
@@ -518,7 +518,7 @@ class ConnextClient extends SwapClient {
     currency: expectedCurrency,
   }: {
     rHash: string;
-    units: number;
+    units: bigint;
     expiry?: number;
     currency?: string;
   }) => {
@@ -841,12 +841,12 @@ class ConnextClient extends SwapClient {
 
     const freeBalanceAmount = this.unitConverter.unitsToAmount({
       currency,
-      units: Number(freeBalanceOffChain),
+      units: BigInt(freeBalanceOffChain),
     });
 
     const nodeFreeBalanceAmount = this.unitConverter.unitsToAmount({
       currency,
-      units: Number(nodeFreeBalanceOffChain),
+      units: BigInt(nodeFreeBalanceOffChain),
     });
 
     this.outboundAmounts.set(currency, freeBalanceAmount);
@@ -890,7 +890,7 @@ class ConnextClient extends SwapClient {
 
     const confirmedBalanceAmount = this.unitConverter.unitsToAmount({
       currency,
-      units: Number(freeBalanceOnChain),
+      units: BigInt(freeBalanceOnChain),
     });
 
     return {
@@ -985,7 +985,7 @@ class ConnextClient extends SwapClient {
     const withdrawResponse = await this.sendRequest('/withdraw', 'POST', {
       publicIdentifier: this.publicIdentifier,
       channelAddress: this.channelAddress,
-      amount: amount.toLocaleString('fullwide', { useGrouping: false }),
+      amount: amount.toString(),
       assetId: this.tokenAddresses.get(currency),
       recipient: destination,
       fee: gasPriceGwei,
@@ -1012,19 +1012,13 @@ class ConnextClient extends SwapClient {
   };
 
   // Withdraw on-chain funds
-  public withdraw = async ({
-    all,
-    currency,
-    amount: argAmount,
-    destination,
-    fee,
-  }: WithdrawArguments): Promise<string> => {
+  public withdraw = async ({ all, currency, amount, destination, fee }: WithdrawArguments): Promise<string> => {
     if (fee) {
       // TODO: allow overwriting gas price
       throw Error('setting fee for Ethereum withdrawals is not supported yet');
     }
 
-    let units = '';
+    let unitsStr: string;
 
     const { freeBalanceOnChain } = await this.getBalance(currency);
 
@@ -1033,21 +1027,23 @@ class ConnextClient extends SwapClient {
         // TODO: query Ether balance, subtract gas price times 21000 (gas usage of transferring Ether), and set that as amount
         throw new Error('withdrawing all ETH is not supported yet');
       }
-      units = freeBalanceOnChain;
-    } else if (argAmount) {
-      const argUnits = this.unitConverter.amountToUnits({
+      unitsStr = freeBalanceOnChain;
+    } else if (amount) {
+      const units = this.unitConverter.amountToUnits({
         currency,
-        amount: argAmount,
+        amount,
       });
-      if (Number(freeBalanceOnChain) < argUnits) {
+      if (Number(freeBalanceOnChain) < units) {
         throw errors.INSUFFICIENT_BALANCE;
       }
-      units = argUnits.toString();
+      unitsStr = units.toString();
+    } else {
+      throw new Error('either all must be true or amount must be non-zero');
     }
 
     const res = await this.sendRequest('/onchain-transfer', 'POST', {
       assetId: this.getTokenAddress(currency),
-      amount: units,
+      amount: unitsStr,
       recipient: destination,
     });
     const { txhash } = await parseResponseBody<OnchainTransferResponse>(res);

lib/connextclient/types.ts

@@ -213,7 +213,7 @@ export type TransfersByRoutingIdResponse = ConnextTransfer[];
 
 export type ExpectedIncomingTransfer = {
   rHash: string;
-  units: number;
+  units: bigint;
   expiry: number;
   tokenAddress: string;
   routingId: string;
@@ -316,7 +316,7 @@ export type TransferReceivedEvent = {
   tokenAddress: string;
   rHash: string;
   expiry: number;
-  units: number;
+  units: bigint;
   routingId: string;
 };
 

lib/http/HttpService.ts

@@ -37,7 +37,7 @@ class HttpService {
       const rHash = lockHash.slice(2);
       const expiry = parseInt(expiryString, 10);
       const { amount } = balance;
-      const units = parseInt(amount[0], 10);
+      const units = BigInt(amount[0]);
       await this.service.transferReceived({
         rHash,
         expiry,

lib/lndclient/LndClient.ts

@@ -30,15 +30,15 @@ import { Chain, ChannelCount, ClientMethods, LndClientConfig, LndInfo } from './
 
 interface LndClient {
   on(event: 'connectionVerified', listener: (swapClientInfo: SwapClientInfo) => void): this;
-  on(event: 'htlcAccepted', listener: (rHash: string, amount: number) => void): this;
+  on(event: 'htlcAccepted', listener: (rHash: string, units: bigint) => void): this;
   on(event: 'channelBackup', listener: (channelBackup: Uint8Array) => void): this;
   on(event: 'channelBackupEnd', listener: () => void): this;
   on(event: 'locked', listener: () => void): this;
 
   once(event: 'initialized', listener: () => void): this;
 
   emit(event: 'connectionVerified', swapClientInfo: SwapClientInfo): boolean;
-  emit(event: 'htlcAccepted', rHash: string, amount: number): boolean;
+  emit(event: 'htlcAccepted', rHash: string, units: bigint): boolean;
   emit(event: 'channelBackup', channelBackup: Uint8Array): boolean;
   emit(event: 'channelBackupEnd'): boolean;
   emit(event: 'locked'): boolean;
@@ -502,7 +502,7 @@ class LndClient extends SwapClient {
           const randomHash = crypto.randomBytes(32).toString('hex');
           this.logger.debug(`checking hold invoice support with hash: ${randomHash}`);
 
-          await this.addInvoice({ rHash: randomHash, units: 1 });
+          await this.addInvoice({ rHash: randomHash, units: 1n });
           await this.removeInvoice(randomHash);
         } catch (err) {
           if (err.code !== grpc.status.UNAVAILABLE) {
@@ -846,7 +846,7 @@ class LndClient extends SwapClient {
     remoteIdentifier,
     units,
     uris,
-    pushUnits = 0,
+    pushUnits = 0n,
     fee = 0,
   }: OpenChannelParams): Promise<string> => {
     if (!remoteIdentifier) {
@@ -857,7 +857,7 @@ class LndClient extends SwapClient {
       await this.connectPeerAddresses(uris);
     }
 
-    const openResponse = await this.openChannelSync(remoteIdentifier, units, pushUnits, fee);
+    const openResponse = await this.openChannelSync(remoteIdentifier, Number(units), Number(pushUnits), fee);
     return openResponse.hasFundingTxidStr()
       ? openResponse.getFundingTxidStr()
       : base64ToHex(openResponse.getFundingTxidBytes_asB64());
@@ -927,9 +927,9 @@ class LndClient extends SwapClient {
     );
   };
 
-  public getRoute = async (units: number, destination: string, _currency: string, finalLock = this.finalLock) => {
+  public getRoute = async (units: bigint, destination: string, _currency: string, finalLock = this.finalLock) => {
     const request = new lndrpc.QueryRoutesRequest();
-    request.setAmt(units);
+    request.setAmt(Number(units));
     request.setFinalCltvDelta(finalLock);
     request.setPubKey(destination);
     const fee = new lndrpc.FeeLimit();
@@ -1090,12 +1090,12 @@ class LndClient extends SwapClient {
     expiry = this.finalLock,
   }: {
     rHash: string;
-    units: number;
+    units: bigint;
     expiry?: number;
   }) => {
     const addHoldInvoiceRequest = new lndinvoices.AddHoldInvoiceRequest();
     addHoldInvoiceRequest.setHash(hexToUint8Array(rHash));
-    addHoldInvoiceRequest.setValue(units);
+    addHoldInvoiceRequest.setValue(Number(units));
     addHoldInvoiceRequest.setCltvExpiry(expiry);
     await this.addHoldInvoice(addHoldInvoiceRequest);
     this.logger.debug(`added invoice of ${units} for ${rHash} with cltvExpiry ${expiry}`);
@@ -1224,7 +1224,7 @@ class LndClient extends SwapClient {
         if (invoice.getState() === lndrpc.Invoice.InvoiceState.ACCEPTED) {
           // we have accepted an htlc for this invoice
           this.logger.debug(`accepted htlc for invoice ${rHash}`);
-          this.emit('htlcAccepted', rHash, invoice.getValue());
+          this.emit('htlcAccepted', rHash, BigInt(invoice.getValue()));
         }
       })
       .on('end', deleteInvoiceSubscription)

lib/orderbook/OrderBook.ts

@@ -1,7 +1,6 @@
 import assert from 'assert';
 import { EventEmitter } from 'events';
 import uuidv1 from 'uuid/v1';
-import { UnitConverter } from '../utils/UnitConverter';
 import { SwapClientType, SwapFailureReason, SwapPhase, SwapRole } from '../constants/enums';
 import { Models } from '../db/DB';
 import { CurrencyCreationAttributes, CurrencyInstance, OrderCreationAttributes, PairInstance } from '../db/types';
@@ -12,6 +11,7 @@ import Pool from '../p2p/Pool';
 import Swaps from '../swaps/Swaps';
 import { SwapDeal, SwapFailure, SwapSuccess } from '../swaps/types';
 import { pubKeyToAlias } from '../utils/aliasUtils';
+import { UnitConverter } from '../utils/UnitConverter';
 import { derivePairId, ms, setTimeoutPromise } from '../utils/utils';
 import errors, { errorCodes } from './errors';
 import OrderBookRepository from './OrderBookRepository';
@@ -92,6 +92,7 @@ class OrderBook extends EventEmitter {
   private strict: boolean;
   private pool: Pool;
   private swaps: Swaps;
+  private unitConverter: UnitConverter;
 
   /** Max time for placeOrder iterations (due to swaps failures retries). */
   private static readonly MAX_PLACEORDER_ITERATIONS_TIME = 60000; // 1 min
@@ -113,6 +114,7 @@ class OrderBook extends EventEmitter {
     thresholds,
     pool,
     swaps,
+    unitConverter,
     nosanityswaps,
     nobalancechecks,
     nomatching = false,
@@ -123,6 +125,7 @@ class OrderBook extends EventEmitter {
     thresholds: OrderBookThresholds;
     pool: Pool;
     swaps: Swaps;
+    unitConverter: UnitConverter;
     nosanityswaps: boolean;
     nobalancechecks: boolean;
     nomatching?: boolean;
@@ -133,6 +136,7 @@ class OrderBook extends EventEmitter {
     this.logger = logger;
     this.pool = pool;
     this.swaps = swaps;
+    this.unitConverter = unitConverter;
     this.nomatching = nomatching;
     this.nosanityswaps = nosanityswaps;
     this.nobalancechecks = nobalancechecks;
@@ -147,7 +151,7 @@ class OrderBook extends EventEmitter {
         outboundCurrency,
         inboundAmount,
         outboundAmount,
-      } = UnitConverter.calculateInboundOutboundAmounts(order.quantity, order.price, order.isBuy, order.pairId);
+      } = this.unitConverter.calculateInboundOutboundAmounts(order.quantity, order.price, order.isBuy, order.pairId);
       this.swaps.swapClientManager.subtractInboundReservedAmount(inboundCurrency, inboundAmount);
       this.swaps.swapClientManager.subtractOutboundReservedAmount(outboundCurrency, outboundAmount);
     };
@@ -160,7 +164,7 @@ class OrderBook extends EventEmitter {
         outboundCurrency,
         inboundAmount,
         outboundAmount,
-      } = UnitConverter.calculateInboundOutboundAmounts(order.quantity, order.price, order.isBuy, order.pairId);
+      } = this.unitConverter.calculateInboundOutboundAmounts(order.quantity, order.price, order.isBuy, order.pairId);
       this.swaps.swapClientManager.addInboundReservedAmount(inboundCurrency, inboundAmount);
       this.swaps.swapClientManager.addOutboundReservedAmount(outboundCurrency, outboundAmount);
     });
@@ -371,10 +375,9 @@ class OrderBook extends EventEmitter {
     if (currency.swapClient === SwapClientType.Connext && !currency.tokenAddress) {
       throw errors.CURRENCY_MISSING_ETHEREUM_CONTRACT_ADDRESS(currency.id);
     }
-    const currencyInstance = await this.repository.addCurrency({
-      ...currency,
-      decimalPlaces: currency.decimalPlaces || 8,
-    });
+    const decimalPlaces = currency.decimalPlaces || 8;
+    const currencyInstance = await this.repository.addCurrency({ ...currency, decimalPlaces });
+    this.unitConverter.setDecimalPlacesPerCurrency(currency.id, decimalPlaces);
     this.currencyInstances.set(currencyInstance.id, currencyInstance);
     await this.swaps.swapClientManager.add(currencyInstance);
   };