Merge branch 'master' into maxHtlcValueInFlight_comparison

# Conflicts:
#	eclair-core/src/main/scala/fr/acinq/eclair/package.scala
#	eclair-core/src/test/scala/fr/acinq/eclair/payment/RelayerSpec.scala

eclair-core/src/main/resources/reference.conf

@@ -117,6 +117,11 @@ eclair {
     broadcast-interval = 60 seconds // see BOLT #7
     init-timeout = 5 minutes
 
+    sync {
+      request-node-announcements = true // if true we will ask for node announcements when we receive channel ids that we don't know
+      encoding-type = zlib // encoding for short_channel_ids and timestamps in query channel sync messages; other possible value is "uncompressed"
+    }
+
     // the values below will be used to perform route searching
     path-finding {
       max-route-length = 6         // max route length for the 'first pass', if none is found then a second pass is made with no limit

eclair-core/src/main/scala/fr/acinq/eclair/CltvExpiry.scala

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2019 ACINQ SAS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fr.acinq.eclair
+
+/**
+ * Created by t-bast on 21/08/2019.
+ */
+
+/**
+ * Bitcoin scripts (in particular HTLCs) need an absolute block expiry (greater than the current block count) to work
+ * with OP_CLTV.
+ *
+ * @param underlying the absolute cltv expiry value (current block count + some delta).
+ */
+case class CltvExpiry(private val underlying: Long) extends Ordered[CltvExpiry] {
+  // @formatter:off
+  def +(d: CltvExpiryDelta): CltvExpiry = CltvExpiry(underlying + d.toInt)
+  def -(d: CltvExpiryDelta): CltvExpiry = CltvExpiry(underlying - d.toInt)
+  def -(other: CltvExpiry): CltvExpiryDelta = CltvExpiryDelta((underlying - other.underlying).toInt)
+  override def compare(other: CltvExpiry): Int = underlying.compareTo(other.underlying)
+  def toLong: Long = underlying
+  // @formatter:on
+}
+
+/**
+ * Channels advertise a cltv expiry delta that should be used when routing through them.
+ * This value needs to be converted to a [[fr.acinq.eclair.CltvExpiry]] to be used in OP_CLTV.
+ *
+ * CltvExpiryDelta can also be used when working with OP_CSV which is by design a delta.
+ *
+ * @param underlying the cltv expiry delta value.
+ */
+case class CltvExpiryDelta(private val underlying: Int) extends Ordered[CltvExpiryDelta] {
+
+  /**
+   * Adds the current block height to the given delta to obtain an absolute expiry.
+   */
+  def toCltvExpiry = CltvExpiry(Globals.blockCount.get() + underlying)
+
+  // @formatter:off
+  def +(other: Int): CltvExpiryDelta = CltvExpiryDelta(underlying + other)
+  def +(other: CltvExpiryDelta): CltvExpiryDelta = CltvExpiryDelta(underlying + other.underlying)
+  def compare(other: CltvExpiryDelta): Int = underlying.compareTo(other.underlying)
+  def toInt: Int = underlying
+  // @formatter:on
+
+}

eclair-core/src/main/scala/fr/acinq/eclair/CoinUtils.scala

@@ -17,14 +17,16 @@
 package fr.acinq.eclair
 
 import java.text.{DecimalFormat, NumberFormat}
+
 import fr.acinq.bitcoin.{Btc, BtcAmount, MilliBtc, Satoshi}
 import grizzled.slf4j.Logging
+
 import scala.util.{Failure, Success, Try}
 
 /**
-  * Internal UI utility class, useful for lossless conversion between BtcAmount.
-  * The issue being that Satoshi contains a Long amount and it can not be converted to MilliSatoshi without losing the decimal part.
-  */
+ * Internal UI utility class, useful for lossless conversion between BtcAmount.
+ * The issue being that Satoshi contains a Long amount and it can not be converted to MilliSatoshi without losing the decimal part.
+ */
 private sealed trait BtcAmountGUILossless {
   def amount_msat: Long
   def unit: CoinUnit
@@ -124,15 +126,15 @@ object CoinUtils extends Logging {
   }
 
   /**
-    * Converts a string amount denominated in a bitcoin unit to a Millisatoshi amount. The amount might be truncated if
-    * it has too many decimals because MilliSatoshi only accepts Long amount.
-    *
-    * @param amount numeric String, can be decimal.
-    * @param unit   bitcoin unit, can be milliSatoshi, Satoshi, Bits, milliBTC, BTC.
-    * @return       amount as a MilliSatoshi object.
-    * @throws NumberFormatException    if the amount parameter is not numeric.
-    * @throws IllegalArgumentException if the unit is not equals to milliSatoshi, Satoshi or milliBTC.
-    */
+   * Converts a string amount denominated in a bitcoin unit to a Millisatoshi amount. The amount might be truncated if
+   * it has too many decimals because MilliSatoshi only accepts Long amount.
+   *
+   * @param amount numeric String, can be decimal.
+   * @param unit   bitcoin unit, can be milliSatoshi, Satoshi, Bits, milliBTC, BTC.
+   * @return amount as a MilliSatoshi object.
+   * @throws NumberFormatException    if the amount parameter is not numeric.
+   * @throws IllegalArgumentException if the unit is not equals to milliSatoshi, Satoshi or milliBTC.
+   */
   @throws(classOf[NumberFormatException])
   @throws(classOf[IllegalArgumentException])
   def convertStringAmountToMsat(amount: String, unit: String): MilliSatoshi = {
@@ -152,13 +154,11 @@ object CoinUtils extends Logging {
   }
 
   def convertStringAmountToSat(amount: String, unit: String): Satoshi =
-   CoinUtils.convertStringAmountToMsat(amount, unit).truncateToSatoshi
+    CoinUtils.convertStringAmountToMsat(amount, unit).truncateToSatoshi
 
   /**
-    * Only BtcUnit, MBtcUnit, BitUnit, SatUnit and MSatUnit codes or label are supported.
-    * @param unit
-    * @return
-    */
+   * Only BtcUnit, MBtcUnit, BitUnit, SatUnit and MSatUnit codes or label are supported.
+   */
   def getUnitFromString(unit: String): CoinUnit = unit.toLowerCase() match {
     case u if u == MSatUnit.code || u == MSatUnit.label.toLowerCase() => MSatUnit
     case u if u == SatUnit.code || u == SatUnit.label.toLowerCase() => SatUnit
@@ -169,53 +169,53 @@ object CoinUtils extends Logging {
   }
 
   /**
-    * Converts BtcAmount to a GUI Unit (wrapper containing amount as a millisatoshi long)
-    *
-    * @param amount BtcAmount
-    * @param unit unit to convert to
-    * @return a GUICoinAmount
-    */
+   * Converts BtcAmount to a GUI Unit (wrapper containing amount as a millisatoshi long)
+   *
+   * @param amount BtcAmount
+   * @param unit   unit to convert to
+   * @return a GUICoinAmount
+   */
   private def convertAmountToGUIUnit(amount: BtcAmount, unit: CoinUnit): BtcAmountGUILossless = (amount, unit) match {
     // amount is msat, so no conversion required
-    case (a: MilliSatoshi, MSatUnit) => GUIMSat(a.amount * MSatUnit.factorToMsat)
-    case (a: MilliSatoshi, SatUnit) => GUISat(a.amount * MSatUnit.factorToMsat)
-    case (a: MilliSatoshi, BitUnit) => GUIBits(a.amount * MSatUnit.factorToMsat)
-    case (a: MilliSatoshi, MBtcUnit) => GUIMBtc(a.amount * MSatUnit.factorToMsat)
-    case (a: MilliSatoshi, BtcUnit) => GUIBtc(a.amount * MSatUnit.factorToMsat)
+    case (a: MilliSatoshi, MSatUnit) => GUIMSat(a.toLong * MSatUnit.factorToMsat)
+    case (a: MilliSatoshi, SatUnit) => GUISat(a.toLong * MSatUnit.factorToMsat)
+    case (a: MilliSatoshi, BitUnit) => GUIBits(a.toLong * MSatUnit.factorToMsat)
+    case (a: MilliSatoshi, MBtcUnit) => GUIMBtc(a.toLong * MSatUnit.factorToMsat)
+    case (a: MilliSatoshi, BtcUnit) => GUIBtc(a.toLong * MSatUnit.factorToMsat)
 
     // amount is satoshi, convert sat -> msat
-    case (a: Satoshi, MSatUnit) => GUIMSat(a.amount * SatUnit.factorToMsat)
-    case (a: Satoshi, SatUnit) => GUISat(a.amount * SatUnit.factorToMsat)
-    case (a: Satoshi, BitUnit) => GUIBits(a.amount * SatUnit.factorToMsat)
-    case (a: Satoshi, MBtcUnit) => GUIMBtc(a.amount * SatUnit.factorToMsat)
-    case (a: Satoshi, BtcUnit) => GUIBtc(a.amount * SatUnit.factorToMsat)
+    case (a: Satoshi, MSatUnit) => GUIMSat(a.toLong * SatUnit.factorToMsat)
+    case (a: Satoshi, SatUnit) => GUISat(a.toLong * SatUnit.factorToMsat)
+    case (a: Satoshi, BitUnit) => GUIBits(a.toLong * SatUnit.factorToMsat)
+    case (a: Satoshi, MBtcUnit) => GUIMBtc(a.toLong * SatUnit.factorToMsat)
+    case (a: Satoshi, BtcUnit) => GUIBtc(a.toLong * SatUnit.factorToMsat)
 
     // amount is mbtc
-    case (a: MilliBtc, MSatUnit) => GUIMSat((a.amount * MBtcUnit.factorToMsat).toLong)
-    case (a: MilliBtc, SatUnit) => GUISat((a.amount * MBtcUnit.factorToMsat).toLong)
-    case (a: MilliBtc, BitUnit) => GUIBits((a.amount * MBtcUnit.factorToMsat).toLong)
-    case (a: MilliBtc, MBtcUnit) => GUIMBtc((a.amount * MBtcUnit.factorToMsat).toLong)
-    case (a: MilliBtc, BtcUnit) => GUIBtc((a.amount * MBtcUnit.factorToMsat).toLong)
+    case (a: MilliBtc, MSatUnit) => GUIMSat((a.toBigDecimal * MBtcUnit.factorToMsat).toLong)
+    case (a: MilliBtc, SatUnit) => GUISat((a.toBigDecimal * MBtcUnit.factorToMsat).toLong)
+    case (a: MilliBtc, BitUnit) => GUIBits((a.toBigDecimal * MBtcUnit.factorToMsat).toLong)
+    case (a: MilliBtc, MBtcUnit) => GUIMBtc((a.toBigDecimal * MBtcUnit.factorToMsat).toLong)
+    case (a: MilliBtc, BtcUnit) => GUIBtc((a.toBigDecimal * MBtcUnit.factorToMsat).toLong)
 
     // amount is mbtc
-    case (a: Btc, MSatUnit) => GUIMSat((a.amount * BtcUnit.factorToMsat).toLong)
-    case (a: Btc, SatUnit) => GUISat((a.amount * BtcUnit.factorToMsat).toLong)
-    case (a: Btc, BitUnit) => GUIBits((a.amount * BtcUnit.factorToMsat).toLong)
-    case (a: Btc, MBtcUnit) => GUIMBtc((a.amount * BtcUnit.factorToMsat).toLong)
-    case (a: Btc, BtcUnit) => GUIBtc((a.amount * BtcUnit.factorToMsat).toLong)
+    case (a: Btc, MSatUnit) => GUIMSat((a.toBigDecimal * BtcUnit.factorToMsat).toLong)
+    case (a: Btc, SatUnit) => GUISat((a.toBigDecimal * BtcUnit.factorToMsat).toLong)
+    case (a: Btc, BitUnit) => GUIBits((a.toBigDecimal * BtcUnit.factorToMsat).toLong)
+    case (a: Btc, MBtcUnit) => GUIMBtc((a.toBigDecimal * BtcUnit.factorToMsat).toLong)
+    case (a: Btc, BtcUnit) => GUIBtc((a.toBigDecimal * BtcUnit.factorToMsat).toLong)
 
-    case (a, _) =>
+    case (_, _) =>
       throw new IllegalArgumentException(s"unhandled conversion from $amount to $unit")
   }
 
   /**
-    * Converts the amount to the user preferred unit and returns a localized formatted String.
-    * This method is useful for read only displays.
-    *
-    * @param amount BtcAmount
-    * @param withUnit if true, append the user unit shortLabel (mBTC, BTC, mSat...)
-    * @return formatted amount
-    */
+   * Converts the amount to the user preferred unit and returns a localized formatted String.
+   * This method is useful for read only displays.
+   *
+   * @param amount   BtcAmount
+   * @param withUnit if true, append the user unit shortLabel (mBTC, BTC, mSat...)
+   * @return formatted amount
+   */
   def formatAmountInUnit(amount: BtcAmount, unit: CoinUnit, withUnit: Boolean = false): String = {
     val formatted = COIN_FORMAT.format(rawAmountInUnit(amount, unit))
     if (withUnit) s"$formatted ${unit.shortLabel}" else formatted
@@ -227,14 +227,14 @@ object CoinUtils extends Logging {
   }
 
   /**
-    * Converts the amount to the user preferred unit and returns the BigDecimal value.
-    * This method is useful to feed numeric text input without formatting.
-    *
-    * Returns -1 if the given amount can not be converted.
-    *
-    * @param amount BtcAmount
-    * @return BigDecimal value of the BtcAmount
-    */
+   * Converts the amount to the user preferred unit and returns the BigDecimal value.
+   * This method is useful to feed numeric text input without formatting.
+   *
+   * Returns -1 if the given amount can not be converted.
+   *
+   * @param amount BtcAmount
+   * @return BigDecimal value of the BtcAmount
+   */
   def rawAmountInUnit(amount: BtcAmount, unit: CoinUnit): BigDecimal = Try(convertAmountToGUIUnit(amount, unit) match {
     case a: BtcAmountGUILossless => BigDecimal(a.amount_msat) / a.unit.factorToMsat
     case a => throw new IllegalArgumentException(s"unhandled unit $a")
@@ -245,5 +245,5 @@ object CoinUtils extends Logging {
       -1
   }
 
-  def rawAmountInUnit(msat: MilliSatoshi, unit: CoinUnit): BigDecimal = BigDecimal(msat.amount) / unit.factorToMsat
+  def rawAmountInUnit(msat: MilliSatoshi, unit: CoinUnit): BigDecimal = BigDecimal(msat.toLong) / unit.factorToMsat
 }

eclair-core/src/main/scala/fr/acinq/eclair/DBCompatChecker.scala

@@ -39,7 +39,7 @@ object DBCompatChecker extends Logging {
     * @param nodeParams
     */
   def checkNetworkDBCompatibility(nodeParams: NodeParams): Unit =
-    Try(nodeParams.db.network.listChannels(), nodeParams.db.network.listNodes(), nodeParams.db.network.listChannelUpdates()) match {
+    Try(nodeParams.db.network.listChannels(), nodeParams.db.network.listNodes()) match {
       case Success(_) => {}
       case Failure(_) => throw IncompatibleNetworkDBException
     }

eclair-core/src/main/scala/fr/acinq/eclair/Eclair.scala

@@ -23,20 +23,20 @@ import akka.pattern._
 import akka.util.Timeout
 import fr.acinq.bitcoin.Crypto.PublicKey
 import fr.acinq.bitcoin.{ByteVector32, Satoshi}
+import fr.acinq.eclair.TimestampQueryFilters._
 import fr.acinq.eclair.channel.Register.{Forward, ForwardShortId}
 import fr.acinq.eclair.channel._
 import fr.acinq.eclair.db.{IncomingPayment, NetworkFee, OutgoingPayment, Stats}
 import fr.acinq.eclair.io.Peer.{GetPeerInfo, PeerInfo}
 import fr.acinq.eclair.io.{NodeURI, Peer}
 import fr.acinq.eclair.payment.PaymentLifecycle._
+import fr.acinq.eclair.payment._
 import fr.acinq.eclair.router.{ChannelDesc, RouteRequest, RouteResponse, Router}
+import fr.acinq.eclair.wire.{ChannelAnnouncement, ChannelUpdate, NodeAddress, NodeAnnouncement}
 import scodec.bits.ByteVector
 
-import scala.concurrent.Future
 import scala.concurrent.duration._
-import fr.acinq.eclair.payment.{GetUsableBalances, PaymentReceived, PaymentRelayed, PaymentRequest, PaymentSent, UsableBalances}
-import fr.acinq.eclair.wire.{ChannelAnnouncement, ChannelUpdate, NodeAddress, NodeAnnouncement}
-import TimestampQueryFilters._
+import scala.concurrent.{ExecutionContext, Future}
 
 case class GetInfoResponse(nodeId: PublicKey, alias: String, chainHash: ByteVector32, blockHeight: Int, publicAddresses: Seq[NodeAddress])
 
@@ -78,13 +78,13 @@ trait Eclair {
 
   def receivedInfo(paymentHash: ByteVector32)(implicit timeout: Timeout): Future[Option[IncomingPayment]]
 
-  def send(recipientNodeId: PublicKey, amount: MilliSatoshi, paymentHash: ByteVector32, assistedRoutes: Seq[Seq[PaymentRequest.ExtraHop]] = Seq.empty, minFinalCltvExpiry_opt: Option[Long] = None, maxAttempts_opt: Option[Int] = None, feeThresholdSat_opt: Option[Satoshi] = None, maxFeePct_opt: Option[Double] = None)(implicit timeout: Timeout): Future[UUID]
+  def send(recipientNodeId: PublicKey, amount: MilliSatoshi, paymentHash: ByteVector32, assistedRoutes: Seq[Seq[PaymentRequest.ExtraHop]] = Seq.empty, minFinalCltvExpiryDelta_opt: Option[CltvExpiryDelta] = None, maxAttempts_opt: Option[Int] = None, feeThresholdSat_opt: Option[Satoshi] = None, maxFeePct_opt: Option[Double] = None)(implicit timeout: Timeout): Future[UUID]
 
   def sentInfo(id: Either[UUID, ByteVector32])(implicit timeout: Timeout): Future[Seq[OutgoingPayment]]
 
   def findRoute(targetNodeId: PublicKey, amount: MilliSatoshi, assistedRoutes: Seq[Seq[PaymentRequest.ExtraHop]] = Seq.empty)(implicit timeout: Timeout): Future[RouteResponse]
 
-  def sendToRoute(route: Seq[PublicKey], amount: MilliSatoshi, paymentHash: ByteVector32, finalCltvExpiry: Long)(implicit timeout: Timeout): Future[UUID]
+  def sendToRoute(route: Seq[PublicKey], amount: MilliSatoshi, paymentHash: ByteVector32, finalCltvExpiryDelta: CltvExpiryDelta)(implicit timeout: Timeout): Future[UUID]
 
   def audit(from_opt: Option[Long], to_opt: Option[Long])(implicit timeout: Timeout): Future[AuditResponse]
 
@@ -111,7 +111,7 @@ trait Eclair {
 
 class EclairImpl(appKit: Kit) extends Eclair {
 
-  implicit val ec = appKit.system.dispatcher
+  implicit val ec: ExecutionContext = appKit.system.dispatcher
 
   override def connect(target: Either[NodeURI, PublicKey])(implicit timeout: Timeout): Future[String] = target match {
     case Left(uri) => (appKit.switchboard ? Peer.Connect(uri)).mapTo[String]
@@ -128,7 +128,7 @@ class EclairImpl(appKit: Kit) extends Eclair {
     (appKit.switchboard ? Peer.OpenChannel(
       remoteNodeId = nodeId,
       fundingSatoshis = fundingAmount,
-      pushMsat = pushAmount_opt.getOrElse(MilliSatoshi(0)),
+      pushMsat = pushAmount_opt.getOrElse(0 msat),
       fundingTxFeeratePerKw_opt = fundingFeerateSatByte_opt.map(feerateByte2Kw),
       channelFlags = flags_opt.map(_.toByte),
       timeout_opt = Some(openTimeout))).mapTo[String]
@@ -186,11 +186,11 @@ class EclairImpl(appKit: Kit) extends Eclair {
     (appKit.router ? RouteRequest(appKit.nodeParams.nodeId, targetNodeId, amount, assistedRoutes)).mapTo[RouteResponse]
   }
 
-  override def sendToRoute(route: Seq[PublicKey], amount: MilliSatoshi, paymentHash: ByteVector32, finalCltvExpiry: Long)(implicit timeout: Timeout): Future[UUID] = {
-    (appKit.paymentInitiator ? SendPaymentToRoute(amount, paymentHash, route, finalCltvExpiry)).mapTo[UUID]
+  override def sendToRoute(route: Seq[PublicKey], amount: MilliSatoshi, paymentHash: ByteVector32, finalCltvExpiryDelta: CltvExpiryDelta)(implicit timeout: Timeout): Future[UUID] = {
+    (appKit.paymentInitiator ? SendPaymentToRoute(amount, paymentHash, route, finalCltvExpiryDelta)).mapTo[UUID]
   }
 
-  override def send(recipientNodeId: PublicKey, amount: MilliSatoshi, paymentHash: ByteVector32, assistedRoutes: Seq[Seq[PaymentRequest.ExtraHop]] = Seq.empty, minFinalCltvExpiry_opt: Option[Long], maxAttempts_opt: Option[Int], feeThreshold_opt: Option[Satoshi], maxFeePct_opt: Option[Double])(implicit timeout: Timeout): Future[UUID] = {
+  override def send(recipientNodeId: PublicKey, amount: MilliSatoshi, paymentHash: ByteVector32, assistedRoutes: Seq[Seq[PaymentRequest.ExtraHop]] = Seq.empty, minFinalCltvExpiryDelta_opt: Option[CltvExpiryDelta], maxAttempts_opt: Option[Int], feeThreshold_opt: Option[Satoshi], maxFeePct_opt: Option[Double])(implicit timeout: Timeout): Future[UUID] = {
     val maxAttempts = maxAttempts_opt.getOrElse(appKit.nodeParams.maxPaymentAttempts)
 
     val defaultRouteParams = Router.getDefaultRouteParams(appKit.nodeParams.routerConf)
@@ -199,8 +199,8 @@ class EclairImpl(appKit: Kit) extends Eclair {
       maxFeeBase = feeThreshold_opt.map(_.toMilliSatoshi).getOrElse(defaultRouteParams.maxFeeBase)
     )
 
-    val sendPayment = minFinalCltvExpiry_opt match {
-      case Some(minCltv) => SendPayment(amount, paymentHash, recipientNodeId, assistedRoutes, finalCltvExpiry = minCltv, maxAttempts = maxAttempts, routeParams = Some(routeParams))
+    val sendPayment = minFinalCltvExpiryDelta_opt match {
+      case Some(minCltv) => SendPayment(amount, paymentHash, recipientNodeId, assistedRoutes, finalCltvExpiryDelta = minCltv, maxAttempts = maxAttempts, routeParams = Some(routeParams))
       case None => SendPayment(amount, paymentHash, recipientNodeId, assistedRoutes, maxAttempts = maxAttempts, routeParams = Some(routeParams))
     }
     (appKit.paymentInitiator ? sendPayment).mapTo[UUID]
@@ -255,8 +255,6 @@ class EclairImpl(appKit: Kit) extends Eclair {
     * Sends a request to a channel and expects a response
     *
     * @param channelIdentifier either a shortChannelId (BOLT encoded) or a channelId (32-byte hex encoded)
-    * @param request
-    * @return
     */
   def sendToChannel(channelIdentifier: Either[ByteVector32, ShortChannelId], request: Any)(implicit timeout: Timeout): Future[Any] = channelIdentifier match {
     case Left(channelId) => appKit.register ? Forward(channelId, request)