Add log entry, rebase on master

eclair-core/src/main/resources/reference.conf

@@ -68,6 +68,11 @@ eclair {
   max-to-local-delay-blocks = 2016 // maximum number of blocks that we are ready to accept for our own delayed outputs (2016 ~ 2 weeks)
   mindepth-blocks = 3
   expiry-delta-blocks = 144
+  // When we receive the pre-image for an HTLC and want to fulfill it but the upstream peer stops responding, we want to
+  // avoid letting its HTLC-timeout transaction become enforceable on-chain (otherwise there is a race condition between
+  // our HTLC-success and their HTLC-timeout).
+  // We will close the channel when the HTLC-timeout will happen in less than this number.
+  fulfill-safety-before-timeout-blocks = 6
 
   fee-base-msat = 1000
   fee-proportional-millionths = 100 // fee charged per transferred satoshi in millionths of a satoshi (100 = 0.01%)

eclair-core/src/main/scala/fr/acinq/eclair/Features.scala

@@ -21,7 +21,6 @@ import java.util.BitSet
 
 import scodec.bits.ByteVector
 
-
 /**
   * Created by PM on 13/02/2017.
   */
@@ -36,12 +35,13 @@ object Features {
   val CHANNEL_RANGE_QUERIES_BIT_MANDATORY = 6
   val CHANNEL_RANGE_QUERIES_BIT_OPTIONAL = 7
 
+  val VARIABLE_LENGTH_ONION_MANDATORY = 8
+  val VARIABLE_LENGTH_ONION_OPTIONAL = 9
 
   def hasFeature(features: BitSet, bit: Int): Boolean = features.get(bit)
 
   def hasFeature(features: ByteVector, bit: Int): Boolean = hasFeature(BitSet.valueOf(features.reverse.toArray), bit)
 
-
   /**
     * Check that the features that we understand are correctly specified, and that there are no mandatory features that
     * we don't understand (even bits)
@@ -51,7 +51,8 @@ object Features {
     for (i <- 0 until bitset.length() by 2) {
       if (bitset.get(i) && !supportedMandatoryFeatures.contains(i)) return false
     }
-    return true
+
+    true
   }
 
   /**

eclair-core/src/main/scala/fr/acinq/eclair/NodeParams.scala

@@ -50,6 +50,7 @@ case class NodeParams(keyManager: KeyManager,
                       maxHtlcValueInFlightMsat: UInt64,
                       maxAcceptedHtlcs: Int,
                       expiryDeltaBlocks: Int,
+                      fulfillSafetyBeforeTimeoutBlocks: Int,
                       htlcMinimumMsat: Int,
                       toRemoteDelayBlocks: Int,
                       maxToLocalDelayBlocks: Int,
@@ -149,14 +150,18 @@ object NodeParams {
     val offeredCLTV = config.getInt("to-remote-delay-blocks")
     require(maxToLocalCLTV <= Channel.MAX_TO_SELF_DELAY && offeredCLTV <= Channel.MAX_TO_SELF_DELAY, s"CLTV delay values too high, max is ${Channel.MAX_TO_SELF_DELAY}")
 
+    val expiryDeltaBlocks = config.getInt("expiry-delta-blocks")
+    val fulfillSafetyBeforeTimeoutBlocks = config.getInt("fulfill-safety-before-timeout-blocks")
+    require(fulfillSafetyBeforeTimeoutBlocks < expiryDeltaBlocks, "fulfill-safety-before-timeout-blocks must be smaller than expiry-delta-blocks")
+
     val nodeAlias = config.getString("node-alias")
     require(nodeAlias.getBytes("UTF-8").length <= 32, "invalid alias, too long (max allowed 32 bytes)")
 
     val overrideFeatures: Map[PublicKey, (ByteVector, ByteVector)] = config.getConfigList("override-features").map { e =>
       val p = PublicKey(ByteVector.fromValidHex(e.getString("nodeid")))
       val gf = ByteVector.fromValidHex(e.getString("global-features"))
       val lf = ByteVector.fromValidHex(e.getString("local-features"))
-      (p -> (gf, lf))
+      p -> (gf, lf)
     }.toMap
 
     val socksProxy_opt = if (config.getBoolean("socks5.enabled")) {
@@ -187,7 +192,8 @@ object NodeParams {
       dustLimitSatoshis = dustLimitSatoshis,
       maxHtlcValueInFlightMsat = UInt64(config.getLong("max-htlc-value-in-flight-msat")),
       maxAcceptedHtlcs = maxAcceptedHtlcs,
-      expiryDeltaBlocks = config.getInt("expiry-delta-blocks"),
+      expiryDeltaBlocks = expiryDeltaBlocks,
+      fulfillSafetyBeforeTimeoutBlocks = fulfillSafetyBeforeTimeoutBlocks,
       htlcMinimumMsat = config.getInt("htlc-minimum-msat"),
       toRemoteDelayBlocks = config.getInt("to-remote-delay-blocks"),
       maxToLocalDelayBlocks = config.getInt("max-to-local-delay-blocks"),

eclair-core/src/main/scala/fr/acinq/eclair/channel/ChannelExceptions.scala

@@ -18,7 +18,7 @@ package fr.acinq.eclair.channel
 
 import fr.acinq.bitcoin.Crypto.PrivateKey
 import fr.acinq.bitcoin.{ByteVector32, Transaction}
-import fr.acinq.eclair.{ShortChannelId, UInt64}
+import fr.acinq.eclair.UInt64
 import fr.acinq.eclair.payment.Origin
 import fr.acinq.eclair.wire.{ChannelUpdate, UpdateAddHtlc}
 
@@ -27,6 +27,7 @@ import fr.acinq.eclair.wire.{ChannelUpdate, UpdateAddHtlc}
   */
 
 class ChannelException(val channelId: ByteVector32, message: String) extends RuntimeException(message)
+
 // @formatter:off
 case class DebugTriggeredException             (override val channelId: ByteVector32) extends ChannelException(channelId, "debug-mode triggered failure")
 case class InvalidChainHash                    (override val channelId: ByteVector32, local: ByteVector32, remote: ByteVector32) extends ChannelException(channelId, s"invalid chainHash (local=$local remote=$remote)")
@@ -49,6 +50,7 @@ case class InvalidFinalScript                  (override val channelId: ByteVect
 case class FundingTxTimedout                   (override val channelId: ByteVector32) extends ChannelException(channelId, "funding tx timed out")
 case class FundingTxSpent                      (override val channelId: ByteVector32, spendingTx: Transaction) extends ChannelException(channelId, s"funding tx has been spent by txid=${spendingTx.txid}")
 case class HtlcTimedout                        (override val channelId: ByteVector32, htlcs: Set[UpdateAddHtlc]) extends ChannelException(channelId, s"one or more htlcs timed out: ids=${htlcs.take(10).map(_.id).mkString}") // we only display the first 10 ids
+case class HtlcWillTimeoutUpstream             (override val channelId: ByteVector32, htlcs: Set[UpdateAddHtlc]) extends ChannelException(channelId, s"one or more htlcs that should be fulfilled are close to timing out upstream: ids=${htlcs.take(10).map(_.id).mkString}") // we only display the first 10 ids
 case class HtlcOverridenByLocalCommit          (override val channelId: ByteVector32) extends ChannelException(channelId, "htlc was overriden by local commit")
 case class FeerateTooSmall                     (override val channelId: ByteVector32, remoteFeeratePerKw: Long) extends ChannelException(channelId, s"remote fee rate is too small: remoteFeeratePerKw=$remoteFeeratePerKw")
 case class FeerateTooDifferent                 (override val channelId: ByteVector32, localFeeratePerKw: Long, remoteFeeratePerKw: Long) extends ChannelException(channelId, s"local/remote feerates are too different: remoteFeeratePerKw=$remoteFeeratePerKw localFeeratePerKw=$localFeeratePerKw")

eclair-core/src/main/scala/fr/acinq/eclair/channel/ChannelTypes.scala

@@ -21,10 +21,9 @@ import java.util.UUID
 import akka.actor.ActorRef
 import fr.acinq.bitcoin.Crypto.PublicKey
 import fr.acinq.bitcoin.{ByteVector32, DeterministicWallet, OutPoint, Satoshi, Transaction}
-import fr.acinq.eclair.crypto.Sphinx
 import fr.acinq.eclair.transactions.CommitmentSpec
 import fr.acinq.eclair.transactions.Transactions.CommitTx
-import fr.acinq.eclair.wire.{AcceptChannel, ChannelAnnouncement, ChannelReestablish, ChannelUpdate, ClosingSigned, FailureMessage, FundingCreated, FundingLocked, FundingSigned, Init, OpenChannel, Shutdown, UpdateAddHtlc}
+import fr.acinq.eclair.wire.{AcceptChannel, ChannelAnnouncement, ChannelReestablish, ChannelUpdate, ClosingSigned, FailureMessage, FundingCreated, FundingLocked, FundingSigned, Init, OnionRoutingPacket, OpenChannel, Shutdown, UpdateAddHtlc}
 import fr.acinq.eclair.{ShortChannelId, UInt64}
 import scodec.bits.{BitVector, ByteVector}
 
@@ -107,7 +106,7 @@ case class BITCOIN_PARENT_TX_CONFIRMED(childTx: Transaction) extends BitcoinEven
  */
 
 sealed trait Command
-final case class CMD_ADD_HTLC(amountMsat: Long, paymentHash: ByteVector32, cltvExpiry: Long, onion: ByteVector = Sphinx.LAST_PACKET.serialize, upstream: Either[UUID, UpdateAddHtlc], commit: Boolean = false, previousFailures: Seq[AddHtlcFailed] = Seq.empty) extends Command
+final case class CMD_ADD_HTLC(amountMsat: Long, paymentHash: ByteVector32, cltvExpiry: Long, onion: OnionRoutingPacket, upstream: Either[UUID, UpdateAddHtlc], commit: Boolean = false, previousFailures: Seq[AddHtlcFailed] = Seq.empty) extends Command
 final case class CMD_FULFILL_HTLC(id: Long, r: ByteVector32, commit: Boolean = false) extends Command
 final case class CMD_FAIL_HTLC(id: Long, reason: Either[ByteVector, FailureMessage], commit: Boolean = false) extends Command
 final case class CMD_FAIL_MALFORMED_HTLC(id: Long, onionHash: ByteVector32, failureCode: Int, commit: Boolean = false) extends Command

eclair-core/src/main/scala/fr/acinq/eclair/channel/Commitments.scala

@@ -26,8 +26,6 @@ import fr.acinq.eclair.transactions._
 import fr.acinq.eclair.wire._
 import fr.acinq.eclair.{Globals, UInt64}
 
-import scala.util.{Failure, Success}
-
 // @formatter:off
 case class LocalChanges(proposed: List[UpdateMessage], signed: List[UpdateMessage], acked: List[UpdateMessage]) {
   def all: List[UpdateMessage] = proposed ++ signed ++ acked
@@ -62,11 +60,23 @@ case class Commitments(channelVersion: ChannelVersion,
 
   def hasNoPendingHtlcs: Boolean = localCommit.spec.htlcs.isEmpty && remoteCommit.spec.htlcs.isEmpty && remoteNextCommitInfo.isRight
 
-  def timedoutOutgoingHtlcs(blockheight: Long): Set[UpdateAddHtlc] =
+  def timedOutOutgoingHtlcs(blockheight: Long): Set[UpdateAddHtlc] =
     (localCommit.spec.htlcs.filter(htlc => htlc.direction == OUT && blockheight >= htlc.add.cltvExpiry) ++
       remoteCommit.spec.htlcs.filter(htlc => htlc.direction == IN && blockheight >= htlc.add.cltvExpiry) ++
       remoteNextCommitInfo.left.toOption.map(_.nextRemoteCommit.spec.htlcs.filter(htlc => htlc.direction == IN && blockheight >= htlc.add.cltvExpiry)).getOrElse(Set.empty[DirectedHtlc])).map(_.add)
 
+  /**
+    * HTLCs that are close to timing out upstream are potentially dangerous. If we received the pre-image for those
+    * HTLCs, we need to get a remote signed updated commitment that removes this HTLC.
+    * Otherwise when we get close to the upstream timeout, we risk an on-chain race condition between their HTLC timeout
+    * and our HTLC success in case of a force-close.
+    */
+  def almostTimedOutIncomingHtlcs(blockheight: Long, fulfillSafety: Int): Set[UpdateAddHtlc] = {
+    localCommit.spec.htlcs.collect {
+      case htlc if htlc.direction == IN && blockheight >= htlc.add.cltvExpiry - fulfillSafety => htlc.add
+    }
+  }
+
   def addLocalProposal(proposal: UpdateMessage): Commitments = Commitments.addLocalProposal(this, proposal)
 
   def addRemoteProposal(proposal: UpdateMessage): Commitments = Commitments.addRemoteProposal(this, proposal)
@@ -87,12 +97,13 @@ case class Commitments(channelVersion: ChannelVersion,
 }
 
 object Commitments {
+
   /**
-    * add a change to our proposed change list
+    * Add a change to our proposed change list.
     *
-    * @param commitments
-    * @param proposal
-    * @return an updated commitment instance
+    * @param commitments current commitments.
+    * @param proposal    proposed change to add.
+    * @return an updated commitment instance.
     */
   private def addLocalProposal(commitments: Commitments, proposal: UpdateMessage): Commitments =
     commitments.copy(localChanges = commitments.localChanges.copy(proposed = commitments.localChanges.proposed :+ proposal))
@@ -212,14 +223,14 @@ object Commitments {
         val fulfill = UpdateFulfillHtlc(commitments.channelId, cmd.id, cmd.r)
         val commitments1 = addLocalProposal(commitments, fulfill)
         (commitments1, fulfill)
-      case Some(htlc) => throw InvalidHtlcPreimage(commitments.channelId, cmd.id)
+      case Some(_) => throw InvalidHtlcPreimage(commitments.channelId, cmd.id)
       case None => throw UnknownHtlcId(commitments.channelId, cmd.id)
     }
 
   def receiveFulfill(commitments: Commitments, fulfill: UpdateFulfillHtlc): Either[Commitments, (Commitments, Origin, UpdateAddHtlc)] =
     getHtlcCrossSigned(commitments, OUT, fulfill.id) match {
       case Some(htlc) if htlc.paymentHash == sha256(fulfill.paymentPreimage) => Right((addRemoteProposal(commitments, fulfill), commitments.originChannels(fulfill.id), htlc))
-      case Some(htlc) => throw InvalidHtlcPreimage(commitments.channelId, fulfill.id)
+      case Some(_) => throw InvalidHtlcPreimage(commitments.channelId, fulfill.id)
       case None => throw UnknownHtlcId(commitments.channelId, fulfill.id)
     }
 
@@ -235,16 +246,16 @@ object Commitments {
         throw UnknownHtlcId(commitments.channelId, cmd.id)
       case Some(htlc) =>
         // we need the shared secret to build the error packet
-        Sphinx.parsePacket(nodeSecret, htlc.paymentHash, htlc.onionRoutingPacket).map(_.sharedSecret) match {
-          case Success(sharedSecret) =>
+        Sphinx.PaymentPacket.peel(nodeSecret, htlc.paymentHash, htlc.onionRoutingPacket) match {
+          case Right(Sphinx.DecryptedPacket(_, _, sharedSecret)) =>
             val reason = cmd.reason match {
-              case Left(forwarded) => Sphinx.forwardErrorPacket(forwarded, sharedSecret)
-              case Right(failure) => Sphinx.createErrorPacket(sharedSecret, failure)
+              case Left(forwarded) => Sphinx.FailurePacket.wrap(forwarded, sharedSecret)
+              case Right(failure) => Sphinx.FailurePacket.create(sharedSecret, failure)
             }
             val fail = UpdateFailHtlc(commitments.channelId, cmd.id, reason)
             val commitments1 = addLocalProposal(commitments, fail)
             (commitments1, fail)
-          case Failure(_) => throw new CannotExtractSharedSecret(commitments.channelId, htlc)
+          case Left(_) => throw CannotExtractSharedSecret(commitments.channelId, htlc)
         }
       case None => throw UnknownHtlcId(commitments.channelId, cmd.id)
     }
@@ -263,7 +274,7 @@ object Commitments {
       } =>
         // we have already sent a fail/fulfill for this htlc
         throw UnknownHtlcId(commitments.channelId, cmd.id)
-      case Some(htlc) =>
+      case Some(_) =>
         val fail = UpdateFailMalformedHtlc(commitments.channelId, cmd.id, cmd.onionHash, cmd.failureCode)
         val commitments1 = addLocalProposal(commitments, fail)
         (commitments1, fail)
@@ -348,9 +359,9 @@ object Commitments {
 
   def remoteHasUnsignedOutgoingHtlcs(commitments: Commitments): Boolean = commitments.remoteChanges.proposed.collectFirst { case u: UpdateAddHtlc => u }.isDefined
 
-  def localHasChanges(commitments: Commitments): Boolean = commitments.remoteChanges.acked.size > 0 || commitments.localChanges.proposed.size > 0
+  def localHasChanges(commitments: Commitments): Boolean = commitments.remoteChanges.acked.nonEmpty || commitments.localChanges.proposed.nonEmpty
 
-  def remoteHasChanges(commitments: Commitments): Boolean = commitments.localChanges.acked.size > 0 || commitments.remoteChanges.proposed.size > 0
+  def remoteHasChanges(commitments: Commitments): Boolean = commitments.localChanges.acked.nonEmpty || commitments.remoteChanges.proposed.nonEmpty
 
   def revocationPreimage(seed: ByteVector32, index: Long): ByteVector32 = ShaChain.shaChainFromSeed(seed, 0xFFFFFFFFFFFFFFFFL - index)
 
@@ -428,21 +439,21 @@ object Commitments {
 
     val sortedHtlcTxs: Seq[TransactionWithInputInfo] = (htlcTimeoutTxs ++ htlcSuccessTxs).sortBy(_.input.outPoint.index)
     if (commit.htlcSignatures.size != sortedHtlcTxs.size) {
-      throw new HtlcSigCountMismatch(commitments.channelId, sortedHtlcTxs.size, commit.htlcSignatures.size)
+      throw HtlcSigCountMismatch(commitments.channelId, sortedHtlcTxs.size, commit.htlcSignatures.size)
     }
     val htlcSigs = sortedHtlcTxs.map(keyManager.sign(_, keyManager.htlcPoint(localParams.channelKeyPath), localPerCommitmentPoint))
     val remoteHtlcPubkey = Generators.derivePubKey(remoteParams.htlcBasepoint, localPerCommitmentPoint)
     // combine the sigs to make signed txes
     val htlcTxsAndSigs = (sortedHtlcTxs, htlcSigs, commit.htlcSignatures).zipped.toList.collect {
       case (htlcTx: HtlcTimeoutTx, localSig, remoteSig) =>
         if (Transactions.checkSpendable(Transactions.addSigs(htlcTx, localSig, remoteSig)).isFailure) {
-          throw new InvalidHtlcSignature(commitments.channelId, htlcTx.tx)
+          throw InvalidHtlcSignature(commitments.channelId, htlcTx.tx)
         }
         HtlcTxAndSigs(htlcTx, localSig, remoteSig)
       case (htlcTx: HtlcSuccessTx, localSig, remoteSig) =>
         // we can't check that htlc-success tx are spendable because we need the payment preimage; thus we only check the remote sig
-        if (Transactions.checkSig(htlcTx, remoteSig, remoteHtlcPubkey) == false) {
-          throw new InvalidHtlcSignature(commitments.channelId, htlcTx.tx)
+        if (!Transactions.checkSig(htlcTx, remoteSig, remoteHtlcPubkey)) {
+          throw InvalidHtlcSignature(commitments.channelId, htlcTx.tx)
         }
         HtlcTxAndSigs(htlcTx, localSig, remoteSig)
     }

eclair-core/src/main/scala/fr/acinq/eclair/crypto/Mac.scala

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2019 ACINQ SAS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fr.acinq.eclair.crypto
+
+import fr.acinq.bitcoin.ByteVector32
+import org.spongycastle.crypto.digests.SHA256Digest
+import org.spongycastle.crypto.macs.HMac
+import org.spongycastle.crypto.params.KeyParameter
+import scodec.bits.ByteVector
+
+/**
+  * Created by t-bast on 04/07/19.
+  */
+
+/**
+  * Create and verify message authentication codes.
+  */
+trait Mac32 {
+
+  def mac(message: ByteVector): ByteVector32
+
+  def verify(mac: ByteVector32, message: ByteVector): Boolean
+
+}
+
+case class Hmac256(key: ByteVector) extends Mac32 {
+
+  override def mac(message: ByteVector): ByteVector32 = Mac32.hmac256(key, message)
+
+  override def verify(mac: ByteVector32, message: ByteVector): Boolean = this.mac(message) === mac
+
+}
+
+object Mac32 {
+
+  def hmac256(key: ByteVector, message: ByteVector): ByteVector32 = {
+    val mac = new HMac(new SHA256Digest())
+    mac.init(new KeyParameter(key.toArray))
+    mac.update(message.toArray, 0, message.length.toInt)
+    val output = new Array[Byte](32)
+    mac.doFinal(output, 0)
+    ByteVector32(ByteVector.view(output))
+  }
+
+}