Router computes network stats

eclair-core/src/main/resources/reference.conf

@@ -116,6 +116,7 @@ eclair {
     randomize-route-selection = true // when computing a route for a payment we randomize the final selection
     channel-exclude-duration = 60 seconds // when a temporary channel failure is returned, we exclude the channel from our payment routes for this duration
     broadcast-interval = 60 seconds // see BOLT #7
+    network-stats-interval = 6 hours // frequency at which we refresh global network statistics (expensive operation)
     init-timeout = 5 minutes
 
     sync {

eclair-core/src/main/scala/fr/acinq/eclair/NodeParams.scala

@@ -243,6 +243,7 @@ object NodeParams {
       routerConf = RouterConf(
         channelExcludeDuration = FiniteDuration(config.getDuration("router.channel-exclude-duration").getSeconds, TimeUnit.SECONDS),
         routerBroadcastInterval = FiniteDuration(config.getDuration("router.broadcast-interval").getSeconds, TimeUnit.SECONDS),
+        networkStatsRefreshInterval = FiniteDuration(config.getDuration("router.network-stats-interval").getSeconds, TimeUnit.SECONDS),
         randomizeRouteSelection = config.getBoolean("router.randomize-route-selection"),
         requestNodeAnnouncements = config.getBoolean("router.sync.request-node-announcements"),
         encodingType = routerSyncEncodingType,

eclair-core/src/main/scala/fr/acinq/eclair/router/Graph.scala

@@ -29,10 +29,21 @@ import scala.collection.mutable
 object Graph {
 
   // @formatter:off
-  // A compound weight for an edge, weight is obtained with (cost X factor),'cost' contains the actual amount+fees in millisatoshi, 'cltvCumulative' the total CLTV necessary to reach this edge
+  /**
+   * The cumulative weight of a set of edges (path in the graph).
+   *
+   * @param cost   amount to send to the recipient + each edge's fees
+   * @param length number of edges in the path
+   * @param cltv   sum of each edge's cltv
+   * @param weight cost multiplied by a factor based on heuristics (see [[WeightRatios]]).
+   */
   case class RichWeight(cost: MilliSatoshi, length: Int, cltv: CltvExpiryDelta, weight: Double) extends Ordered[RichWeight] {
-    override def compare(that:  RichWeight): Int = this.weight.compareTo(that.weight)
+    override def compare(that: RichWeight): Int = this.weight.compareTo(that.weight)
   }
+  /**
+   * We use heuristics to calculate the weight of an edge based on channel age, cltv delta and capacity.
+   * We favor older channels, with bigger capacity and small cltv delta.
+   */
   case class WeightRatios(cltvDeltaFactor: Double, ageFactor: Double, capacityFactor: Double) {
     require(0 < cltvDeltaFactor + ageFactor + capacityFactor && cltvDeltaFactor + ageFactor + capacityFactor <= 1, "The sum of heuristics ratios must be between 0 and 1 (included)")
   }
@@ -57,18 +68,17 @@ object Graph {
   }
 
   /**
-   * Yen's algorithm to find the k-shortest (loopless) paths in a graph, uses dijkstra as search algo. Is guaranteed to terminate finding
+   * Yen's algorithm to find the k-shortest (loop-less) paths in a graph, uses dijkstra as search algo. Is guaranteed to terminate finding
    * at most @pathsToFind paths sorted by cost (the cheapest is in position 0).
    *
-   * @param graph
-   * @param sourceNode
-   * @param targetNode
-   * @param amount
-   * @param pathsToFind
+   * @param graph              graph representing the whole network
+   * @param sourceNode         sender node (payer)
+   * @param targetNode         target node (final recipient)
+   * @param amount             amount to send to the last node
+   * @param pathsToFind        number of distinct paths to be returned
    * @param wr                 an object containing the ratios used to 'weight' edges when searching for the shortest path
    * @param currentBlockHeight the height of the chain tip (latest block)
    * @param boundaries         a predicate function that can be used to impose limits on the outcome of the search
-   * @return
    */
   def yenKshortestPaths(graph: DirectedGraph,
                         sourceNode: PublicKey,
@@ -175,7 +185,6 @@ object Graph {
    * @param boundaries         a predicate function that can be used to impose limits on the outcome of the search
    * @return
    */
-
   def dijkstraShortestPath(g: DirectedGraph,
                            sourceNode: PublicKey,
                            targetNode: PublicKey,
@@ -237,7 +246,6 @@ object Graph {
             boundaries(newMinimumKnownWeight) && // check if this neighbor edge would break off the 'boundaries'
             !ignoredEdges.contains(edge.desc) && !ignoredVertices.contains(neighbor)
           ) {
-
             // we call containsKey first because "getOrDefault" is not available in JDK7
             val neighborCost = weight.containsKey(neighbor) match {
               case false => RichWeight(MilliSatoshi(Long.MaxValue), Int.MaxValue, CltvExpiryDelta(Int.MaxValue), Double.MaxValue)
@@ -332,7 +340,7 @@ object Graph {
   // Calculates the total cost of a path (amount + fees), direct channels with the source will have a cost of 0 (pay no fees)
   def pathWeight(path: Seq[GraphEdge], amountMsat: MilliSatoshi, isPartial: Boolean, currentBlockHeight: Long, wr: Option[WeightRatios]): RichWeight = {
     path.drop(if (isPartial) 0 else 1).foldRight(RichWeight(amountMsat, 0, CltvExpiryDelta(0), 0)) { (edge, prev) =>
-      edgeWeight(edge, prev, false, currentBlockHeight, wr)
+      edgeWeight(edge, prev, isNeighborTarget = false, currentBlockHeight, wr)
     }
   }
 
@@ -353,11 +361,6 @@ object Graph {
     /**
      * Normalize the given value between (0, 1). If the @param value is outside the min/max window we flatten it to something very close to the
      * extremes but always bigger than zero so it's guaranteed to never return zero
-     *
-     * @param value
-     * @param min
-     * @param max
-     * @return
      */
     def normalize(value: Double, min: Double, max: Double) = {
       if (value <= min) 0.00001D
@@ -367,7 +370,7 @@ object Graph {
   }
 
   /**
-   * A graph data structure that uses the adjacency lists, stores the incoming edges of the neighbors
+   * A graph data structure that uses an adjacency list, stores the incoming edges of the neighbors
    */
   object GraphStructure {
 
@@ -388,13 +391,12 @@ object Graph {
       }
 
       /**
-       * Adds and edge to the graph, if one of the two vertices is not found, it will be created.
+       * Adds an edge to the graph. If one of the two vertices is not found it will be created.
        *
        * @param edge the edge that is going to be added to the graph
        * @return a new graph containing this edge
        */
       def addEdge(edge: GraphEdge): DirectedGraph = {
-
         val vertexIn = edge.desc.a
         val vertexOut = edge.desc.b
 
@@ -412,7 +414,7 @@ object Graph {
        * NB: this operation does NOT remove any vertex
        *
        * @param desc the channel description associated to the edge that will be removed
-       * @return
+       * @return a new graph without this edge
        */
       def removeEdge(desc: ChannelDesc): DirectedGraph = {
         containsEdge(desc) match {
@@ -426,7 +428,6 @@ object Graph {
       }
 
       /**
-       * @param edge
        * @return For edges to be considered equal they must have the same in/out vertices AND same shortChannelId
        */
       def getEdge(edge: GraphEdge): Option[GraphEdge] = getEdge(edge.desc)
@@ -440,7 +441,7 @@ object Graph {
       /**
        * @param keyA the key associated with the starting vertex
        * @param keyB the key associated with the ending vertex
-       * @return all the edges going from keyA --> keyB (there might be more than one if it refers to different shortChannelId)
+       * @return all the edges going from keyA --> keyB (there might be more than one if there are multiple channels)
        */
       def getEdgesBetween(keyA: PublicKey, keyB: PublicKey): Seq[GraphEdge] = {
         vertices.get(keyB) match {
@@ -450,30 +451,22 @@ object Graph {
       }
 
       /**
-       * The the incoming edges for vertex @param keyB
-       *
-       * @param keyB
-       * @return
+       * @param keyB the key associated with the target vertex
+       * @return all edges incoming to that vertex
        */
       def getIncomingEdgesOf(keyB: PublicKey): Seq[GraphEdge] = {
         vertices.getOrElse(keyB, List.empty)
       }
 
       /**
-       * Removes a vertex and all it's associated edges (both incoming and outgoing)
-       *
-       * @param key
-       * @return
+       * Removes a vertex and all its associated edges (both incoming and outgoing)
        */
       def removeVertex(key: PublicKey): DirectedGraph = {
         DirectedGraph(removeEdges(getIncomingEdgesOf(key).map(_.desc)).vertices - key)
       }
 
       /**
        * Adds a new vertex to the graph, starting with no edges
-       *
-       * @param key
-       * @return
        */
       def addVertex(key: PublicKey): DirectedGraph = {
         vertices.get(key) match {
@@ -485,8 +478,7 @@ object Graph {
       /**
        * Note this operation will traverse all edges in the graph (expensive)
        *
-       * @param key
-       * @return a list of the outgoing edges of vertex @param key, if the edge doesn't exists an empty list is returned
+       * @return a list of the outgoing edges of the given vertex. If the vertex doesn't exists an empty list is returned.
        */
       def edgesOf(key: PublicKey): Seq[GraphEdge] = {
         edgeSet().filter(_.desc.a == key).toSeq
@@ -503,13 +495,11 @@ object Graph {
       def edgeSet(): Iterable[GraphEdge] = vertices.values.flatten
 
       /**
-       * @param key
        * @return true if this graph contain a vertex with this key, false otherwise
        */
       def containsVertex(key: PublicKey): Boolean = vertices.contains(key)
 
       /**
-       * @param desc
        * @return true if this edge desc is in the graph. For edges to be considered equal they must have the same in/out vertices AND same shortChannelId
        */
       def containsEdge(desc: ChannelDesc): Boolean = {
@@ -528,20 +518,25 @@ object Graph {
 
     object DirectedGraph {
 
-      // convenience constructors
+      // @formatter:off
       def apply(): DirectedGraph = new DirectedGraph(Map())
-
       def apply(key: PublicKey): DirectedGraph = new DirectedGraph(Map(key -> List.empty))
-
       def apply(edge: GraphEdge): DirectedGraph = new DirectedGraph(Map()).addEdge(edge.desc, edge.update)
-
       def apply(edges: Seq[GraphEdge]): DirectedGraph = {
         DirectedGraph().addEdges(edges.map(e => (e.desc, e.update)))
       }
+      // @formatter:on
 
-      // optimized constructor
+      /**
+       * This is the recommended way of creating the network graph.
+       * We don't include private channels: they would bloat the graph without providing any value (if they are private
+       * they likely don't want to be involved in routing other people's payments).
+       * The only private channels we know are ours: we should check them to see if our destination can be reached in a
+       * single hop via a private channel before using the public network graph.
+       *
+       * @param channels map of all known public channels in the network.
+       */
       def makeGraph(channels: SortedMap[ShortChannelId, PublicChannel]): DirectedGraph = {
-
         // initialize the map with the appropriate size to avoid resizing during the graph initialization
         val mutableMap = new {} with mutable.HashMap[PublicKey, List[GraphEdge]] {
           override def initialSize: Int = channels.size + 1
@@ -560,7 +555,7 @@ object Graph {
           }
         }
 
-        def addDescToMap(desc: ChannelDesc, u: ChannelUpdate) = {
+        def addDescToMap(desc: ChannelDesc, u: ChannelUpdate): Unit = {
           mutableMap.put(desc.b, GraphEdge(desc, u) +: mutableMap.getOrElse(desc.b, List.empty[GraphEdge]))
           mutableMap.get(desc.a) match {
             case None => mutableMap += desc.a -> List.empty[GraphEdge]

eclair-core/src/main/scala/fr/acinq/eclair/router/NetworkStats.scala

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2019 ACINQ SAS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fr.acinq.eclair.router
+
+import com.google.common.math.Quantiles.percentiles
+import fr.acinq.bitcoin.Satoshi
+import fr.acinq.eclair.wire.ChannelUpdate
+import fr.acinq.eclair.{CltvExpiryDelta, MilliSatoshi}
+
+import scala.collection.JavaConverters._
+
+/**
+ * Created by t-bast on 30/08/2019.
+ */
+
+case class Stats[T](median: T, percentile5: T, percentile10: T, percentile25: T, percentile75: T, percentile90: T, percentile95: T)
+
+object Stats {
+  def apply[T](values: Seq[Long], fromDouble: Double => T): Stats[T] = {
+    require(values.nonEmpty, "can't compute stats on empty values")
+    val stats = percentiles().indexes(5, 10, 25, 50, 75, 90, 95).compute(values.map(java.lang.Long.valueOf).asJavaCollection)
+    Stats(fromDouble(stats.get(50)), fromDouble(stats.get(5)), fromDouble(stats.get(10)), fromDouble(stats.get(25)), fromDouble(stats.get(75)), fromDouble(stats.get(90)), fromDouble(stats.get(95)))
+  }
+}
+
+case class NetworkStats(channels: Int, nodes: Int, capacity: Stats[Satoshi], cltvExpiryDelta: Stats[CltvExpiryDelta], feeBase: Stats[MilliSatoshi], feeProportional: Stats[Long])
+
+object NetworkStats {
+  /**
+   * Computes various network statistics (expensive).
+   * Network statistics won't change noticeably very quickly, so this should not be re-computed too often.
+   */
+  def apply(publicChannels: Seq[PublicChannel]): Option[NetworkStats] = {
+    // We need at least one channel update to be able to compute stats.
+    if (publicChannels.isEmpty || publicChannels.flatMap(pc => getChannelUpdateField(pc, _ => true)).isEmpty) {
+      None
+    } else {
+      val capacityStats = Stats(publicChannels.map(_.capacity.toLong), d => Satoshi(d.toLong))
+      val cltvStats = Stats(publicChannels.flatMap(pc => getChannelUpdateField(pc, u => u.cltvExpiryDelta.toInt.toLong)), d => CltvExpiryDelta(d.toInt))
+      val feeBaseStats = Stats(publicChannels.flatMap(pc => getChannelUpdateField(pc, u => u.feeBaseMsat.toLong)), d => MilliSatoshi(d.toLong))
+      val feeProportionalStats = Stats(publicChannels.flatMap(pc => getChannelUpdateField(pc, u => u.feeProportionalMillionths)), d => d.toLong)
+      val nodes = publicChannels.flatMap(pc => pc.ann.nodeId1 :: pc.ann.nodeId2 :: Nil).toSet.size
+      Some(NetworkStats(publicChannels.size, nodes, capacityStats, cltvStats, feeBaseStats, feeProportionalStats))
+    }
+  }
+
+  private def getChannelUpdateField[T](pc: PublicChannel, f: ChannelUpdate => T): Seq[T] =
+    pc.update_1_opt.map(u => f(u) :: Nil).getOrElse(Nil) ++ pc.update_2_opt.map(u => f(u) :: Nil).getOrElse(Nil)
+
+}