Dev (#190)

- Made RPC polling optional
- Update configuration examples to be closer to real world usage
- Added no RPC polling config example
- Don't expose payment processing config extension data via API. Fixes #103
- Keep extra data in api result but sanitize it first
- Increase PG command timeout
- Add support for ZEN and XVG
- Return Coin POW Algorithm in API results
- Expose pool total miner earnings via API
- Improved LTC block explorer links
- Neoscrypt update
- Neoscrypt tests
- Support GBX & CRC
- Don't tie ZMQ block notify to a single address. Fixes #183
- X17 hash integration
- Drop paged retries from Monero Payments as those won't work anyway when transfer_split fails.
- Support Verge X17 variant
- Added support for MaxActiveJobs Bitcoin Pool extra config

src/MiningCore.Tests/Crypto/HashingTests.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Diagnostics;
 using System.Linq;
 using MiningCore.Crypto.Hashing.Algorithms;
 using MiningCore.Crypto.Hashing.Equihash;
@@ -13,6 +14,9 @@ public class HashingTests : TestBase
     {
         private static readonly byte[] testValue = Enumerable.Repeat((byte) 0x80, 32).ToArray();
 
+        // some algos need 80 byte input buffers
+        private static readonly byte[] testValue2 = Enumerable.Repeat((byte)0x80, 80).ToArray();
+
         [Fact]
         public void Blake_Hash_Should_Match()
         {
@@ -77,6 +81,22 @@ public void Scrypt_Hash_Should_Throw_On_Null_Input()
             Assert.Throws<ArgumentNullException>(() => hasher.Digest(null));
         }
 
+        [Fact]
+        public void NeoScrypt_Hash_Should_Match()
+        {
+            var hasher = new NeoScrypt(0);
+            var result = hasher.Digest(testValue2).ToHexString();
+
+            Assert.Equal("7915d56de262bf23b1fb9104cf5d2a13fcbed2f6b4b9b657309c222b09f54bc0", result);
+        }
+
+        [Fact]
+        public void NeoScrypt_Hash_Should_Throw_On_Null_Input()
+        {
+            var hasher = new NeoScrypt(0);
+            Assert.Throws<ArgumentNullException>(() => hasher.Digest(null));
+        }
+
         [Fact]
         public void ScryptN_Hash_Should_Match()
         {
@@ -166,6 +186,22 @@ public void X11_Hash_Should_Throw_On_Null_Input()
             Assert.Throws<ArgumentNullException>(() => hasher.Digest(null));
         }
 
+        [Fact]
+        public void X17_Hash_Should_Match()
+        {
+            var hasher = new X17();
+            var result = hasher.Digest(testValue).ToHexString();
+
+            Assert.Equal("6a9a4f558168e60241e46fe44365021c4d7e7344144ab1739d6fb0125ac4c592", result);
+        }
+
+        [Fact]
+        public void X17_Hash_Should_Throw_On_Null_Input()
+        {
+            var hasher = new Sha256S();
+            Assert.Throws<ArgumentNullException>(() => hasher.Digest(null));
+        }
+
         [Fact]
         public void Skein_Hash_Should_Match()
         {

src/MiningCore/Api/ApiServer.cs

@@ -227,6 +227,7 @@ private async Task GetPoolInfosAsync(HttpContext context, Match m)
                     var result = config.ToPoolInfo(mapper, stats);
 
                     // enrich
+                    result.TotalPaid = cf.Run(con => statsRepo.GetTotalPoolPayments(con, config.Id));
 #if DEBUG
                     var from = new DateTime(2018, 1, 6, 16, 0, 0);
 #else
@@ -256,9 +257,10 @@ private async Task GetPoolInfoAsync(HttpContext context, Match m)
             var response = new GetPoolResponse
             {
                 Pool = pool.ToPoolInfo(mapper, stats)
-            };
-
+            };
+
             // enrich
+            response.Pool.TotalPaid = cf.Run(con => statsRepo.GetTotalPoolPayments(con, pool.Id));
 #if DEBUG
             var from = new DateTime(2018, 1, 7, 16, 0, 0);
 #else
@@ -380,7 +382,7 @@ private async Task PagePoolPaymentsAsync(HttpContext context, Match m)
                 .ToArray();
 
             // enrich payments
-            CoinMetaData.PaymentInfoLinks.TryGetValue(pool.Coin.Type, out var txInfobaseUrl);
+            CoinMetaData.TxInfoLinks.TryGetValue(pool.Coin.Type, out var txInfobaseUrl);
             CoinMetaData.AddressInfoLinks.TryGetValue(pool.Coin.Type, out var addressInfobaseUrl);
 
             foreach (var payment in payments)
@@ -426,7 +428,7 @@ private async Task GetMinerInfoAsync(HttpContext context, Match m)
                     stats.LastPayment = statsResult.LastPayment.Created;
 
                     // Compute info link
-                    if (CoinMetaData.PaymentInfoLinks.TryGetValue(pool.Coin.Type, out var baseUrl))
+                    if (CoinMetaData.TxInfoLinks.TryGetValue(pool.Coin.Type, out var baseUrl))
                         stats.LastPaymentLink = string.Format(baseUrl, statsResult.LastPayment.TransactionConfirmationData);
                 }
 
@@ -464,7 +466,7 @@ private async Task PageMinerPaymentsAsync(HttpContext context, Match m)
                 .ToArray();
 
             // enrich payments
-            CoinMetaData.PaymentInfoLinks.TryGetValue(pool.Coin.Type, out var txInfobaseUrl);
+            CoinMetaData.TxInfoLinks.TryGetValue(pool.Coin.Type, out var txInfobaseUrl);
             CoinMetaData.AddressInfoLinks.TryGetValue(pool.Coin.Type, out var addressInfobaseUrl);
 
             foreach (var payment in payments)

src/MiningCore/Api/Extensions/MiningPoolExtensions.cs

@@ -2,7 +2,9 @@
 using AutoMapper;
 using MiningCore.Api.Responses;
 using MiningCore.Blockchain;
+using MiningCore.Blockchain.Ethereum.Configuration;
 using MiningCore.Configuration;
+using MiningCore.Extensions;
 using MiningCore.Mining;
 
 namespace MiningCore.Api.Extensions
@@ -13,6 +15,9 @@ public static PoolInfo ToPoolInfo(this PoolConfig pool, IMapper mapper, Persiste
         {
             var poolInfo = mapper.Map<PoolInfo>(pool);
 
+            // enrich with basic information
+            poolInfo.Coin.Algorithm = GetPoolAlgorithm(pool);
+
             poolInfo.PoolStats = mapper.Map<PoolStats>(stats);
             poolInfo.NetworkStats = mapper.Map<BlockchainStats>(stats);
 
@@ -24,7 +29,32 @@ public static PoolInfo ToPoolInfo(this PoolConfig pool, IMapper mapper, Persiste
             // pool fees
             poolInfo.PoolFeePercent = (float)pool.RewardRecipients.Sum(x => x.Percentage);
 
+            // strip security critical stuff
+            if (poolInfo.PaymentProcessing.Extra != null)
+            {
+                var extra = poolInfo.PaymentProcessing.Extra;
+
+                extra.StripValue(nameof(EthereumPoolPaymentProcessingConfigExtra.CoinbasePassword));
+            }
+
             return poolInfo;
         }
+
+        private static string GetPoolAlgorithm(PoolConfig pool)
+        {
+            var result = pool.Coin.Algorithm;
+
+            if (string.IsNullOrEmpty(result))
+            {
+                if (CoinMetaData.CoinAlgorithm.TryGetValue(pool.Coin.Type, out var getter))
+                    result = getter(pool.Coin.Type);
+            }
+
+            // Capitalize
+            if (!string.IsNullOrEmpty(result) && result.Length > 1)
+                result = result.Substring(0, 1).ToUpper() + result.Substring(1);
+
+            return result;
+        }
     }
 }

src/MiningCore/Api/Responses/GetPoolsResponse.cs

@@ -30,6 +30,7 @@ namespace MiningCore.Api.Responses
     public class ApiCoinConfig
     {
         public string Type { get; set; }
+        public string Algorithm { get; set; }
     }
 
     public class ApiPoolPaymentProcessingConfig
@@ -63,6 +64,7 @@ public partial class PoolInfo
         public PoolStats PoolStats { get; set; }
         public BlockchainStats NetworkStats { get; set; }
         public MinerPerformanceStats[] TopMiners { get; set; }
+        public decimal TotalPaid { get; set; }
     }
 
     public class GetPoolsResponse

src/MiningCore/Blockchain/Bitcoin/BitcoinConstants.cs

@@ -80,6 +80,7 @@ public class KnownAddresses
         public static readonly Dictionary<CoinType, string> DevFeeAddresses = new Dictionary<CoinType, string>
         {
             {CoinType.BTC, "17QnVor1B6oK1rWnVVBrdX9gFzVkZZbhDm"},
+            {CoinType.BCH, "1LJGTzNDTuTvkHpTxNSdmAEBAXAnEHDVqQ"},
             {CoinType.LTC, "LTK6CWastkmBzGxgQhTTtCUjkjDA14kxzC"},
             {CoinType.DOGE, "DGDuKRhBewGP1kbUz4hszNd2p6dDzWYy9Q"},
             {CoinType.NMC, "NDSLDpFEcTbuRVcWHdJyiRZThVAcb5Z79o"},
@@ -92,9 +93,13 @@ public class KnownAddresses
             {CoinType.MONA, "MBbkeAM3VQKg474bgxJEXrtcnMg8cjHY3S"},
             {CoinType.VTC, "VfCAvPVrksYvwcpU7E44e51HxfvVhcxMXf"},
             {CoinType.ZEC, "t1YHZHz2DGVMJiggD2P4fBQ2TAPgtLSUwZ7"},
-            {CoinType.ZCL, "t1Ysa2CHdpMu8T2zjYXQNQxPgGh8ehe9QYo"},
+            {CoinType.ZCL, "t1MFU1vD3YKgsK6Uh8hW7UTY8mKAV2xVqBr"},
+            {CoinType.ZEN, "znigQacfTvRiwD2TRhwkBHLNchQ2AZisD94"},
             {CoinType.BTG, "GQb77ZuMCyJGZFyxpzqNfm7GB1rQreP4n6"},
-            {CoinType.XMR, "475YVJbPHPedudkhrcNp1wDcLMTGYusGPF5fqE7XjnragVLPdqbCHBdZg3dF4dN9hXMjjvGbykS6a77dTAQvGrpiQqHp2eH"}
+            {CoinType.MOON, "2QvpGimMYLyqKsczQXZjv56h6me3M8orwj" },
+            {CoinType.XVG, "D5xPoHLM6HPkwWSqAweECTSQirJBmRjS8i" },
+            {CoinType.XMR, "475YVJbPHPedudkhrcNp1wDcLMTGYusGPF5fqE7XjnragVLPdqbCHBdZg3dF4dN9hXMjjvGbykS6a77dTAQvGrpiQqHp2eH"},
+            {CoinType.ETN, "etnkQJwBCjmR1MfkU8D355ZWxxLMhs8miPrtKHWN4U3uUowq9ugeKccVBoEG3n13n74us5AkT8tEoTog46w4HBgn8sMuBRhm9h"},
         };
     }
 

src/MiningCore/Blockchain/Bitcoin/BitcoinJob.cs

@@ -23,7 +23,6 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 using System.Globalization;
 using System.IO;
 using System.Linq;
-using System.Numerics;
 using System.Text;
 using MiningCore.Blockchain.Bitcoin.DaemonResponses;
 using MiningCore.Configuration;
@@ -333,16 +332,19 @@ protected virtual BitcoinShare ProcessShareInternal(StratumClient worker, string
             var result = new BitcoinShare
             {
                 BlockHeight = BlockTemplate.Height,
-                IsBlockCandidate = isBlockCandidate
+                BlockReward = rewardToPool.ToDecimal(MoneyUnit.BTC),
+                NetworkDifficulty = Difficulty * shareMultiplier,
+                Difficulty = stratumDifficulty,
             };
 
             var blockBytes = SerializeBlock(headerBytes, coinbase);
-            result.BlockHex = blockBytes.ToHexString();
-            result.BlockHash = blockHasher.Digest(headerBytes, nTime).ToHexString();
-            result.BlockHeight = BlockTemplate.Height;
-            result.BlockReward = rewardToPool.ToDecimal(MoneyUnit.BTC);
-            result.NetworkDifficulty = Difficulty * shareMultiplier;
-            result.Difficulty = stratumDifficulty;
+
+            if (isBlockCandidate)
+            {
+                result.IsBlockCandidate = true;
+                result.BlockHex = blockBytes.ToHexString();
+                result.BlockHash = blockHasher.Digest(headerBytes, nTime).ToHexString();
+            }
 
             return result;
         }

src/MiningCore/Blockchain/Bitcoin/BitcoinJobManager.cs

@@ -20,13 +20,9 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 using System;
 using System.Collections.Generic;
-using System.IO;
 using System.Linq;
 using System.Net;
-using System.Reactive.Disposables;
 using System.Reactive.Linq;
-using System.Text;
-using System.Threading;
 using System.Threading.Tasks;
 using Autofac;
 using MiningCore.Blockchain.Bitcoin.Configuration;
@@ -38,14 +34,11 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 using MiningCore.Crypto.Hashing.Special;
 using MiningCore.DaemonInterface;
 using MiningCore.Extensions;
-using MiningCore.Mining;
 using MiningCore.Notifications;
 using MiningCore.Stratum;
 using MiningCore.Time;
 using MiningCore.Util;
 using NBitcoin;
-using NetMQ;
-using NetMQ.Sockets;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using NLog;
@@ -76,12 +69,12 @@ public BitcoinJobManager(
         protected readonly NotificationService notificationService;
         protected readonly IMasterClock clock;
         protected DaemonClient daemon;
-        protected BitcoinPoolConfigExtra extraPoolConfig;
         protected readonly IExtraNonceProvider extraNonceProvider;
         protected const int ExtranonceBytes = 4;
         protected readonly IHashAlgorithm sha256d = new Sha256D();
         protected readonly IHashAlgorithm sha256dReverse = new DigestReverser(new Sha256D());
-        protected const int MaxActiveJobs = 4;
+        protected int maxActiveJobs = 4;
+        protected BitcoinPoolConfigExtra extraPoolConfig;
         protected readonly IHashAlgorithm sha256s = new Sha256S();
         protected readonly List<TJob> validJobs = new List<TJob>();
         protected IHashAlgorithm blockHasher;
@@ -112,60 +105,21 @@ protected virtual void SetupJobUpdates()
             var sources = new List<IObservable<bool>>();
             var cancelTimeout = new List<IObservable<bool>>();
 
-            // block updates via ZMQ pub/sub
-            var zmqPublisherSocket = extraPoolConfig?.ZmqBlockNotifySocket?.Trim();
+            // collect ports
+            var zmq = poolConfig.Daemons
+                .Where(x => !string.IsNullOrEmpty(x.Extra.SafeExtensionDataAs<BitcoinDaemonEndpointConfigExtra>()?.ZmqBlockNotifySocket))
+                .ToDictionary(x => x, x => x.Extra.SafeExtensionDataAs<BitcoinDaemonEndpointConfigExtra>().ZmqBlockNotifySocket);
 
-            if (!string.IsNullOrEmpty(zmqPublisherSocket))
+            if (zmq.Count > 0)
             {
-                var newJobsPubSub = Observable.Defer(()=> Observable.Create<bool>(obs =>
-                {
-                    var tcs = new CancellationTokenSource();
-
-                    Task.Factory.StartNew(() =>
-                    {
-                        while (!tcs.IsCancellationRequested)
-                        {
-                            try
-                            {
-                                using (var subSocket = new SubscriberSocket())
-                                {
-                                    //subSocket.Options.ReceiveHighWatermark = 1000;
-                                    subSocket.Connect(zmqPublisherSocket);
-                                    subSocket.Subscribe(BitcoinConstants.ZmqPublisherTopicBlockHash);
-
-                                    logger.Info($"Subscribed to {zmqPublisherSocket}/{BitcoinConstants.ZmqPublisherTopicBlockHash} for ZMQ pub/sub block updates");
-
-                                    while (true)
-                                    {
-                                        subSocket.ReceiveMultipartMessage(2);
-                                        //var msg = subSocket.ReceiveMultipartMessage(2);
-                                        //var topic = msg.First().ConvertToString(Encoding.UTF8);
-                                        //var body = msg.Last().ConvertToString(Encoding.UTF8);
-
-                                        obs.OnNext(true);
-                                    }
-                                }
-                            }
-
-                            catch (Exception ex)
-                            {
-                                logger.Error(ex);
-                            }
-
-                            // do not consume all CPU cycles in case of a long lasting error condition
-                            Thread.Sleep(1000);
-                        }
-                    }, tcs.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default);
+                logger.Info(() => $"[{LogCat}] Subscribing to ZMQ push-updates from {string.Join(", ", zmq.Values)}");
 
-                    return Disposable.Create(() =>
-                    {
-                        tcs.Cancel();
-                    });
-                }))
-                .Select(_ => Observable.FromAsync(() => UpdateJob(false, "ZMQ pub/sub")))
-                .Concat()
-                .Publish()
-                .RefCount();
+                var newJobsPubSub = daemon.ZmqSubscribe(zmq, BitcoinConstants.ZmqPublisherTopicBlockHash, 2)
+                    .Do(x=> x.Dispose())    // we don't care about the contents
+                    .Select(_ => Observable.FromAsync(() => UpdateJob(false, "ZMQ pub/sub")))
+                    .Concat()
+                    .Publish()
+                    .RefCount();
 
                 sources.Add(newJobsPubSub);
                 cancelTimeout.Add(newJobsPubSub);
@@ -475,6 +429,9 @@ public override void Configure(PoolConfig poolConfig, ClusterConfig clusterConfi
         {
             extraPoolConfig = poolConfig.Extra.SafeExtensionDataAs<BitcoinPoolConfigExtra>();
 
+            if (extraPoolConfig?.MaxActiveJobs.HasValue == true)
+                maxActiveJobs = extraPoolConfig.MaxActiveJobs.Value;
+
             base.Configure(poolConfig, clusterConfig);
         }
 
@@ -657,8 +614,9 @@ protected virtual async Task<bool> UpdateJob(bool forceUpdate, string via = null
 
                 var job = currentJob;
                 var isNew = job == null ||
-                            job.BlockTemplate?.PreviousBlockhash != blockTemplate.PreviousBlockhash ||
-                            job.BlockTemplate?.Height < blockTemplate.Height;
+                    (blockTemplate != null &&
+                    job.BlockTemplate?.PreviousBlockhash != blockTemplate.PreviousBlockhash &&
+                    blockTemplate.Height > job.BlockTemplate?.Height);
 
                 if (isNew || forceUpdate)
                 {
@@ -685,7 +643,7 @@ protected virtual async Task<bool> UpdateJob(bool forceUpdate, string via = null
                         validJobs.Add(job);
 
                         // trim active jobs
-                        while (validJobs.Count > MaxActiveJobs)
+                        while (validJobs.Count > maxActiveJobs)
                             validJobs.RemoveAt(0);
                     }