Merge branch 'v4.0.105' of github.com:ergoplatform/ergo into 5.0-testnet

README.md

@@ -82,7 +82,7 @@ To run specific Ergo version `<VERSION>` as a service with custom config `/path/
         -e MAX_HEAP=3G \
         ergoplatform/ergo:<VERSION> --<networkId> -c /etc/myergo.conf
 
-Available versions can be found on [Ergo Docker image page](https://hub.docker.com/r/ergoplatform/ergo/tags), for example, `v4.0.46`.
+Available versions can be found on [Ergo Docker image page](https://hub.docker.com/r/ergoplatform/ergo/tags), for example, `v4.0.105`.
 
 This will connect to the Ergo mainnet or testnet following your configuration passed in `myergo.conf` and network flag `--<networkId>`. Every default config value would be overwritten with corresponding value in `myergo.conf`. `MAX_HEAP` variable can be used to control how much memory can the node consume.
 

ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/EncryptedSecret.scala

@@ -13,18 +13,21 @@ import scorex.util.encode.Base16
   * @param iv           - cipher initialization vector
   * @param authTag      - message authentication tag
   * @param cipherParams - cipher params
+  * @param usePre1627KeyDerivation - use incorrect(previous) BIP32 derivation, expected to be false for new wallets, and true for old pre-1627 wallets (see https://github.com/ergoplatform/ergo/issues/1627 for details)
   */
 final case class EncryptedSecret(cipherText: String, salt: String, iv: String, authTag: String,
-                                 cipherParams: EncryptionSettings)
+                                 cipherParams: EncryptionSettings, usePre1627KeyDerivation: Option[Boolean])
 
 object EncryptedSecret {
   def apply(cipherText: Array[Byte], salt: Array[Byte], iv: Array[Byte], authTag: Array[Byte],
-            cipherParams: EncryptionSettings): EncryptedSecret = {
+            cipherParams: EncryptionSettings, usePre1627KeyDerivation: Option[Boolean]): EncryptedSecret = {
     new EncryptedSecret(
       Base16.encode(cipherText),
       Base16.encode(salt),
       Base16.encode(iv),
-      Base16.encode(authTag), cipherParams)
+      Base16.encode(authTag), 
+      cipherParams, 
+      usePre1627KeyDerivation)
   }
 
   implicit object EncryptedSecretEncoder extends Encoder[EncryptedSecret] {
@@ -35,7 +38,8 @@ object EncryptedSecret {
         "salt" -> secret.salt.asJson,
         "iv" -> secret.iv.asJson,
         "authTag" -> secret.authTag.asJson,
-        "cipherParams" -> secret.cipherParams.asJson
+        "cipherParams" -> secret.cipherParams.asJson,
+        "usePre1627KeyDerivation" -> secret.usePre1627KeyDerivation.asJson
       )
     }
 
@@ -50,7 +54,8 @@ object EncryptedSecret {
         iv <- cursor.downField("iv").as[String]
         authTag <- cursor.downField("authTag").as[String]
         cipherParams <- cursor.downField("cipherParams").as[EncryptionSettings]
-      } yield EncryptedSecret(cipherText, salt, iv, authTag, cipherParams)
+        usePre1627KeyDerivation <- cursor.downField("usePre1627KeyDerivation").as[Option[Boolean]]
+      } yield EncryptedSecret(cipherText, salt, iv, authTag, cipherParams, usePre1627KeyDerivation)
     }
 
   }

ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedKey.scala

@@ -15,10 +15,6 @@ package org.ergoplatform.wallet.secrets
   */
 trait ExtendedKey[T <: ExtendedKey[T]] {
 
-  val keyBytes: Array[Byte]
-
-  val chainCode: Array[Byte]
-
   val path: DerivationPath
 
   /** Returns subtype reference.

ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedPublicKey.scala

@@ -16,9 +16,9 @@ import scala.annotation.tailrec
   * Public key, its chain code and path in key tree.
   * (see: https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki)
   */
-final class ExtendedPublicKey(val keyBytes: Array[Byte],
-                              val chainCode: Array[Byte],
-                              val path: DerivationPath)
+final class ExtendedPublicKey(private[secrets] val keyBytes: Array[Byte],
+                              private[secrets] val chainCode: Array[Byte],
+                              val path: DerivationPath) 
   extends ExtendedKey[ExtendedPublicKey] {
 
   def selfReflection: ExtendedPublicKey = this

ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKey.scala

@@ -16,8 +16,9 @@ import sigmastate.interpreter.CryptoConstants
   * Secret, its chain code and path in key tree.
   * (see: https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki)
   */
-final class ExtendedSecretKey(val keyBytes: Array[Byte],
-                              val chainCode: Array[Byte],
+final class ExtendedSecretKey(private[secrets] val keyBytes: Array[Byte],
+                              private[secrets] val chainCode: Array[Byte],
+                              private[secrets] val usePre1627KeyDerivation: Boolean,
                               val path: DerivationPath)
   extends ExtendedKey[ExtendedSecretKey] with SecretKey {
 
@@ -69,8 +70,17 @@ object ExtendedSecretKey {
       .mod(CryptoConstants.groupOrder)
     if (childKeyProtoDecoded.compareTo(CryptoConstants.groupOrder) >= 0 || childKey.equals(BigInteger.ZERO))
       deriveChildSecretKey(parentKey, idx + 1)
-    else
-      new ExtendedSecretKey(BigIntegers.asUnsignedByteArray(childKey), childChainCode, parentKey.path.extended(idx))
+    else {
+      val keyBytes = if (parentKey.usePre1627KeyDerivation) {
+        // maybe less than 32 bytes if childKey is small enough while BIP32 requires 32 bytes.
+        // see https://github.com/ergoplatform/ergo/issues/1627 for details
+        BigIntegers.asUnsignedByteArray(childKey)
+      } else {
+        // padded with leading zeroes to 32 bytes
+        BigIntegers.asUnsignedByteArray(Constants.SecretKeyLength, childKey)
+      }
+      new ExtendedSecretKey(keyBytes, childChainCode, parentKey.usePre1627KeyDerivation, parentKey.path.extended(idx))
+      }
   }
 
   def deriveChildPublicKey(parentKey: ExtendedSecretKey, idx: Int): ExtendedPublicKey = {
@@ -80,9 +90,15 @@ object ExtendedSecretKey {
     new ExtendedPublicKey(derivedPk, derivedSecret.chainCode, derivedPath)
   }
 
-  def deriveMasterKey(seed: Array[Byte]): ExtendedSecretKey = {
+
+  /**
+   * Derives master secret key from the seed 
+   * @param seed - seed bytes
+   * @param usePre1627KeyDerivation - use incorrect(previous) BIP32 derivation, expected to be false for new wallets, and true for old pre-1627 wallets (see https://github.com/ergoplatform/ergo/issues/1627 for details)
+   */
+  def deriveMasterKey(seed: Array[Byte], usePre1627KeyDerivation: Boolean): ExtendedSecretKey = {
     val (masterKey, chainCode) = HmacSHA512.hash(Constants.BitcoinSeed, seed).splitAt(Constants.SecretKeyLength)
-    new ExtendedSecretKey(masterKey, chainCode, DerivationPath.MasterPath)
+    new ExtendedSecretKey(masterKey, chainCode, usePre1627KeyDerivation, DerivationPath.MasterPath)
   }
 
 }
@@ -104,7 +120,7 @@ object ExtendedSecretKeySerializer extends ErgoWalletSerializer[ExtendedSecretKe
     val chainCode = r.getBytes(Constants.SecretKeyLength)
     val pathLen = r.getUInt().toIntExact
     val path = DerivationPathSerializer.parseBytes(r.getBytes(pathLen))
-    new ExtendedSecretKey(keyBytes, chainCode, path)
+    new ExtendedSecretKey(keyBytes, chainCode, false, path)
   }
 
 }

ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/JsonSecretStorage.scala

@@ -39,9 +39,11 @@ final class JsonSecretStorage(val secretFile: File, encryptionSettings: Encrypti
     * @param mnemonicPassOpt - optional SecretString mnemonic password to be erased after use.
     */
   override def checkSeed(mnemonic: SecretString, mnemonicPassOpt: Option[SecretString]): Boolean = {
-    val seed = Mnemonic.toSeed(mnemonic, mnemonicPassOpt)
-    val secret = ExtendedSecretKey.deriveMasterKey(seed)
-    unlockedSecret.fold(false)(s => secret.equals(s))
+    unlockedSecret.fold(false){ uSecret => 
+      val seed = Mnemonic.toSeed(mnemonic, mnemonicPassOpt)
+      val secret = ExtendedSecretKey.deriveMasterKey(seed, uSecret.usePre1627KeyDerivation)
+      secret.equals(uSecret)
+    }
   }
 
   /**
@@ -58,20 +60,20 @@ final class JsonSecretStorage(val secretFile: File, encryptionSettings: Encrypti
           .flatMap(txt => Base16.decode(encryptedSecret.salt)
             .flatMap(salt => Base16.decode(encryptedSecret.iv)
               .flatMap(iv => Base16.decode(encryptedSecret.authTag)
-                .map(tag => (txt, salt, iv, tag))
+                .map(tag => (txt, salt, iv, tag, encryptedSecret.usePre1627KeyDerivation))
               )
             )
           )
-          .flatMap { case (cipherText, salt, iv, tag) => {
+          .flatMap { case (cipherText, salt, iv, tag, usePre1627KeyDerivation) => {
               val res = crypto.AES.decrypt(cipherText, pass.getData(), salt, iv, tag)(encryptionSettings)
               pass.erase()
               res
+                .map(seed => unlockedSecret = Some(ExtendedSecretKey.deriveMasterKey(seed, usePre1627KeyDerivation.getOrElse(true))))
             }
           }
       }
-      .fold(Failure(_), Success(_))
+      . fold(Failure(_), Success(_))
       .flatten
-      .map(seed => unlockedSecret = Some(ExtendedSecretKey.deriveMasterKey(seed)))
   }
 
   /**
@@ -87,13 +89,16 @@ final class JsonSecretStorage(val secretFile: File, encryptionSettings: Encrypti
 object JsonSecretStorage {
 
   /**
-    * Initializes storage instance with new wallet file encrypted with the given `pass`.
-    */
-  def init(seed: Array[Byte], pass: SecretString)(settings: SecretStorageSettings): JsonSecretStorage = {
+   * Initializes storage instance with new wallet file encrypted with the given `pass`.
+   * @param seed   - seed bytes
+   * @param pass   - encryption password
+   * @param usePre1627KeyDerivation - use incorrect(previous) BIP32 derivation, expected to be false for new wallets, and true for old pre-1627 wallets (see https://github.com/ergoplatform/ergo/issues/1627 for details)
+   */
+  def init(seed: Array[Byte], pass: SecretString, usePre1627KeyDerivation: Boolean)(settings: SecretStorageSettings): JsonSecretStorage = {
     val iv = scorex.utils.Random.randomBytes(crypto.AES.NonceBitsLen / 8)
     val salt = scorex.utils.Random.randomBytes(32)
     val (ciphertext, tag) = crypto.AES.encrypt(seed, pass.getData(), salt, iv)(settings.encryption)
-    val encryptedSecret = EncryptedSecret(ciphertext, salt, iv, tag, settings.encryption)
+    val encryptedSecret = EncryptedSecret(ciphertext, salt, iv, tag, settings.encryption, Some(usePre1627KeyDerivation))
     val uuid = UUID.nameUUIDFromBytes(ciphertext)
     new File(settings.secretDir).mkdirs()
     val file = new File(s"${settings.secretDir}/$uuid.json")
@@ -110,14 +115,19 @@ object JsonSecretStorage {
   }
 
   /**
-    * Initializes storage with the seed derived from an existing mnemonic phrase.
-    */
+   * Initializes storage with the seed derived from an existing mnemonic phrase.
+   * @param mnemonic - mnemonic phase
+   * @param mnemonicPassOpt - optional mnemonic password
+   * @param encryptionPass - encryption password
+   * @param usePre1627KeyDerivation - use incorrect(previous) BIP32 derivation, expected to be false for new wallets, and true for old pre-1627 wallets (see https://github.com/ergoplatform/ergo/issues/1627 for details)
+   */
   def restore(mnemonic: SecretString,
               mnemonicPassOpt: Option[SecretString],
               encryptionPass: SecretString,
-              settings: SecretStorageSettings): JsonSecretStorage = {
+              settings: SecretStorageSettings, 
+              usePre1627KeyDerivation: Boolean): JsonSecretStorage = {
     val seed = Mnemonic.toSeed(mnemonic, mnemonicPassOpt)
-    init(seed, encryptionPass)(settings)
+    init(seed, encryptionPass, usePre1627KeyDerivation)(settings)
   }
 
   def readFile(settings: SecretStorageSettings): Try[JsonSecretStorage] = {

ergo-wallet/src/test/java/org/ergoplatform/wallet/AddressGenerationDemo.java

@@ -19,7 +19,7 @@ public static byte[] secretSeedFromMnemonic(String mnemonic) {
     }
 
     public static ExtendedSecretKey masterSecretFromSeed(byte[] seed) {
-        ExtendedSecretKey rootSk = ExtendedSecretKey.deriveMasterKey(seed);
+        ExtendedSecretKey rootSk = ExtendedSecretKey.deriveMasterKey(seed, false);
         return rootSk;
     }
 

ergo-wallet/src/test/java/org/ergoplatform/wallet/CreateTransactionDemo.java

@@ -33,7 +33,7 @@ public static void createTransaction() throws Exception {
 
         // Create an address
         byte[] entropy = Random.randomBytes(32);
-        ExtendedSecretKey extendedSecretKey = ExtendedSecretKey.deriveMasterKey(entropy);
+        ExtendedSecretKey extendedSecretKey = ExtendedSecretKey.deriveMasterKey(entropy, false);
         ErgoAddress myAddress = P2PKAddress.apply(extendedSecretKey.privateInput().publicImage(), encoder);
 
         int transferAmt = 25000000; // amount to transfer
@@ -73,7 +73,7 @@ public static void createMultiPaymentTransaction() {
 
         // Create an address
         byte[] entropy = Random.randomBytes(32);
-        ExtendedSecretKey extendedSecretKey = ExtendedSecretKey.deriveMasterKey(entropy);
+        ExtendedSecretKey extendedSecretKey = ExtendedSecretKey.deriveMasterKey(entropy, false);
         ErgoAddress myAddress = P2PKAddress.apply(extendedSecretKey.privateInput().publicImage(), encoder);
 
         int feeAmt = 1000000; // minimal fee amount
@@ -118,11 +118,11 @@ public static void createTransactionMultipleKeys() throws Exception {
 
         // Create second address
         byte[] entropy1 = Random.randomBytes(32);
-        ExtendedSecretKey extendedSecretKey1 = ExtendedSecretKey.deriveMasterKey(entropy1);
+        ExtendedSecretKey extendedSecretKey1 = ExtendedSecretKey.deriveMasterKey(entropy1, false);
         ErgoAddress changeAddress = P2PKAddress.apply(extendedSecretKey1.privateInput().publicImage(), encoder);
 
         byte[] entropy2 = Random.randomBytes(32);
-        ExtendedSecretKey extendedSecretKey2 = ExtendedSecretKey.deriveMasterKey(entropy2);
+        ExtendedSecretKey extendedSecretKey2 = ExtendedSecretKey.deriveMasterKey(entropy2, false);
 
 
         int transferAmt = 25000000; // amount to transfer

ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/ErgoProvingInterpreterSpec.scala

@@ -23,7 +23,7 @@ class ErgoProvingInterpreterSpec
     with Generators
     with InterpreterSpecCommon {
 
-  private def obtainSecretKey() = ExtendedSecretKey.deriveMasterKey(Random.randomBytes(32))
+  private def obtainSecretKey() = ExtendedSecretKey.deriveMasterKey(Random.randomBytes(32), usePre1627KeyDerivation = false)
 
   it should "produce proofs with primitive secrets" in {
     val extendedSecretKey = obtainSecretKey()

ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/ErgoUnsafeProverSpec.scala

@@ -17,7 +17,7 @@ class ErgoUnsafeProverSpec
 
   it should "produce the same proof as a fully-functional prover" in {
     val entropy = Random.randomBytes(32)
-    val extendedSecretKey = ExtendedSecretKey.deriveMasterKey(entropy)
+    val extendedSecretKey = ExtendedSecretKey.deriveMasterKey(entropy, usePre1627KeyDerivation = false)
     val fullProver = ErgoProvingInterpreter(extendedSecretKey, parameters)
     val unsafeProver = ErgoUnsafeProver
 

ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/DerivationPathSpec.scala

@@ -33,7 +33,7 @@ class DerivationPathSpec
 
     val seed = Mnemonic.toSeed(SecretString.create(mnemonic), None)
 
-    val masterKey = ExtendedSecretKey.deriveMasterKey(seed)
+    val masterKey = ExtendedSecretKey.deriveMasterKey(seed, usePre1627KeyDerivation = false)
     val dp = DerivationPath.nextPath(IndexedSeq(masterKey), usePreEip3Derivation = false).get
     val sk = masterKey.derive(dp)
     val pk = sk.publicKey.key
@@ -69,7 +69,7 @@ class DerivationPathSpec
 
     val seed = Mnemonic.toSeed(SecretString.create(mnemonic), None)
 
-    val masterKey = ExtendedSecretKey.deriveMasterKey(seed)
+    val masterKey = ExtendedSecretKey.deriveMasterKey(seed, usePre1627KeyDerivation = false)
     val dp = DerivationPath.nextPath(IndexedSeq(masterKey), usePreEip3Derivation = true).get
     val sk = masterKey.derive(dp)
     val pk = sk.publicKey.key
@@ -106,7 +106,7 @@ class DerivationPathSpec
 
     val seed = Mnemonic.toSeed(SecretString.create(mnemonic), None)
 
-    val masterKey = ExtendedSecretKey.deriveMasterKey(seed)
+    val masterKey = ExtendedSecretKey.deriveMasterKey(seed, usePre1627KeyDerivation = false)
     P2PKAddress(masterKey.publicKey.key).toString() shouldBe address
 
     masterKey.path shouldBe masterKeyDerivation

ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/ExtendedPublicKeySpec.scala

@@ -8,7 +8,7 @@ class ExtendedPublicKeySpec
     with ScalaCheckPropertyChecks
     with Generators {
 
-  val rootSecret: ExtendedSecretKey = ExtendedSecretKey.deriveMasterKey(seed)
+  val rootSecret: ExtendedSecretKey = ExtendedSecretKey.deriveMasterKey(seed, usePre1627KeyDerivation = false)
 
   property("public key tree derivation from seed (test vectors from BIP32 check)") {
     val expectedRoot = "kTV6HY41wXZVSqdpoe1heA8pBZFEN2oq5T59ZCMpqKKJ"

ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKeySpec.scala

@@ -7,6 +7,8 @@ import org.scalatest.matchers.should.Matchers
 import org.scalatest.propspec.AnyPropSpec
 import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks
 import scorex.util.encode.Base58
+import org.ergoplatform.P2PKAddress
+import org.ergoplatform.ErgoAddressEncoder
 
 class ExtendedSecretKeySpec
   extends AnyPropSpec
@@ -16,6 +18,9 @@ class ExtendedSecretKeySpec
   val seedStr = "edge talent poet tortoise trumpet dose"
   val seed: Array[Byte] = Mnemonic.toSeed(SecretString.create(seedStr))
 
+  def equalBase58(v1: Array[Byte], v2b58: String): Assertion =
+    Base58.encode(v1) shouldEqual v2b58
+
   property("key tree derivation from seed (test vectors from BIP32 check)") {
     val expectedRoot = "4rEDKLd17LX4xNR8ss4ithdqFRc3iFnTiTtQbanWJbCT"
     val cases = Seq(
@@ -24,7 +29,7 @@ class ExtendedSecretKeySpec
       ("DWMp3L9JZiywxSb5gSjc5dYxPwEZ6KkmasNiHD6VRcpJ", Index.hardIndex(2))
     )
 
-    val root = ExtendedSecretKey.deriveMasterKey(seed)
+    val root = ExtendedSecretKey.deriveMasterKey(seed, usePre1627KeyDerivation = false)
 
     equalBase58(root.keyBytes, expectedRoot)
 
@@ -42,14 +47,33 @@ class ExtendedSecretKeySpec
       ("DWMp3L9JZiywxSb5gSjc5dYxPwEZ6KkmasNiHD6VRcpJ", "m/1/2/2'")
     )
 
-    val root = ExtendedSecretKey.deriveMasterKey(seed)
+    val root = ExtendedSecretKey.deriveMasterKey(seed, usePre1627KeyDerivation = false)
 
     cases.foreach { case (expectedKey, path) =>
       val derived = root.derive(DerivationPath.fromEncoded(path).get)
       equalBase58(derived.keyBytes, expectedKey)
     }
   }
 
-  def equalBase58(v1: Array[Byte], v2b58: String): Assertion = Base58.encode(v1) shouldEqual v2b58
+  property("1627 BIP32 key derivation fix (31 bit child key)") {
+    // see https://github.com/ergoplatform/ergo/issues/1627 for details
+    val seedStr =
+      "race relax argue hair sorry riot there spirit ready fetch food hedgehog hybrid mobile pretty"
+    val seed: Array[Byte] = Mnemonic.toSeed(SecretString.create(seedStr))
+    val path = "m/44'/429'/0'/0/0"
+    val addressEncoder = ErgoAddressEncoder(ErgoAddressEncoder.MainnetNetworkPrefix)
+
+    val pre1627DerivedSecretKey = ExtendedSecretKey.deriveMasterKey(seed, usePre1627KeyDerivation = true)
+      .derive(DerivationPath.fromEncoded(path).get)
+
+    P2PKAddress(pre1627DerivedSecretKey.publicKey.key)(addressEncoder).toString shouldEqual 
+      "9ewv8sxJ1jfr6j3WUSbGPMTVx3TZgcJKdnjKCbJWhiJp5U62uhP"  
+
+    val fixedDerivedSecretKey = ExtendedSecretKey.deriveMasterKey(seed, usePre1627KeyDerivation = false)
+      .derive(DerivationPath.fromEncoded(path).get)
+
+    P2PKAddress(fixedDerivedSecretKey.publicKey.key)(addressEncoder).toString shouldEqual 
+      "9eYMpbGgBf42bCcnB2nG3wQdqPzpCCw5eB1YaWUUen9uCaW3wwm"  
+  }
 
 }