Refactor test framework and common utility functions used in test cas…

…es (#1920)

* Refactor test framework methods into common utility functions

* Fix get_id_token bug

* Refactor token_index_in_account and get_decimal_amount into common utilities

* Use utility method for feature_negative_interest test

* Refactor make_utxo, almost_equal, truncate functions into common utilities. Clean up function descriptions

* Refactor find spendable utxo function into common utilities

* Remove unused functions

* Add token utility segment in utilities

* Place get_decimal_amount into utility functions segment

* Extracted common utility transaction functions from rpc_updatemasternode test

* Function desc fix

* Resolve lint errors

* Fix bug in pkg_local_ensure_osx_sysroot function that does not exit out of entered dir if the package exists. Include new gnu-tar support for macos in platform_init function to support tar --transform on macos

* Fix _tar func in make.sh to pass all args to gnu-tar, and fix docker release build workflow to only push x64 linux to docker registry.

* Re-order build targets to select major dev envs first.

* fix build and make.sh script

* revert make.sh changes

.github/workflows/build-release.yaml

@@ -42,7 +42,7 @@ jobs:
           docker tag defichain-x86_64-pc-linux-gnu:${ver} defi/defichain:${tag}
         done
         docker push defi/defichain:${ver}
-    
+
   linux-armhf:
     runs-on: ubuntu-latest
     if: startsWith(github.ref, 'refs/tags/')
@@ -62,20 +62,6 @@ jobs:
         name: defichain-${{ env.BUILD_VERSION }}-arm-linux-gnueabihf
         path: ./build/defichain-${{ env.BUILD_VERSION }}-arm-linux-gnueabihf.tar.gz
 
-    - name: Login to Docker Hub
-      uses: docker/login-action@v2
-      with:
-        username: ${{ env.DOCKER_HUB_USER }}
-        password: ${{ secrets.DOCKER_HUB_TOKEN }}
-
-    - name: Push to Docker Hub
-      run: |
-        set -e; ver=${{ env.BUILD_VERSION }}
-        for tag in $(echo $ver latest); do 
-          docker tag defichain-arm-linux-gnueabihf:${ver} defi/defichain:${tag}
-        done
-        docker push defi/defichain:${ver}
-
   linux-aarch64:
     runs-on: ubuntu-latest
     if: startsWith(github.ref, 'refs/tags/')

test/functional/feature_accounts_n_utxos.py

@@ -9,10 +9,14 @@
 """
 
 from test_framework.test_framework import DefiTestFramework
-
 from test_framework.authproxy import JSONRPCException
-from test_framework.util import assert_equal, assert_raises_rpc_error, \
-    connect_nodes_bi
+from test_framework.fixture_util import CommonFixture
+from test_framework.util import (
+    assert_equal,
+    assert_raises_rpc_error,
+    connect_nodes_bi,
+    get_id_token,
+)
 
 
 class AccountsAndUTXOsTest(DefiTestFramework):
@@ -29,13 +33,13 @@ def run_test(self):
         assert_equal(len(self.nodes[0].listtokens()), 1)  # only one token == DFI
 
         print("Generating initial chain...")
-        self.setup_tokens()
+        CommonFixture.setup_default_tokens(self)
 
         # Stop node #2 for future revert
         self.stop_node(2)
 
-        symbolGOLD = "GOLD#" + self.get_id_token("GOLD")
-        symbolSILVER = "SILVER#" + self.get_id_token("SILVER")
+        symbolGOLD = "GOLD#" + get_id_token(self.nodes[0], "GOLD")
+        symbolSILVER = "SILVER#" + get_id_token(self.nodes[0], "SILVER")
 
         idGold = list(self.nodes[0].gettoken(symbolGOLD).keys())[0]
         idSilver = list(self.nodes[0].gettoken(symbolSILVER).keys())[0]

test/functional/feature_any_accounts_to_accounts.py

@@ -9,10 +9,12 @@
 """
 
 from test_framework.test_framework import DefiTestFramework
-
 from test_framework.authproxy import JSONRPCException
-from test_framework.util import assert_equal, \
-    connect_nodes_bi
+from test_framework.fixture_util import CommonFixture
+from test_framework.util import (
+    assert_equal,
+    connect_nodes_bi,
+)
 
 from decimal import Decimal
 
@@ -67,7 +69,7 @@ def run_test(self):
             }
         ]
         # inside this function "tokenId" and "symbolId" will be assigned for each token obj
-        self.setup_tokens(tokens)
+        CommonFixture.setup_default_tokens(self, tokens)
 
         # list_tokens = self.nodes[0].listtokens()
         # pprint(list_tokens)

test/functional/feature_dusd_loans.py

@@ -6,24 +6,18 @@
 
 from test_framework.test_framework import DefiTestFramework
 from test_framework.authproxy import JSONRPCException
-from test_framework.util import assert_equal, assert_raises_rpc_error
+from test_framework.util import (
+    assert_equal,
+    assert_raises_rpc_error,
+    token_index_in_account,
+    get_decimal_amount,
+)
+
 import calendar
 import time
 from decimal import Decimal
 
 
-def get_decimal_amount(amount):
-    account_tmp = amount.split('@')[0]
-    return Decimal(account_tmp)
-
-
-def token_index_in_account(accounts, symbol):
-    for id in range(len(accounts)):
-        if symbol in accounts[id]:
-            return id
-    return -1
-
-
 ERR_STRING_MIN_COLLATERAL_DFI_PCT = "At least 50% of the minimum required collateral must be in DFI"
 ERR_STRING_MIN_COLLATERAL_DFI_DUSD_PCT = "At least 50% of the minimum required collateral must be in DFI or DUSD"
 

test/functional/feature_negative_interest.py

@@ -6,19 +6,16 @@
 
 from test_framework.test_framework import DefiTestFramework
 from test_framework.authproxy import JSONRPCException
-
-from test_framework.util import assert_equal, assert_greater_than_or_equal
+from test_framework.util import (
+    assert_equal,
+    assert_greater_than_or_equal,
+    get_decimal_amount,
+)
 
 import calendar
 import time
 from decimal import ROUND_DOWN, ROUND_UP, Decimal
 
-
-def getDecimalAmount(amount):
-    amountTmp = amount.split('@')[0]
-    return Decimal(amountTmp)
-
-
 class NegativeInterestTest(DefiTestFramework):
     def set_test_params(self):
         self.num_nodes = 1
@@ -770,14 +767,14 @@ def payback_interests_and_continue_with_negative_interest(self):
 
         amounts0 = []
         for amount in vault["loanAmounts"]:
-            amounts0.append(getDecimalAmount(amount))
+            amounts0.append(get_decimal_amount(amount))
 
         self.nodes[0].generate(1)
 
         vault = self.nodes[0].getvault(self.vaultId7, verbose)
         amounts1 = []
         for amount in vault["loanAmounts"]:
-            amounts1.append(getDecimalAmount(amount))
+            amounts1.append(get_decimal_amount(amount))
 
         for amount in amounts0:
             assert_greater_than_or_equal(amount, amounts1[amounts0.index(amount)])
@@ -791,7 +788,7 @@ def payback_interests_and_continue_with_negative_interest(self):
         vault = self.nodes[0].getvault(self.vaultId7, verbose)
         amounts2 = []
         for amount in vault["loanAmounts"]:
-            amounts2.append(getDecimalAmount(amount))
+            amounts2.append(get_decimal_amount(amount))
 
         for amount in amounts1:
             assert_greater_than_or_equal(amount, amounts2[amounts1.index(amount)])
@@ -870,15 +867,15 @@ def let_loan_be_paid_by_negative_interests(self):
         assert_equal(storedInterest["interestToHeight"], '0.000000000000000000000000')
 
         storedLoans = loanTokens
-        storedAmount = getDecimalAmount(storedLoans[0])
+        storedAmount = get_decimal_amount(storedLoans[0])
         self.nodes[0].generate(10)
 
         self.nodes[0].takeloan({
             'vaultId': self.vaultId8,
             'amounts': "0.00000001@" + self.symboldUSD})
         self.nodes[0].generate(1)
         storedLoans1 = self.nodes[0].getloantokens(self.vaultId8)
-        storedAmount1 = getDecimalAmount(storedLoans1[0])
+        storedAmount1 = get_decimal_amount(storedLoans1[0])
         storedInterest1 = self.nodes[0].getstoredinterest(self.vaultId8, self.symboldUSD)
 
         assert (storedAmount > storedAmount1)
@@ -918,7 +915,7 @@ def various_payback_tests(self):
         assert_equal(vault["interestPerBlockValue"], '-0.095129280821917808219178')
 
         storedLoans = self.nodes[0].getloantokens(self.vaultId9)
-        storedAmount = getDecimalAmount(storedLoans[0])
+        storedAmount = get_decimal_amount(storedLoans[0])
         assert_equal(storedAmount, Decimal('10.00000000'))
 
         [balanceDUSDbefore, _] = self.nodes[0].getaccount(self.account)[0].split('@')
@@ -946,13 +943,13 @@ def various_payback_tests(self):
         self.nodes[0].generate(10)
 
         accountInfo = self.nodes[0].getaccount(self.account)
-        assert_equal(getDecimalAmount(accountInfo[0]), Decimal('10.00000000') + Decimal(balanceDUSDafter))
+        assert_equal(get_decimal_amount(accountInfo[0]), Decimal('10.00000000') + Decimal(balanceDUSDafter))
         vault = self.nodes[0].getvault(self.vaultId9, verbose)
         assert_equal(vault["loanValue"], Decimal('9.04870720'))
         assert_equal(vault["interestPerBlockValue"], '-0.095129280821917808219178')
 
         storedLoans = self.nodes[0].getloantokens(self.vaultId9)
-        storedAmount = getDecimalAmount(storedLoans[0])
+        storedAmount = get_decimal_amount(storedLoans[0])
         storedInterest = self.nodes[0].getstoredinterest(self.vaultId9, self.symboldUSD)
         assert_equal(storedAmount, Decimal('10.00000000'))
 
@@ -973,7 +970,7 @@ def various_payback_tests(self):
         assert_equal(Decimal(loanAmount), Decimal(loanAmountBefore) - Decimal('0.00000001') + Decimal(interestAmount))
 
         storedLoans1 = self.nodes[0].getloantokens(self.vaultId9)
-        storedAmount1 = getDecimalAmount(storedLoans1[0])
+        storedAmount1 = get_decimal_amount(storedLoans1[0])
         expectedAmount = Decimal(
             Decimal('10') + 10 * Decimal(storedInterest["interestPerBlock"]) - Decimal('0.00000001')).quantize(
             Decimal('1E-8'), ROUND_UP)

test/functional/feature_poolpair_liquidity.py

@@ -9,10 +9,13 @@
 """
 
 from test_framework.test_framework import DefiTestFramework
-
 from test_framework.authproxy import JSONRPCException
-from test_framework.util import assert_equal, \
-    connect_nodes_bi
+from test_framework.fixture_util import CommonFixture
+from test_framework.util import (
+    assert_equal,
+    connect_nodes_bi,
+    get_id_token,
+)
 
 from decimal import Decimal, ROUND_DOWN
 
@@ -40,14 +43,14 @@ def run_test(self):
         assert_equal(len(self.nodes[0].listtokens()), 1)  # only one token == DFI
 
         print("Generating initial chain...")
-        self.setup_tokens()
+        CommonFixture.setup_default_tokens(self)
 
         # stop node #2 for future revert
         self.stop_node(2)
         connect_nodes_bi(self.nodes, 0, 3)
 
-        symbolGOLD = "GOLD#" + self.get_id_token("GOLD")
-        symbolSILVER = "SILVER#" + self.get_id_token("SILVER")
+        symbolGOLD = "GOLD#" + get_id_token(self.nodes[0], "GOLD")
+        symbolSILVER = "SILVER#" + get_id_token(self.nodes[0], "SILVER")
 
         idGold = list(self.nodes[0].gettoken(symbolGOLD).keys())[0]
         idSilver = list(self.nodes[0].gettoken(symbolSILVER).keys())[0]

test/functional/feature_poolswap.py

@@ -9,12 +9,13 @@
 """
 
 from test_framework.test_framework import DefiTestFramework
-
 from test_framework.authproxy import JSONRPCException
+from test_framework.fixture_util import CommonFixture
 from test_framework.util import (
     assert_equal,
     disconnect_nodes,
     assert_raises_rpc_error,
+    get_id_token,
 )
 
 from decimal import Decimal
@@ -39,9 +40,9 @@ def set_test_params(self):
 
     def setup(self):
         assert_equal(len(self.nodes[0].listtokens()), 1)  # only one token == DFI
-        self.setup_tokens()
-        self.symbolGOLD = "GOLD#" + self.get_id_token("GOLD")
-        self.symbolSILVER = "SILVER#" + self.get_id_token("SILVER")
+        CommonFixture.setup_default_tokens(self)
+        self.symbolGOLD = "GOLD#" + get_id_token(self.nodes[0], "GOLD")
+        self.symbolSILVER = "SILVER#" + get_id_token(self.nodes[0], "SILVER")
         self.idGold = list(self.nodes[0].gettoken(self.symbolGOLD).keys())[0]
         self.idSilver = list(self.nodes[0].gettoken(self.symbolSILVER).keys())[0]
 

test/functional/feature_poolswap_composite.py

@@ -6,12 +6,13 @@
 """Test poolpair composite swap RPC."""
 
 from test_framework.test_framework import DefiTestFramework
-
 from test_framework.authproxy import JSONRPCException
+from test_framework.fixture_util import CommonFixture
 from test_framework.util import (
     assert_equal,
     disconnect_nodes,
-    assert_raises_rpc_error
+    assert_raises_rpc_error,
+    get_id_token,
 )
 
 from decimal import Decimal
@@ -68,14 +69,14 @@ def run_test(self):
                 "amount": 1000000
             },
         ]
-        self.setup_tokens(tokens)
+        CommonFixture.setup_default_tokens(self, tokens)
         disconnect_nodes(self.nodes[0], 1)
 
-        symbolDOGE = "DOGE#" + self.get_id_token("DOGE")
-        symbolTSLA = "TSLA#" + self.get_id_token("TSLA")
-        symbolDUSD = "DUSD#" + self.get_id_token("DUSD")
-        symbolLTC = "LTC#" + self.get_id_token("LTC")
-        symbolUSDC = "USDC#" + self.get_id_token("USDC")
+        symbolDOGE = "DOGE#" + get_id_token(self.nodes[0], "DOGE")
+        symbolTSLA = "TSLA#" + get_id_token(self.nodes[0], "TSLA")
+        symbolDUSD = "DUSD#" + get_id_token(self.nodes[0], "DUSD")
+        symbolLTC = "LTC#" + get_id_token(self.nodes[0], "LTC")
+        symbolUSDC = "USDC#" + get_id_token(self.nodes[0], "USDC")
 
         idDOGE = list(self.nodes[0].gettoken(symbolDOGE).keys())[0]
         idTSLA = list(self.nodes[0].gettoken(symbolTSLA).keys())[0]

test/functional/feature_poolswap_mainnet.py

@@ -10,7 +10,11 @@
 
 from test_framework.test_framework import DefiTestFramework
 from test_framework.authproxy import JSONRPCException
-from test_framework.util import assert_equal
+from test_framework.util import (
+    assert_equal,
+    get_id_token,
+)
+
 from decimal import Decimal
 
 
@@ -50,9 +54,9 @@ def create_tokens(self):
             "collateralAddress": self.account0
         })
         self.nodes[0].generate(1)
-        self.symbol_key_GOLD = "GOLD#" + str(self.get_id_token(self.symbolGOLD))
-        self.symbol_key_SILVER = "SILVER#" + str(self.get_id_token(self.symbolSILVER))
-        self.symbol_key_DOGE = "DOGE#" + str(self.get_id_token(self.symbolDOGE))
+        self.symbol_key_GOLD = "GOLD#" + str(get_id_token(self.nodes[0], self.symbolGOLD))
+        self.symbol_key_SILVER = "SILVER#" + str(get_id_token(self.nodes[0], self.symbolSILVER))
+        self.symbol_key_DOGE = "DOGE#" + str(get_id_token(self.nodes[0], self.symbolDOGE))
 
     def mint_tokens(self, amount=1000):