From ce6b9e32b6fcf123f51612235002800ed67011b5 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Tue, 24 Aug 2021 17:55:36 +0200 Subject: [PATCH 1/5] inlt files run through sys._MEIPASS for binaries --- build_configs/macos/build.spec | 2 +- .../key_handling/key_derivation/mnemonic.py | 19 ++++--------------- eth2deposit/utils/file_handling.py | 16 ++++++++++++++++ eth2deposit/utils/intl.py | 4 ++++ 4 files changed, 25 insertions(+), 16 deletions(-) create mode 100644 eth2deposit/utils/file_handling.py diff --git a/build_configs/macos/build.spec b/build_configs/macos/build.spec index ddc3fd37..1b5e4938 100644 --- a/build_configs/macos/build.spec +++ b/build_configs/macos/build.spec @@ -7,7 +7,7 @@ a = Analysis(['../../eth2deposit/deposit.py'], binaries=None, datas=[ ('../../eth2deposit/key_handling/key_derivation/word_lists/*.txt', './eth2deposit/key_handling/key_derivation/word_lists/'), - ('../../eth2deposit/intl', './eth2deposit/intl',) + ('../../eth2deposit/intl', './eth2deposit/intl'), ], hiddenimports=[], hookspath=[], diff --git a/eth2deposit/key_handling/key_derivation/mnemonic.py b/eth2deposit/key_handling/key_derivation/mnemonic.py index 3e3b0d1f..c0e61b21 100644 --- a/eth2deposit/key_handling/key_derivation/mnemonic.py +++ b/eth2deposit/key_handling/key_derivation/mnemonic.py @@ -1,5 +1,4 @@ import os -import sys from unicodedata import normalize from secrets import randbits from typing import ( @@ -14,19 +13,9 @@ SHA256, PBKDF2, ) - - -def _resource_path(relative_path: str) -> str: - """ - Get the absolute path to a resource in a manner friendly to PyInstaller. - PyInstaller creates a temp folder and stores path in _MEIPASS which this function swaps - into a resource path so it is avaible both when building binaries and running natively. - """ - try: - base_path = sys._MEIPASS # type: ignore - except Exception: - base_path = os.path.abspath(".") - return os.path.join(base_path, relative_path) +from eth2deposit.utils.file_handling import ( + resource_path, +) def _get_word_list(language: str, path: str) -> Sequence[str]: @@ -35,7 +24,7 @@ def _get_word_list(language: str, path: str) -> Sequence[str]: Ref: https://github.com/bitcoin/bips/blob/master/bip-0039/bip-0039-wordlists.md """ - path = _resource_path(path) + path = resource_path(path) dirty_list = open(os.path.join(path, '%s.txt' % language), encoding='utf-8').readlines() return [word.replace('\n', '') for word in dirty_list] diff --git a/eth2deposit/utils/file_handling.py b/eth2deposit/utils/file_handling.py new file mode 100644 index 00000000..9b6e4c33 --- /dev/null +++ b/eth2deposit/utils/file_handling.py @@ -0,0 +1,16 @@ +import os +import sys + + +def resource_path(relative_path: str) -> str: + """ + Get the absolute path to a resource in a manner friendly to PyInstaller. + PyInstaller creates a temp folder and stores path in _MEIPASS which this function swaps + into a resource path so it is available both when building binaries and running natively. + """ + try: + base_path = sys._MEIPASS # type: ignore + except Exception: + base_path = os.path.abspath(".") + print(base_path, relative_path) + return os.path.join(base_path, relative_path) diff --git a/eth2deposit/utils/intl.py b/eth2deposit/utils/intl.py index 43242d4b..47b84eaf 100644 --- a/eth2deposit/utils/intl.py +++ b/eth2deposit/utils/intl.py @@ -16,6 +16,9 @@ from eth2deposit.utils.constants import ( INTL_CONTENT_PATH, ) +from eth2deposit.utils.file_handling import ( + resource_path, +) from eth2deposit.exceptions import ValidationError @@ -53,6 +56,7 @@ def load_text(params: List[str], file_path: str='', func: str='', lang: str='') file_path_list = os.path.normpath(file_path).split(os.path.sep) rel_path_list = file_path_list[file_path_list.index('eth2deposit') + 1:] json_path = os.path.join(INTL_CONTENT_PATH, lang, *rel_path_list) + json_path = resource_path(json_path) try: # browse json until text is found From 64baba7231f66c00efe12daa20a9bab2464e6edb Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 30 Aug 2021 10:04:27 +0200 Subject: [PATCH 2/5] CI checks binaries in dedicated folder --- .circleci/config.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 27dcac02..c13e126b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -129,6 +129,11 @@ jobs: export PYTHONHASHSEED=42 export CIRCLE_SHORT_SHA1=$(eval echo $CIRCLE_SHA1 | cut -c -7) export BUILD_FILE_NAME=eth2deposit-cli-${CIRCLE_SHORT_SHA1}-linux-amd64; + export TEST_FOLDER_NAME=TMP_TEST_FOLDER + mkdir ${TEST_FOLDER_NAME} + cp ${BUILD_FILE_NAME} ${TEST_FOLDER_NAME} + cp test_binary_script.py ${TEST_FOLDER_NAME} + cd ${TEST_FOLDER_NAME} python test_binary_script.py ./${BUILD_FILE_NAME}; - run: name: Compress the file @@ -166,6 +171,11 @@ jobs: $PYTHONHASHSEED = 42 $CIRCLE_SHORT_SHA1 = $env:CIRCLE_SHA1.substring(0,7) $BUILD_FILE_NAME = "eth2deposit-cli-" + $CIRCLE_SHORT_SHA1 + "-windows-amd64" + $TEST_FOLDER_NAME = "TMP_TEST_FOLDER" + mkdir ${TEST_FOLDER_NAME} + copy ${BUILD_FILE_NAME} ${TEST_FOLDER_NAME} + copy test_binary_script.py ${TEST_FOLDER_NAME} + cd ${TEST_FOLDER_NAME} python test_binary_script.py .\${BUILD_FILE_NAME} - run: name: Compress the file @@ -207,6 +217,11 @@ jobs: export PYTHONHASHSEED=42 export CIRCLE_SHORT_SHA1=$(eval echo $CIRCLE_SHA1 | cut -c -7) export BUILD_FILE_NAME=eth2deposit-cli-${CIRCLE_SHORT_SHA1}-darwin-amd64; + export TEST_FOLDER_NAME=TMP_TEST_FOLDER + mkdir ${TEST_FOLDER_NAME} + cp ${BUILD_FILE_NAME} ${TEST_FOLDER_NAME} + cp test_binary_script.py ${TEST_FOLDER_NAME} + cd ${TEST_FOLDER_NAME} python3 test_binary_script.py ./${BUILD_FILE_NAME}; - run: name: Compress the file From 0f7b8aea9da85a1813323df4debdf836778639b4 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 30 Aug 2021 10:34:59 +0200 Subject: [PATCH 3/5] copy folders with "-r" --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c13e126b..15d17c22 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -131,7 +131,7 @@ jobs: export BUILD_FILE_NAME=eth2deposit-cli-${CIRCLE_SHORT_SHA1}-linux-amd64; export TEST_FOLDER_NAME=TMP_TEST_FOLDER mkdir ${TEST_FOLDER_NAME} - cp ${BUILD_FILE_NAME} ${TEST_FOLDER_NAME} + cp -r ${BUILD_FILE_NAME} ${TEST_FOLDER_NAME} cp test_binary_script.py ${TEST_FOLDER_NAME} cd ${TEST_FOLDER_NAME} python test_binary_script.py ./${BUILD_FILE_NAME}; @@ -176,7 +176,7 @@ jobs: copy ${BUILD_FILE_NAME} ${TEST_FOLDER_NAME} copy test_binary_script.py ${TEST_FOLDER_NAME} cd ${TEST_FOLDER_NAME} - python test_binary_script.py .\${BUILD_FILE_NAME} + python test_binary_script.py ${BUILD_FILE_NAME} - run: name: Compress the file command: | @@ -219,7 +219,7 @@ jobs: export BUILD_FILE_NAME=eth2deposit-cli-${CIRCLE_SHORT_SHA1}-darwin-amd64; export TEST_FOLDER_NAME=TMP_TEST_FOLDER mkdir ${TEST_FOLDER_NAME} - cp ${BUILD_FILE_NAME} ${TEST_FOLDER_NAME} + cp -r ${BUILD_FILE_NAME} ${TEST_FOLDER_NAME} cp test_binary_script.py ${TEST_FOLDER_NAME} cd ${TEST_FOLDER_NAME} python3 test_binary_script.py ./${BUILD_FILE_NAME}; From 24e3fd726a44d36729af57002c8bdf7a83b09a3a Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Mon, 30 Aug 2021 11:05:42 +0200 Subject: [PATCH 4/5] Uses Powershells recursive copy --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 15d17c22..ad7d5538 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -173,7 +173,7 @@ jobs: $BUILD_FILE_NAME = "eth2deposit-cli-" + $CIRCLE_SHORT_SHA1 + "-windows-amd64" $TEST_FOLDER_NAME = "TMP_TEST_FOLDER" mkdir ${TEST_FOLDER_NAME} - copy ${BUILD_FILE_NAME} ${TEST_FOLDER_NAME} + Copy-item ${BUILD_FILE_NAME} -destination ${TEST_FOLDER_NAME} -recurse copy test_binary_script.py ${TEST_FOLDER_NAME} cd ${TEST_FOLDER_NAME} python test_binary_script.py ${BUILD_FILE_NAME} From aba63e813dee521f60c5745bf992058c39ab83f9 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Thu, 2 Sep 2021 15:40:10 +0200 Subject: [PATCH 5/5] Apply suggestions from code review Co-authored-by: Hsiao-Wei Wang --- eth2deposit/utils/file_handling.py | 1 - eth2deposit/utils/intl.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/eth2deposit/utils/file_handling.py b/eth2deposit/utils/file_handling.py index 9b6e4c33..f0142491 100644 --- a/eth2deposit/utils/file_handling.py +++ b/eth2deposit/utils/file_handling.py @@ -12,5 +12,4 @@ def resource_path(relative_path: str) -> str: base_path = sys._MEIPASS # type: ignore except Exception: base_path = os.path.abspath(".") - print(base_path, relative_path) return os.path.join(base_path, relative_path) diff --git a/eth2deposit/utils/intl.py b/eth2deposit/utils/intl.py index 47b84eaf..56baeda7 100644 --- a/eth2deposit/utils/intl.py +++ b/eth2deposit/utils/intl.py @@ -55,8 +55,7 @@ def load_text(params: List[str], file_path: str='', func: str='', lang: str='') # Determine path to json text file_path_list = os.path.normpath(file_path).split(os.path.sep) rel_path_list = file_path_list[file_path_list.index('eth2deposit') + 1:] - json_path = os.path.join(INTL_CONTENT_PATH, lang, *rel_path_list) - json_path = resource_path(json_path) + json_path = resource_path(os.path.join(INTL_CONTENT_PATH, lang, *rel_path_list)) try: # browse json until text is found