From f08eacfc0f91bcff47b6edcf644a52826796e085 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 16 Jul 2020 23:33:28 +0530 Subject: [PATCH 1/8] Expose all .whl, and other files and Create an archive It will upload all .whl,.ABOUT, .NOTICE,.LICENCE files of thirdparty repo to github new repo as an asset of specific OS and python 3 version and also generate archive for each Oses and all python and release them an asset on github. Signed-off-by: Abhishek Kumar --- deps_archive.py | 87 +++++++++++++++++++++++++ github_release.py | 157 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 244 insertions(+) create mode 100644 deps_archive.py create mode 100644 github_release.py diff --git a/deps_archive.py b/deps_archive.py new file mode 100644 index 00000000000..c6a45c507ff --- /dev/null +++ b/deps_archive.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/scancode-toolkit/ +# The ScanCode software is licensed under the Apache License version 2.0. +# Data generated with ScanCode require an acknowledgment. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with ScanCode or any ScanCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# ScanCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode-toolkit/ for support and download. + +from __future__ import absolute_import +from __future__ import print_function + +import argparse +from commoncode.system import on_windows +import os +from shutil import make_archive +import sys + + +def generate_OS_archive(input_dir, output): + """ + Generate an archive for dependencies for specific OS and + given version of python by taking directory as an input. + """ + + root_dir = os.path.expanduser(os.path.join("~", input_dir)) + output_dir = os.path.expanduser(os.path.join("~", output)) + if on_windows: + make_archive(output_dir, "zip", root_dir) + else: + make_archive(output_dir, "gztar", root_dir) + + +def main_with_args(args: str) -> None: + parser = argparse.ArgumentParser( + description="""Creates a archive for specific OS and specific python. +EXAMPLE: +generate_OS_archive.py \\ + --input scancode-toolkit/thirdparty \\ + --output Desktop/macOS_py36 \\ +""", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + parser.add_argument( + "--input", + help="Required: Thirdparty Dependencies directory to be archived. ", + type=str, + required=True, + ) + + parser.add_argument( + "--output", + help="Required: The Generated archive file name without extension. ", + type=str, + required=True, + ) + + args = parser.parse_args() + + input_dir = args.input + output = args.output + generate_OS_archive(input_dir, output) + + +def main() -> None: + main_with_args(sys.argv[1:]) + + +if __name__ == "__main__": + main() diff --git a/github_release.py b/github_release.py new file mode 100644 index 00000000000..7008b1719b6 --- /dev/null +++ b/github_release.py @@ -0,0 +1,157 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/scancode-toolkit/ +# The ScanCode software is licensed under the Apache License version 2.0. +# Data generated with ScanCode require an acknowledgment. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with ScanCode or any ScanCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# ScanCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode-toolkit/ for support and download. + +from __future__ import absolute_import +from __future__ import print_function + +import argparse +from commoncode.fileutils import resource_iter +import fnmatch +import os +import subprocess +import sys + + +def release_asset(token, tag, repo, body, user, limit, file): + """ + Release .whl,.ABOUT,.NOTICE,.LICENSE to github repository from + given directory. + """ + + cwd = os.getcwd() + os.environ["GITHUB_TOKEN"] = token + thirdparty = list(resource_iter(file, with_dirs=False)) + # FIXME this code is for py 3.6 and later we will update for all version + dependencies = [ + files + for files in thirdparty + if fnmatch.fnmatchcase(files, "*py3*") + or fnmatch.fnmatchcase(files, "*cp36*") + or ( + fnmatch.fnmatchcase(files, "*tar.gz*") + and not fnmatch.fnmatchcase(files, "*py2-ipaddress-3.4.1.tar.gz*") + ) + ] + for deps in dependencies: + print(deps) + subprocess.run( + [ + "python3", + "-m", + "github_release_retry.github_release_retry", + "--user", + user, + "--repo", + repo, + "--tag_name", + tag, + "--body_string", + body, + "--retry_limit", + limit, + deps, + ] + ) + + +def main_with_args(args: str) -> None: + parser = argparse.ArgumentParser( + description="""Creates a GitHub release (if it does not already exist) and uploads files to the release. +Please pass the GITHUB_TOKEN as an argument. +EXAMPLE: +github-release-asset \\ + --user Abhishek-Dev09 \\ + --repo thirdparty \\ + --tag_name v1.0 \\ + --body_string "Python 3.6 wheels" \\ + hello-world.zip RELEASE_NOTES.txt +""", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + parser.add_argument( + "--user", + help="Required: The GitHub username or organization name in which the repo resides. ", + type=str, + required=True, + ) + + parser.add_argument( + "--token", + help="Required: The Github token is required to acess the repository where you want to upload . ", + type=str, + required=True, + ) + + parser.add_argument( + "--repo", + help="Required: The GitHub repo name in which to make the release. ", + type=str, + required=True, + ) + + parser.add_argument( + "--tag_name", + help="Required: The name of the tag to create or use. ", + type=str, + required=True, + ) + + parser.add_argument( + "--body_string", + help="Required (or use --body_file): Text describing the release. Ignored if the release already exists.", + type=str, + ) + + parser.add_argument( + "--retry_limit", + help="The number of times to retry creating/getting the release and/or uploading each file. ", + type=str, + default="10", + ) + + parser.add_argument( + "--files", type=str, help="The files to upload to the release.", + ) + + args = parser.parse_args() + + token = args.token + tag = args.tag_name + repo = args.repo + body = args.body_string + user = args.user + limit = args.retry_limit + file = args.files + + release_asset(token, tag, repo, body, user, limit, file) + + +def main() -> None: + main_with_args(sys.argv[1:]) + + +if __name__ == "__main__": + main() From 89e957e7b19a79d164ca6cd114ec9d50936ed0e0 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 21 Jul 2020 19:41:45 +0530 Subject: [PATCH 2/8] Move all script to etc/scripts Signed-off-by: Abhishek Kumar --- .../scripts/deps_archive.py | 8 +++--- .../scripts/github_release.py | 26 +++++++++---------- 2 files changed, 17 insertions(+), 17 deletions(-) rename deps_archive.py => etc/scripts/deps_archive.py (96%) rename github_release.py => etc/scripts/github_release.py (88%) diff --git a/deps_archive.py b/etc/scripts/deps_archive.py similarity index 96% rename from deps_archive.py rename to etc/scripts/deps_archive.py index c6a45c507ff..7cf1ca7c39e 100644 --- a/deps_archive.py +++ b/etc/scripts/deps_archive.py @@ -33,7 +33,7 @@ import sys -def generate_OS_archive(input_dir, output): +def generate_os_archive(input_dir, output): """ Generate an archive for dependencies for specific OS and given version of python by taking directory as an input. @@ -60,14 +60,14 @@ def main_with_args(args: str) -> None: parser.add_argument( "--input", - help="Required: Thirdparty Dependencies directory to be archived. ", + help="Required: Thirdparty Dependencies directory to be archived.", type=str, required=True, ) parser.add_argument( "--output", - help="Required: The Generated archive file name without extension. ", + help="Required: The Generated archive file name without extension.", type=str, required=True, ) @@ -76,7 +76,7 @@ def main_with_args(args: str) -> None: input_dir = args.input output = args.output - generate_OS_archive(input_dir, output) + generate_os_archive(input_dir, output) def main() -> None: diff --git a/github_release.py b/etc/scripts/github_release.py similarity index 88% rename from github_release.py rename to etc/scripts/github_release.py index 7008b1719b6..66938e62a91 100644 --- a/github_release.py +++ b/etc/scripts/github_release.py @@ -33,29 +33,29 @@ import subprocess import sys +python_version = str(sys.version_info[0]) + str(sys.version_info[1]) +py_abi = "{0}cp{1}{0}".format("*", python_version) -def release_asset(token, tag, repo, body, user, limit, file): + +def release_asset(token, tag, repo, body, user, limit, asset_dir): """ Release .whl,.ABOUT,.NOTICE,.LICENSE to github repository from given directory. """ - cwd = os.getcwd() os.environ["GITHUB_TOKEN"] = token - thirdparty = list(resource_iter(file, with_dirs=False)) - # FIXME this code is for py 3.6 and later we will update for all version + thirdparty = list(resource_iter(asset_dir, with_dirs=False)) dependencies = [ files for files in thirdparty if fnmatch.fnmatchcase(files, "*py3*") - or fnmatch.fnmatchcase(files, "*cp36*") + or fnmatch.fnmatchcase(files, py_abi) or ( fnmatch.fnmatchcase(files, "*tar.gz*") and not fnmatch.fnmatchcase(files, "*py2-ipaddress-3.4.1.tar.gz*") ) ] for deps in dependencies: - print(deps) subprocess.run( [ "python3", @@ -93,28 +93,28 @@ def main_with_args(args: str) -> None: parser.add_argument( "--user", - help="Required: The GitHub username or organization name in which the repo resides. ", + help="Required: The GitHub username or organization name in which the repo resides.", type=str, required=True, ) parser.add_argument( "--token", - help="Required: The Github token is required to acess the repository where you want to upload . ", + help="Required: The Github token is required to acess the repository where you want to upload.", type=str, required=True, ) parser.add_argument( "--repo", - help="Required: The GitHub repo name in which to make the release. ", + help="Required: The GitHub repo name in which to make the release.", type=str, required=True, ) parser.add_argument( "--tag_name", - help="Required: The name of the tag to create or use. ", + help="Required: The name of the tag to create or use.", type=str, required=True, ) @@ -127,7 +127,7 @@ def main_with_args(args: str) -> None: parser.add_argument( "--retry_limit", - help="The number of times to retry creating/getting the release and/or uploading each file. ", + help="The number of times to retry creating/getting the release and/or uploading each file.", type=str, default="10", ) @@ -144,9 +144,9 @@ def main_with_args(args: str) -> None: body = args.body_string user = args.user limit = args.retry_limit - file = args.files + deps_dir = args.files - release_asset(token, tag, repo, body, user, limit, file) + release_asset(token, tag, repo, body, user, limit, deps_dir) def main() -> None: From 90bff2b98fe476ec86af623b5f4c4a08afd47917 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 23 Jul 2020 11:00:32 +0530 Subject: [PATCH 3/8] Use abspath instead of expanduser Signed-off-by: Abhishek Kumar --- etc/scripts/deps_archive.py | 50 +++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/etc/scripts/deps_archive.py b/etc/scripts/deps_archive.py index 7cf1ca7c39e..8bb3e21fa92 100644 --- a/etc/scripts/deps_archive.py +++ b/etc/scripts/deps_archive.py @@ -30,17 +30,32 @@ from commoncode.system import on_windows import os from shutil import make_archive +import subprocess import sys -def generate_os_archive(input_dir, output): +def generate_os_archive(links, requirement, output_file): """ Generate an archive for dependencies for specific OS and given version of python by taking directory as an input. """ - - root_dir = os.path.expanduser(os.path.join("~", input_dir)) - output_dir = os.path.expanduser(os.path.join("~", output)) + subprocess.run( + [ + "pip", + "download", + "--verbose", + "--no-cache-dir", + "--no-index", + "--find-links", + links, + "-r", + requirement, + "--dest", + "my_deps", + ] + ) + root_dir = os.path.abspath("my_deps") + output_dir = os.path.abspath(output_file) if on_windows: make_archive(output_dir, "zip", root_dir) else: @@ -51,22 +66,30 @@ def main_with_args(args: str) -> None: parser = argparse.ArgumentParser( description="""Creates a archive for specific OS and specific python. EXAMPLE: -generate_OS_archive.py \\ - --input scancode-toolkit/thirdparty \\ - --output Desktop/macOS_py36 \\ +deps_archive.py \\ + --input thirdparty \\ + --r requirements.txt + --output_file macOS_py36 \\ """, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( - "--input", - help="Required: Thirdparty Dependencies directory to be archived.", + "--find_links", + help="Required: A url or path to an html file, then parse for links to archives. If a local path or file://url that's a directory, then look for archives in the directory listing", type=str, required=True, ) parser.add_argument( - "--output", + "--req", + help="A requirement_file with hashes", + type=str, + default="requirements.txt", + ) + + parser.add_argument( + "--output_file", help="Required: The Generated archive file name without extension.", type=str, required=True, @@ -74,9 +97,10 @@ def main_with_args(args: str) -> None: args = parser.parse_args() - input_dir = args.input - output = args.output - generate_os_archive(input_dir, output) + link = args.find_links + req = args.req + output = args.output_file + generate_os_archive(link, req, output) def main() -> None: From 2e80d159ddef7eba516d53247a177c9a0d0e71e9 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sat, 25 Jul 2020 11:17:17 +0530 Subject: [PATCH 4/8] Format the script properly Signed-off-by: Abhishek Kumar --- etc/scripts/deps_archive.py | 47 ++++++++++--------- etc/scripts/github_release.py | 85 ++++++++++++++++++----------------- 2 files changed, 66 insertions(+), 66 deletions(-) diff --git a/etc/scripts/deps_archive.py b/etc/scripts/deps_archive.py index 8bb3e21fa92..58006fa9ad5 100644 --- a/etc/scripts/deps_archive.py +++ b/etc/scripts/deps_archive.py @@ -30,7 +30,7 @@ from commoncode.system import on_windows import os from shutil import make_archive -import subprocess +from subprocess import run import sys @@ -39,27 +39,26 @@ def generate_os_archive(links, requirement, output_file): Generate an archive for dependencies for specific OS and given version of python by taking directory as an input. """ - subprocess.run( - [ - "pip", - "download", - "--verbose", - "--no-cache-dir", - "--no-index", - "--find-links", + pip_agrs =[ + 'pip', + 'download', + '--verbose', + '--no-cache-dir', + '--no-index', + '--find-links', links, - "-r", + '-r', requirement, - "--dest", - "my_deps", + '--dest', + 'my_deps', ] - ) - root_dir = os.path.abspath("my_deps") + run(pip_agrs) + root_dir = os.path.abspath('my_deps') output_dir = os.path.abspath(output_file) if on_windows: - make_archive(output_dir, "zip", root_dir) + make_archive(output_dir, 'zip', root_dir) else: - make_archive(output_dir, "gztar", root_dir) + make_archive(output_dir, 'gztar', root_dir) def main_with_args(args: str) -> None: @@ -68,29 +67,29 @@ def main_with_args(args: str) -> None: EXAMPLE: deps_archive.py \\ --input thirdparty \\ - --r requirements.txt + --req requirements.txt --output_file macOS_py36 \\ """, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( - "--find_links", + '--find_links', help="Required: A url or path to an html file, then parse for links to archives. If a local path or file://url that's a directory, then look for archives in the directory listing", type=str, required=True, ) parser.add_argument( - "--req", - help="A requirement_file with hashes", + '--req', + help='A requirement_file with hashes', type=str, - default="requirements.txt", + required=True, ) parser.add_argument( - "--output_file", - help="Required: The Generated archive file name without extension.", + '--output_file', + help='Required: The Generated archive file name without extension.', type=str, required=True, ) @@ -107,5 +106,5 @@ def main() -> None: main_with_args(sys.argv[1:]) -if __name__ == "__main__": +if __name__ == '__main__': main() diff --git a/etc/scripts/github_release.py b/etc/scripts/github_release.py index 66938e62a91..e540cf5af0a 100644 --- a/etc/scripts/github_release.py +++ b/etc/scripts/github_release.py @@ -28,13 +28,13 @@ import argparse from commoncode.fileutils import resource_iter -import fnmatch +from fnmatch import fnmatchcase import os -import subprocess +from subprocess import run import sys python_version = str(sys.version_info[0]) + str(sys.version_info[1]) -py_abi = "{0}cp{1}{0}".format("*", python_version) +py_abi = '{0}cp{1}{0}'.format('*', python_version) def release_asset(token, tag, repo, body, user, limit, asset_dir): @@ -43,37 +43,36 @@ def release_asset(token, tag, repo, body, user, limit, asset_dir): given directory. """ - os.environ["GITHUB_TOKEN"] = token + os.environ['GITHUB_TOKEN'] = token thirdparty = list(resource_iter(asset_dir, with_dirs=False)) dependencies = [ files for files in thirdparty - if fnmatch.fnmatchcase(files, "*py3*") - or fnmatch.fnmatchcase(files, py_abi) + if fnmatchcase(files, '*py3*') + or fnmatchcase(files, py_abi) or ( - fnmatch.fnmatchcase(files, "*tar.gz*") - and not fnmatch.fnmatchcase(files, "*py2-ipaddress-3.4.1.tar.gz*") + fnmatchcase(files, '*tar.gz*') + and not fnmatchcase(files, '*py2-ipaddress-3.4.1.tar.gz*') ) ] for deps in dependencies: - subprocess.run( - [ - "python3", - "-m", - "github_release_retry.github_release_retry", - "--user", - user, - "--repo", - repo, - "--tag_name", - tag, - "--body_string", - body, - "--retry_limit", - limit, - deps, - ] - ) + github_args = [ + 'python3', + '-m', + 'github_release_retry.github_release_retry', + '--user', + user, + '--repo', + repo, + '--tag_name', + tag, + '--body_string', + body, + '--retry_limit', + limit, + deps, + ] + run(github_args) def main_with_args(args: str) -> None: @@ -81,8 +80,9 @@ def main_with_args(args: str) -> None: description="""Creates a GitHub release (if it does not already exist) and uploads files to the release. Please pass the GITHUB_TOKEN as an argument. EXAMPLE: -github-release-asset \\ +github-release.py \\ --user Abhishek-Dev09 \\ + --token XXXXXXXXXXXXX \\ --repo thirdparty \\ --tag_name v1.0 \\ --body_string "Python 3.6 wheels" \\ @@ -92,48 +92,49 @@ def main_with_args(args: str) -> None: ) parser.add_argument( - "--user", - help="Required: The GitHub username or organization name in which the repo resides.", + '--user', + help='Required: The GitHub username or organization name in which the repo resides.', type=str, required=True, ) parser.add_argument( - "--token", - help="Required: The Github token is required to acess the repository where you want to upload.", + '--token', + help='Required: The Github token is required to acess the repository where you want to upload.', type=str, required=True, ) parser.add_argument( - "--repo", - help="Required: The GitHub repo name in which to make the release.", + '--repo', + help='Required: The GitHub repo name in which to make the release.', type=str, required=True, ) parser.add_argument( - "--tag_name", - help="Required: The name of the tag to create or use.", + '--tag_name', + help='Required: The name of the tag to create or use.', type=str, required=True, ) parser.add_argument( - "--body_string", - help="Required (or use --body_file): Text describing the release. Ignored if the release already exists.", + '--body_string', + help='Required : Text describing the release. Ignored if the release already exists.', type=str, + required=True, ) parser.add_argument( - "--retry_limit", - help="The number of times to retry creating/getting the release and/or uploading each file.", + '--retry_limit', + help='The number of times to retry creating/getting the release and/or uploading each file.', type=str, - default="10", + default='10', ) parser.add_argument( - "--files", type=str, help="The files to upload to the release.", + '--files', type=str, help='The files to upload to the release.', ) args = parser.parse_args() @@ -153,5 +154,5 @@ def main() -> None: main_with_args(sys.argv[1:]) -if __name__ == "__main__": +if __name__ == '__main__': main() From 393fad44eafd8da601971939868702266cae2e92 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sun, 26 Jul 2020 10:36:48 +0530 Subject: [PATCH 5/8] Sort imports,remove examples,use proper variable name Signed-off-by: Abhishek Kumar --- etc/scripts/deps_archive.py | 27 +++++++++------------ etc/scripts/github_release.py | 45 ++++++++++++++++------------------- 2 files changed, 32 insertions(+), 40 deletions(-) diff --git a/etc/scripts/deps_archive.py b/etc/scripts/deps_archive.py index 58006fa9ad5..6c03a696ec4 100644 --- a/etc/scripts/deps_archive.py +++ b/etc/scripts/deps_archive.py @@ -27,17 +27,17 @@ from __future__ import print_function import argparse -from commoncode.system import on_windows import os from shutil import make_archive from subprocess import run import sys +from commoncode.system import on_windows def generate_os_archive(links, requirement, output_file): """ - Generate an archive for dependencies for specific OS and - given version of python by taking directory as an input. + Generate an archive as an outtput_file for specific OS + and python by taking links, requirement_file as an input. """ pip_agrs =[ 'pip', @@ -64,31 +64,26 @@ def generate_os_archive(links, requirement, output_file): def main_with_args(args: str) -> None: parser = argparse.ArgumentParser( description="""Creates a archive for specific OS and specific python. -EXAMPLE: -deps_archive.py \\ - --input thirdparty \\ - --req requirements.txt - --output_file macOS_py36 \\ """, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( - '--find_links', + '--find-links', help="Required: A url or path to an html file, then parse for links to archives. If a local path or file://url that's a directory, then look for archives in the directory listing", type=str, required=True, ) parser.add_argument( - '--req', - help='A requirement_file with hashes', + '--requirement', + help='Required: A requirement_file with hashes', type=str, required=True, ) parser.add_argument( - '--output_file', + '--output-filename', help='Required: The Generated archive file name without extension.', type=str, required=True, @@ -96,10 +91,10 @@ def main_with_args(args: str) -> None: args = parser.parse_args() - link = args.find_links - req = args.req - output = args.output_file - generate_os_archive(link, req, output) + find_links = args.find_links + requirement = args.requirement + output_filename = args.output_filename + generate_os_archive(find_links, requirement, output_filename) def main() -> None: diff --git a/etc/scripts/github_release.py b/etc/scripts/github_release.py index e540cf5af0a..239183f1187 100644 --- a/etc/scripts/github_release.py +++ b/etc/scripts/github_release.py @@ -27,20 +27,22 @@ from __future__ import print_function import argparse -from commoncode.fileutils import resource_iter from fnmatch import fnmatchcase import os from subprocess import run import sys +from commoncode.fileutils import resource_iter + python_version = str(sys.version_info[0]) + str(sys.version_info[1]) py_abi = '{0}cp{1}{0}'.format('*', python_version) -def release_asset(token, tag, repo, body, user, limit, asset_dir): +def release_asset(token, tag, repo, body_string, user, retry_limit, asset_dir): """ - Release .whl,.ABOUT,.NOTICE,.LICENSE to github repository from - given directory. + Release .whl,.ABOUT,.NOTICE,.LICENSE to github repository(repo) from asset_directory. + It takes user and token as credential and tag_name,body_string for description of + release. By default retry_limit is 10. """ os.environ['GITHUB_TOKEN'] = token @@ -67,10 +69,10 @@ def release_asset(token, tag, repo, body, user, limit, asset_dir): '--tag_name', tag, '--body_string', - body, + body_string, '--retry_limit', - limit, - deps, + retry_limit, + deps ] run(github_args) @@ -79,14 +81,6 @@ def main_with_args(args: str) -> None: parser = argparse.ArgumentParser( description="""Creates a GitHub release (if it does not already exist) and uploads files to the release. Please pass the GITHUB_TOKEN as an argument. -EXAMPLE: -github-release.py \\ - --user Abhishek-Dev09 \\ - --token XXXXXXXXXXXXX \\ - --repo thirdparty \\ - --tag_name v1.0 \\ - --body_string "Python 3.6 wheels" \\ - hello-world.zip RELEASE_NOTES.txt """, formatter_class=argparse.RawDescriptionHelpFormatter, ) @@ -113,41 +107,44 @@ def main_with_args(args: str) -> None: ) parser.add_argument( - '--tag_name', + '--tag-name', help='Required: The name of the tag to create or use.', type=str, required=True, ) parser.add_argument( - '--body_string', + '--body-string', help='Required : Text describing the release. Ignored if the release already exists.', type=str, required=True, ) parser.add_argument( - '--retry_limit', + '--retry-limit', help='The number of times to retry creating/getting the release and/or uploading each file.', type=str, default='10', ) parser.add_argument( - '--files', type=str, help='The files to upload to the release.', + '--directory', + help='Required: The directory that contains files to upload to the release.', + type=str, + required=True, ) args = parser.parse_args() token = args.token - tag = args.tag_name + tag_name = args.tag_name repo = args.repo - body = args.body_string + body_string = args.body_string user = args.user - limit = args.retry_limit - deps_dir = args.files + retry_limit = args.retry_limit + directory = args.directory - release_asset(token, tag, repo, body, user, limit, deps_dir) + release_asset(token, tag_name, repo, body_string, user, retry_limit, directory) def main() -> None: From d0e138623202820b52e07d842214bea89ee1cfd7 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sun, 26 Jul 2020 19:31:50 +0530 Subject: [PATCH 6/8] Improve the description and documentation of function Signed-off-by: Abhishek Kumar --- etc/scripts/deps_archive.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/etc/scripts/deps_archive.py b/etc/scripts/deps_archive.py index 6c03a696ec4..c37717104c2 100644 --- a/etc/scripts/deps_archive.py +++ b/etc/scripts/deps_archive.py @@ -34,10 +34,11 @@ from commoncode.system import on_windows -def generate_os_archive(links, requirement, output_file): +def generate_os_archive(links, requirement, archive_name): """ - Generate an archive as an outtput_file for specific OS - and python by taking links, requirement_file as an input. + Generate an archive as `archive_name.tar.gz` and `archive_name` directory + that contains wheels and sdist for specific OS and python by taking links, + requirement as an input. """ pip_agrs =[ 'pip', @@ -50,11 +51,11 @@ def generate_os_archive(links, requirement, output_file): '-r', requirement, '--dest', - 'my_deps', + archive_name, ] run(pip_agrs) - root_dir = os.path.abspath('my_deps') - output_dir = os.path.abspath(output_file) + root_dir = os.path.abspath(archive_name) + output_dir = os.path.abspath(archive_name) if on_windows: make_archive(output_dir, 'zip', root_dir) else: @@ -63,7 +64,9 @@ def generate_os_archive(links, requirement, output_file): def main_with_args(args: str) -> None: parser = argparse.ArgumentParser( - description="""Creates a archive for specific OS and specific python. + description="""Generate an archive as `archive_name.tar.gz` and `archive_name` directory +that contains wheels and sdist for specific OS and python by taking links, +requirement as an input. """, formatter_class=argparse.RawDescriptionHelpFormatter, ) @@ -83,7 +86,7 @@ def main_with_args(args: str) -> None: ) parser.add_argument( - '--output-filename', + '--archive-name', help='Required: The Generated archive file name without extension.', type=str, required=True, @@ -93,8 +96,8 @@ def main_with_args(args: str) -> None: find_links = args.find_links requirement = args.requirement - output_filename = args.output_filename - generate_os_archive(find_links, requirement, output_filename) + archive_name = args.archive_name + generate_os_archive(find_links, requirement, archive_name) def main() -> None: From 27c4e86d4d8204126867885624f8feda4c389333 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 26 Jul 2020 17:16:59 +0200 Subject: [PATCH 7/8] Improve help texts Signed-off-by: Philippe Ombredanne --- etc/scripts/deps_archive.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/etc/scripts/deps_archive.py b/etc/scripts/deps_archive.py index c37717104c2..037428b2bb5 100644 --- a/etc/scripts/deps_archive.py +++ b/etc/scripts/deps_archive.py @@ -65,29 +65,26 @@ def generate_os_archive(links, requirement, archive_name): def main_with_args(args: str) -> None: parser = argparse.ArgumentParser( description="""Generate an archive as `archive_name.tar.gz` and `archive_name` directory -that contains wheels and sdist for specific OS and python by taking links, -requirement as an input. -""", - formatter_class=argparse.RawDescriptionHelpFormatter, + that contains wheels and sdist for specific OS and python.""", ) parser.add_argument( '--find-links', - help="Required: A url or path to an html file, then parse for links to archives. If a local path or file://url that's a directory, then look for archives in the directory listing", + help="A directory or URL where to find packages. See pip help for details.", type=str, required=True, ) parser.add_argument( '--requirement', - help='Required: A requirement_file with hashes', + help='An existing requirement file (with hashes) listing required packages.', type=str, required=True, ) parser.add_argument( '--archive-name', - help='Required: The Generated archive file name without extension.', + help='Path to the archive file base name to create (without extension).', type=str, required=True, ) From c062d214c6bbdd3af543243aa6cbabac1c955b8b Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 26 Jul 2020 23:25:22 +0200 Subject: [PATCH 8/8] Improve release creation and upload #2072 Do not spawn a process but call a function instead. Clarify variable and options names and their help text. Use access token from a cli option or an environment variable. Signed-off-by: Philippe Ombredanne --- etc/scripts/github_release.py | 161 ++++++++++++++++++---------------- 1 file changed, 86 insertions(+), 75 deletions(-) diff --git a/etc/scripts/github_release.py b/etc/scripts/github_release.py index 239183f1187..49dfd5fa8fb 100644 --- a/etc/scripts/github_release.py +++ b/etc/scripts/github_release.py @@ -27,127 +27,138 @@ from __future__ import print_function import argparse -from fnmatch import fnmatchcase +from pathlib import Path import os -from subprocess import run import sys +from github_release_retry import github_release_retry as grr + from commoncode.fileutils import resource_iter -python_version = str(sys.version_info[0]) + str(sys.version_info[1]) -py_abi = '{0}cp{1}{0}'.format('*', python_version) + +""" +Create GitHUb releases and upload files there. +This depends on the `github_release_retry` utility +https://github.com/google/github-release-retry +""" -def release_asset(token, tag, repo, body_string, user, retry_limit, asset_dir): +def create_or_update_release_and_upload_directory( + user, + repo, + tag_name, + token, + directory, + retry_limit=10, + description=None +): """ - Release .whl,.ABOUT,.NOTICE,.LICENSE to github repository(repo) from asset_directory. - It takes user and token as credential and tag_name,body_string for description of - release. By default retry_limit is 10. + Create or update a GitHub release at https://github.com// for + `tag_name` tag using the optional `description` for this release. + Use the provided `token` as a GitHub token for API calls authentication. + Upload all files found in the `directory` tree to that GitHub release. + Retry API calls up to `retry_limit` time to work around instability the + GitHub API. """ - os.environ['GITHUB_TOKEN'] = token - thirdparty = list(resource_iter(asset_dir, with_dirs=False)) - dependencies = [ - files - for files in thirdparty - if fnmatchcase(files, '*py3*') - or fnmatchcase(files, py_abi) - or ( - fnmatchcase(files, '*tar.gz*') - and not fnmatchcase(files, '*py2-ipaddress-3.4.1.tar.gz*') - ) - ] - for deps in dependencies: - github_args = [ - 'python3', - '-m', - 'github_release_retry.github_release_retry', - '--user', - user, - '--repo', - repo, - '--tag_name', - tag, - '--body_string', - body_string, - '--retry_limit', - retry_limit, - deps - ] - run(github_args) - - -def main_with_args(args: str) -> None: + api = grr.GithubApi( + github_api_url='https://api.github.com', + user=user, + repo=repo, + token=token, + retry_limit=retry_limit, + ) + release = grr.Release(tag_name=tag_name, body=description) + files = [Path(r) for r in resource_iter(directory, with_dirs=False)] + grr.make_release(api, release, files) + + +def main_with_args(args): parser = argparse.ArgumentParser( - description="""Creates a GitHub release (if it does not already exist) and uploads files to the release. -Please pass the GITHUB_TOKEN as an argument. -""", - formatter_class=argparse.RawDescriptionHelpFormatter, + description=( + 'Create (or update) a GitHub release and upload all the ' + 'files of DIRECTORY to that release.' + ), ) parser.add_argument( '--user', - help='Required: The GitHub username or organization name in which the repo resides.', + help='The GitHub username or organization in which the repository resides.', type=str, required=True, ) parser.add_argument( - '--token', - help='Required: The Github token is required to acess the repository where you want to upload.', + '--repo', + help=' The GitHub repository name in which to create the release.', type=str, required=True, ) parser.add_argument( - '--repo', - help='Required: The GitHub repo name in which to make the release.', + '--tag-name', + help='The name of the tag to create (or re-use) for this release.', type=str, required=True, ) parser.add_argument( - '--tag-name', - help='Required: The name of the tag to create or use.', - type=str, - required=True, + '--directory', + help='The directory that contains files to upload to the release.', + type=str, + required=True, ) + TOKEN_HELP = ( + 'The Github personal acess token is used to authenticate API calls. ' + 'Required unless you set the GITHUB_TOKEN environment variable as an alternative. ' + 'See for details: https://github.com/settings/tokens and ' + 'https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token' + ) + parser.add_argument( - '--body-string', - help='Required : Text describing the release. Ignored if the release already exists.', + '--token', + help=TOKEN_HELP, type=str, - required=True, + required=False, ) parser.add_argument( - '--retry-limit', - help='The number of times to retry creating/getting the release and/or uploading each file.', + '--description', + help='Text description for the release. Ignored if the release exists.', type=str, - default='10', + required=False, ) parser.add_argument( - '--directory', - help='Required: The directory that contains files to upload to the release.', - type=str, - required=True, + '--retry_limit', + help=( + 'Number of retries when making failing GitHub API calls. ' + 'Retrying helps work around transient failures of the GitHub API.' + ), + type=int, + default=10, ) args = parser.parse_args() - - token = args.token - tag_name = args.tag_name - repo = args.repo - body_string = args.body_string - user = args.user - retry_limit = args.retry_limit - directory = args.directory - - release_asset(token, tag_name, repo, body_string, user, retry_limit, directory) + token = args.token or os.environ.get('GITHUB_TOKEN', None) + if not token: + print('--token required option is missing.') + print(TOKEN_HELP) + sys.exit(1) + + create_or_update_release_and_upload_directory( + user=args.user, + repo=args.repo, + tag_name=args.tag_name, + description=args.description, + retry_limit=args.retry_limit, + token=token, + directory=args.directory, + ) -def main() -> None: +def main(): main_with_args(sys.argv[1:])