From 6f8123420c739c9e79ee4ee2861f4b518e813cc5 Mon Sep 17 00:00:00 2001 From: Jesse Marks <32715488+jaamarks@users.noreply.github.com> Date: Mon, 26 Feb 2024 10:50:46 -0500 Subject: [PATCH 01/27] Enable workflow for pull requests in dockerimage.yml --- .github/workflows/dockerimage.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index 5d7efe7..cd02117 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -9,6 +9,14 @@ on: - '.gitignore' - 'README.md' - '*/*/README.md' + pull_request: + branches: + - master + paths-ignore: + - '.github/**' + - '.gitignore' + - 'README.md' + - '*/*/README.md' jobs: build: From 79b0ab5f73b7690e998592962de591e281b17abf Mon Sep 17 00:00:00 2001 From: Jesse Marks <32715488+jaamarks@users.noreply.github.com> Date: Mon, 26 Feb 2024 11:35:15 -0500 Subject: [PATCH 02/27] rtibiocloud -> jessemarks --- .github/workflows/dockerimage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index cd02117..5ce21b1 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -33,7 +33,7 @@ jobs: - name: Build, Tag, Publish Docker uses: ./.github/actions/build-image with: - organization: rtibiocloud + organization: jessemarks changed_files: ${{ steps.getfile.outputs.files }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} From 030bdd1245dbe4f3189ac529db9af8f5b1cbb735 Mon Sep 17 00:00:00 2001 From: jaamarks Date: Mon, 26 Feb 2024 12:01:28 -0500 Subject: [PATCH 03/27] create a new tool to test our github actions --- .../v1/Dockerfile | 29 +++ s3_presigned_url_generator_jesse/v1/README.md | 179 ++++++++++++++++++ .../v1/requirements.txt | 1 + .../v1/s3_presigned_upload.py | 99 ++++++++++ 4 files changed, 308 insertions(+) create mode 100644 s3_presigned_url_generator_jesse/v1/Dockerfile create mode 100644 s3_presigned_url_generator_jesse/v1/README.md create mode 100644 s3_presigned_url_generator_jesse/v1/requirements.txt create mode 100644 s3_presigned_url_generator_jesse/v1/s3_presigned_upload.py diff --git a/s3_presigned_url_generator_jesse/v1/Dockerfile b/s3_presigned_url_generator_jesse/v1/Dockerfile new file mode 100644 index 0000000..3d928b3 --- /dev/null +++ b/s3_presigned_url_generator_jesse/v1/Dockerfile @@ -0,0 +1,29 @@ +# Use an official Python runtime as the base image +FROM python:3.12-alpine + +# Add Container Labels +LABEL maintainer="Jesse Marks " +LABEL description="A script to generate presigned URLs to upload to S3." +LABEL base-image="python:3.12-alpine" + +# Install System Dependencies +RUN apt-get update && apt-get install -y \ + vim \ + less \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Set the working directory in the container +WORKDIR /opt/ + +# Copy the script and requirements file to the container +COPY s3_presigned_upload.py requirements.txt ./ + +# Install the required dependencies +RUN pip install --no-cache-dir -r requirements.txt + +# Set the entry point command +ENTRYPOINT ["python", "s3_presigned_upload.py"] + +# Set the default command arguments +CMD ["--help"] diff --git a/s3_presigned_url_generator_jesse/v1/README.md b/s3_presigned_url_generator_jesse/v1/README.md new file mode 100644 index 0000000..d0fb2e5 --- /dev/null +++ b/s3_presigned_url_generator_jesse/v1/README.md @@ -0,0 +1,179 @@ +# S3 Presigned URL Generator + +A command-line interface (CLI) tool to generate a bash script containing `curl` commands with presigned URLs for uploading files to Amazon S3. This tool enables external collaborators to upload their files to your S3 bucket securely using presigned URLs, eliminating the need for separate AWS accounts. + +
+ + +[Click here to go to the recommended docker usage example](#docker-anchor) + +
+ + + + + + +## Usage + +```shell +python s3_presigned_upload.py \ + --infile \ + --outfile \ + --bucket \ + --key-prefix \ + --expiration-days \ + --aws-access-key \ + --aws-secret-access-key +``` + + +Replace the following placeholders with the appropriate values: + +- ``: Path to the input file containing a list of file names to generate presigned URLs for. +- ``: Path to the output bash script file that will contain the generated curl commands. +- ``: Name of the S3 bucket where the files will be uploaded. +- ``: Prefix to be prepended to each file name as the S3 object key. +- ``: Expiration duration in days for the generated presigned URLs. +- ``: AWS access key ID for authentication. +- ``: AWS secret access key for authentication. + +* _Note_: you can typically find your access keys in your AWS CLI Configuration Files (`~/.aws/credentials`) + +Example: + +Let's assume you have an input file named `file_list.txt` containing the following filenames: + +``` +file1.txt +file2.jpg +file3.pdf +``` + +You want to generate a bash script named `upload_script.sh` that will contain the curl commands with presigned URLs for uploading these files to the S3 bucket `my-bucket` with the key prefix `uploads/` and a URL expiration of 7 days. + +You can execute the script as follows: + +```shell +python s3_presigned_upload.py \ + --infile file_list.txt \ + --outfile upload_script.sh \ + --bucket my-bucket \ + --key-prefix uploads/ \ + --expiration-days 7 \ + --aws-access-key YOUR_ACCESS_KEY \ + --aws-secret-access-key YOUR_SECRET_ACCESS_KEY +``` + +The generated `upload_script.sh` will contain the curl commands necessary to upload the files using presigned URLs. Share the `upload_script.sh` with the external collaborators, and they can execute it in the same folder as their files to upload them to your S3 account. + +
+ + + + + + +## Docker usage +**This is the recommended approach.**
+Here is a toy example of how you can use this script with just a docker command. +``` +docker run --rm -v $PWD/:/data/ rtibiocloud/s3_presigned_url_generator:v1_23c8ea4 \ + --infile /data/file_list.txt \ + --outfile /data/upload_script3.sh \ + --bucket rti-cool-project \ + --key-prefix scratch/some_rti_user/ \ + --expiration-days 7 \ + --aws-access-key AKIACCESSkeyEXAMPLE \ + --aws-secret-access-key qFyQSECRECTaccessKEYexample +``` +* _Note_: check the DockerHub rtibiocloud repository for the latest tag (i.e., replace `v1_23c8ea4` if necessary), and don't forget to change the access keys in this toy example. + +
+ + + + + + +## Using the Upload Script + +The generated `upload_script.sh` contains the necessary `curl` commands to upload files to the S3 location using presigned URLs. To use the script, follow these steps: + +1. Ensure that you have the `upload_script.sh` and the files you want to upload in the same directory. +2. Open a terminal and navigate to the directory containing the `upload_script.sh` and the files. +3. Make the `upload_script.sh` file executable.`chmod +x upload_script.sh` +4. Execute the `upload_script.sh` script. `./upload_script.sh` + +The script will start executing the `curl` commands, uploading each file to the specified S3 location using the presigned URLs. + +_Note_: Depending on the number and size of the files, the upload process may take some time. Monitor the progress in the terminal. +Once the script finishes executing, all the files should be successfully uploaded to the S3 bucket and location specified in the script. + +
+ + + + + +## Communicating with Collaborators + +To ensure the successful upload of files by external collaborators, it is recommended to communicate with them and provide necessary instructions. Here's a template for an email you can send to collaborators: + +
+ mock email + +
+ + **Subject**: Uploading files to [Your Project Name] - Action Required + +Dear Collaborator, + +We are excited to work with you on [Your Project Name]. As part of our collaboration, we kindly request you to upload your files to our Amazon S3 bucket using the provided presigned URLs. This process ensures secure and efficient file transfers without requiring separate AWS accounts. + +Here are the steps to upload your files: + +1. Place the attached `upload_script.sh` file in the same directory as the files you want to upload. + +2. Open a terminal and navigate to the directory containing the `upload_script.sh` and your files. + +3. Execute the `upload_script.sh` script: + ```shell + bash upload_script.sh + ``` + +This will start the upload process. The script will automatically upload your files to our S3 bucket using presigned URLs. +Once the upload is complete, please reply to this email with the MD5 checksum for each uploaded file. This will allow us to verify the integrity of the transferred files. + +If you encounter any issues or have any questions during the upload process, please feel free to reach out to us. We are here to assist you. + +Thank you for your collaboration! + +Best regards,
+[Your Name]
+[Your Organization] +
+ + +
+ + + + + + +## Limitations +When using AWS presigned URLs, there is a limitation of 5GB for file uploads ([reference](https://docs.aws.amazon.com/AmazonS3/latest/userguide/upload-objects.html)). If your file size exceeds this limit, you will need to consider alternative methods or break down the file into smaller parts to accommodate the restriction. + + + +

+___ + + + + + + +## Support +For support or any questions, please reach out to Jesse Marks (jmarks@rti.org) diff --git a/s3_presigned_url_generator_jesse/v1/requirements.txt b/s3_presigned_url_generator_jesse/v1/requirements.txt new file mode 100644 index 0000000..bf892f6 --- /dev/null +++ b/s3_presigned_url_generator_jesse/v1/requirements.txt @@ -0,0 +1 @@ +boto3==1.24.28 diff --git a/s3_presigned_url_generator_jesse/v1/s3_presigned_upload.py b/s3_presigned_url_generator_jesse/v1/s3_presigned_upload.py new file mode 100644 index 0000000..410704e --- /dev/null +++ b/s3_presigned_url_generator_jesse/v1/s3_presigned_upload.py @@ -0,0 +1,99 @@ +import argparse +import boto3 + + +def generate_presigned_urls(infile, outfile, bucket, key_prefix, expiration_days, access_key, secret_access_key): + """ + Generate a bash script containing curl commands with presigned URLs for uploading files to S3. + + This script takes an input file containing a list of file names, and for each file, it generates a presigned URL + using the provided AWS credentials. The presigned URL allows external collaborators to upload their files to the + specified S3 bucket using curl commands. The generated curl commands are written to the output file as a bash script. + + Args: + infile (str): Path to the input file containing the list of file names to generate presigned URLs for. + outfile (str): Path to the output bash script file that will contain the generated curl commands. + bucket (str): Name of the S3 bucket where the files will be uploaded. + key_prefix (str): Prefix to be prepended to each file name as the S3 object key. + expiration_days (int): Expiration duration in days for the generated presigned URLs. + access_key (str): AWS access key to be used for authentication. + secret_access_key (str): AWS secret access key to be used for authentication. + + Example: + Let's assume you have an input file named 'file_list.txt' containing the following filenames: + ``` + file1.txt + file2.jpg + file3.pdf + ``` + + You want to generate a bash script named 'upload_script.sh' that will contain the curl commands with presigned + URLs for uploading these files to the S3 bucket 'my-bucket' with the key prefix 'uploads/' and a URL expiration + of 7 days. + + You can execute the script as follows: + ``` + python s3_presigned_upload.py \ + --infile file_list.txt \ + --outfile upload_script.sh \ + --bucket my-bucket \ + --key-prefix uploads/ \ + --expiration-days 7 \ + --aws-access-key YOUR_ACCESS_KEY \ + --aws-secret-access-key YOUR_SECRET_ACCESS_KEY + ``` + + The generated 'upload_script.sh' will contain the curl commands to upload the files using presigned URLs. + You can share the 'upload_script.sh' with the external collaborators, and they can execute it in the same + folder as their files to upload them to your S3 account. + """ + + session = boto3.Session(aws_access_key_id=access_key, aws_secret_access_key=secret_access_key) + s3 = session.client("s3") + + with open(infile) as inF, open(outfile, "w") as outF: + outF.write("#!/bin/bash\n\n") + line = inF.readline() + + while line: + seconds = expiration_days * 60 * 60 * 24 + + key = "{}{}".format(key_prefix, line.strip()) + outurl = s3.generate_presigned_url( + 'put_object', + Params={'Bucket': bucket, 'Key': key}, + ExpiresIn=seconds, + HttpMethod='PUT' + ) + + outline1 = "##{}".format(line) # comment line + outline2 = "curl --request PUT --upload-file {} '{}'\n\n".format(line.strip(), outurl) + + outF.write(outline1) + outF.write(outline2) + line = inF.readline() + + outF.write("echo 'File(s) successfully uploaded to S3!'") + print(f"\n\nSuccess!\nCreated the bash script '{outfile}' for uploading files to S3 via presigned URLs.") + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Generate presigned URLs for S3 objects") + parser.add_argument("--infile", required=True, help="Input file path") + parser.add_argument("--outfile", required=True, help="Output file path") + parser.add_argument("--bucket", required=True, help="S3 bucket name") + parser.add_argument("--key-prefix", "-k", dest="key_prefix", required=True, help="S3 key prefix") + parser.add_argument("--expiration-days", "-e", dest="expiration_days", type=int, help="URL expiration in days") + parser.add_argument("--aws-access-key","-a", dest="access_key", required=True, type=str, help="AWS access key ID") + parser.add_argument("--aws-secret-access-key", "-s", dest="secret_access_key", required=True, type=str, help="AWS secret access key") + + args = parser.parse_args() + + generate_presigned_urls( + args.infile, + args.outfile, + args.bucket, + args.key_prefix, + args.expiration_days, + args.access_key, + args.secret_access_key + ) From c42cd38909de74bb1a0a3227c1ed5d7bc6ccaacc Mon Sep 17 00:00:00 2001 From: Jesse Marks <32715488+jaamarks@users.noreply.github.com> Date: Mon, 26 Feb 2024 12:17:49 -0500 Subject: [PATCH 04/27] Delete .github/actions/build-image directory --- .github/actions/build-image/Dockerfile | 18 --- .github/actions/build-image/action.yml | 27 ---- .github/actions/build-image/entrypoint.sh | 182 ---------------------- .github/actions/build-image/mock.sh | 22 --- .github/actions/build-image/test.bats | 57 ------- 5 files changed, 306 deletions(-) delete mode 100644 .github/actions/build-image/Dockerfile delete mode 100644 .github/actions/build-image/action.yml delete mode 100755 .github/actions/build-image/entrypoint.sh delete mode 100755 .github/actions/build-image/mock.sh delete mode 100755 .github/actions/build-image/test.bats diff --git a/.github/actions/build-image/Dockerfile b/.github/actions/build-image/Dockerfile deleted file mode 100644 index f3b9b51..0000000 --- a/.github/actions/build-image/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM docker:19.03.2 as runtime - -RUN apk update \ - && apk upgrade \ - && apk add --no-cache git \ - bash - -ADD entrypoint.sh /entrypoint.sh -ENTRYPOINT ["/entrypoint.sh"] - -# FROM runtime as testEnv -# RUN apk add --no-cache coreutils bats -# ADD test.bats /test.bats -# ADD mock.sh /usr/local/bin/docker -# ADD mock.sh /usr/bin/date -# RUN /test.bats - -FROM runtime diff --git a/.github/actions/build-image/action.yml b/.github/actions/build-image/action.yml deleted file mode 100644 index 405dcda..0000000 --- a/.github/actions/build-image/action.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: 'Build and Publish Docker' -author: 'Chris Ball' -branding: - icon: 'code' - color: 'green' -description: 'Builds the specified Dockerfile and pushes the image to Docker Hub.' -inputs: - changed_files: - description: 'The files changed in the triggering commit.' - required: true - username: - description: 'The login username for the registry' - required: true - password: - description: 'The login password for the registry' - required: true - organization: - description: 'The organization to push the image to' - required: true -outputs: - tag: - description: 'Is the tag, which was pushed' - digest: - description: 'Is the digest of the image, which was pushed' -runs: - using: 'docker' - image: 'Dockerfile' diff --git a/.github/actions/build-image/entrypoint.sh b/.github/actions/build-image/entrypoint.sh deleted file mode 100755 index ed227db..0000000 --- a/.github/actions/build-image/entrypoint.sh +++ /dev/null @@ -1,182 +0,0 @@ -#!/bin/bash -set -e - -function main() { - echo "" # see https://github.com/actions/toolkit/issues/168 - cd /github/workspace - - sanitize "${INPUT_USERNAME}" "username" - sanitize "${INPUT_PASSWORD}" "password" - sanitize "${INPUT_ORGANIZATION}" "organization" - sanitize "${INPUT_CHANGED_FILES}" "changed_files" - - # CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r ${GITHUB_SHA}) # dfe37af2c9a8c753fcd6392ea2f5e711a04b38e1 - CHANGED_FILES="${INPUT_CHANGED_FILES}" - - # Can only build 1 Docker image in 1 actions run/commit - if [[ $(echo $CHANGED_FILES | tr " " "\n" | grep -c "Dockerfile") -gt 1 ]]; then - echo "Only one changed Dockerfile is allowed per commit." - exit 1 - fi - - # Only changes to 1 Docker image directory allowed per commit - BASE_DIR_ARR=() - for FILE in ${CHANGED_FILES} - do - IFS='/'; arrFILE=($FILE); unset IFS; - BASE_DIR_ARR+=(${arrFILE[0]}) - done - UNIQUE_DIRS=($(echo "${BASE_DIR_ARR[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')) - if [[ ${#UNIQUE_DIRS[@]} -gt 1 ]]; then - echo "Only 1 Docker image directory allowed per commit" - exit 1 - fi - - CFS_arr=($(echo "$CHANGED_FILES" | tr " " "\n")) - FIRST_FILE=${CFS_arr[0]} - - IFS='/'; arrFILE=($FIRST_FILE); unset IFS; - if [[ ${#arrFILE[@]} -eq 3 ]]; then - REGISTRY_NO_PROTOCOL=${arrFILE[0]} - SOFTWARE_VERSION=${arrFILE[1]} - INPUT_WORKDIR=${arrFILE[0]}/${arrFILE[1]} - fi - if [[ ${#arrFILE[@]} -eq 2 ]]; then - REGISTRY_NO_PROTOCOL=${arrFILE[0]} - SOFTWARE_VERSION="none" - INPUT_WORKDIR=${arrFILE[0]} - fi - if [[ ${#arrFILE[@]} -eq 1 ]]; then - echo "File is not in a directory." - exit 0 - fi - if [[ ${#arrFILE[@]} -eq 0 ]]; then - echo "No changed files found." - exit 0 - fi - - # INPUT_REGISTRY="${ORGANIZATION}/${REGISTRY_NO_PROTOCOL}" - # INPUT_NAME="${INPUT_REGISTRY}" - INPUT_NAME="${INPUT_ORGANIZATION}/${REGISTRY_NO_PROTOCOL}" - - if uses "${INPUT_WORKDIR}"; then - changeWorkingDirectory - fi - - # echo ${INPUT_PASSWORD} | docker login -u ${INPUT_USERNAME} --password-stdin ${INPUT_REGISTRY} - echo ${INPUT_PASSWORD} | docker login -u ${INPUT_USERNAME} --password-stdin - - SHA_TAG="${SOFTWARE_VERSION}"_$(echo "${GITHUB_SHA}" | cut -c1-7) - - TAGS=("${SHA_TAG}") - FIRST_TAG=$(echo $TAGS | cut -d ' ' -f1) - DOCKERNAME="${INPUT_NAME}:${FIRST_TAG}" - BUILDPARAMS="" - CONTEXT="." - - # if uses "${INPUT_DOCKERFILE}"; then - # useCustomDockerfile - # fi - # if uses "${INPUT_BUILDARGS}"; then - # addBuildArgs - # fi - # if uses "${INPUT_CONTEXT}"; then - # CONTEXT="${INPUT_CONTEXT}" - # fi - # if usesBoolean "${INPUT_CACHE}"; then - # useBuildCache - # fi - # if usesBoolean "${INPUT_SNAPSHOT}"; then - # useSnapshot - # fi - - push - - echo "::set-output name=tag::${FIRST_TAG}" - DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' ${DOCKERNAME}) - echo "::set-output name=digest::${DIGEST}" - - docker logout -} - -function sanitize() { - if [ -z "${1}" ]; then - >&2 echo "Unable to find the ${2}. Did you set with.${2}?" - exit 1 - fi -} - -# function isPartOfTheName() { -# [ $(echo "${INPUT_NAME}" | sed -e "s/${1}//g") != "${INPUT_NAME}" ] -# } - - -# function hasCustomTag() { -# [ $(echo "${INPUT_NAME}" | sed -e "s/://g") != "${INPUT_NAME}" ] -# } - -# function isOnMaster() { -# [ "${BRANCH}" = "master" ] -# } - -# function isGitTag() { -# [ $(echo "${GITHUB_REF}" | sed -e "s/refs\/tags\///g") != "${GITHUB_REF}" ] -# } - -# function isPullRequest() { -# [ $(echo "${GITHUB_REF}" | sed -e "s/refs\/pull\///g") != "${GITHUB_REF}" ] -# } - -function changeWorkingDirectory() { - cd "${INPUT_WORKDIR}" -} - -# function useCustomDockerfile() { -# BUILDPARAMS="${BUILDPARAMS} -f ${INPUT_DOCKERFILE}" -# } - -# function addBuildArgs() { -# for ARG in $(echo "${INPUT_BUILDARGS}" | tr ',' '\n'); do -# BUILDPARAMS="${BUILDPARAMS} --build-arg ${ARG}" -# echo "::add-mask::${ARG}" -# done -# } - -# function useBuildCache() { -# if docker pull ${DOCKERNAME} 2>/dev/null; then -# BUILDPARAMS="$BUILDPARAMS --cache-from ${DOCKERNAME}" -# fi -# } - -function uses() { - [ ! -z "${1}" ] -} - -# function usesBoolean() { -# [ ! -z "${1}" ] && [ "${1}" = "true" ] -# } - -# function useSnapshot() { -# local TIMESTAMP=`date +%Y%m%d%H%M%S` -# local SHORT_SHA=$(echo "${GITHUB_SHA}" | cut -c1-6) -# local SNAPSHOT_TAG="${TIMESTAMP}${SHORT_SHA}" -# TAGS="${TAGS} ${SNAPSHOT_TAG}" -# echo ::set-output name=snapshot-tag::"${SNAPSHOT_TAG}" -# } - -function push() { - local BUILD_TAGS="" - for TAG in ${TAGS} - do - BUILD_TAGS="${BUILD_TAGS}-t ${INPUT_NAME}:${TAG} " - done - # docker build ${INPUT_BUILDOPTIONS} ${BUILDPARAMS} ${BUILD_TAGS} ${CONTEXT} - docker build ${BUILDPARAMS} ${BUILD_TAGS} ${CONTEXT} - - for TAG in ${TAGS} - do - docker push "${INPUT_NAME}:${TAG}" - done -} - -main diff --git a/.github/actions/build-image/mock.sh b/.github/actions/build-image/mock.sh deleted file mode 100755 index 9b195ad..0000000 --- a/.github/actions/build-image/mock.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -binary="$0" -parameters="$@" -echo "${binary} ${parameters}" >> mockCalledWith - -function mockShouldFail() { - [ "${MOCK_RETURNS[${binary}]}" = "_${parameters}" ] -} - -source mockReturns -if [ ! -z "${MOCK_RETURNS[${binary}]}" ] || [ ! -z "${MOCK_RETURNS[${binary} $1]}" ]; then - if mockShouldFail ; then - exit 1 - fi - if [ ! -z "${MOCK_RETURNS[${binary} $1]}" ]; then - echo ${MOCK_RETURNS[${binary} $1]} - exit 0 - fi - echo ${MOCK_RETURNS[${binary}]} -fi - -exit 0 diff --git a/.github/actions/build-image/test.bats b/.github/actions/build-image/test.bats deleted file mode 100755 index a2bf269..0000000 --- a/.github/actions/build-image/test.bats +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bats - -setup(){ - cat /dev/null >| mockCalledWith - - declare -A -p MOCK_RETURNS=( - ['/usr/local/bin/docker']="" - ) > mockReturns - - export GITHUB_REF='refs/heads/master' - export INPUT_USERNAME='USERNAME' - export INPUT_PASSWORD='PASSWORD' - export ORGANIZATION='my_org' -} - -teardown() { - unset INPUT_TAG_NAMES - unset INPUT_SNAPSHOT - unset INPUT_DOCKERFILE - unset INPUT_REGISTRY - unset INPUT_CACHE - unset GITHUB_SHA - unset INPUT_PULL_REQUESTS - unset MOCK_ERROR_CONDITION -} - -@test "it builds and pushes the Dockerimage with single Dockerfile in commit" { - export GITHUB_SHA='12169ed809255604e557a82617264e9c373faca7' - export INPUT_USERNAME=username - export INPUT_PASSWORD=password - - run /entrypoint.sh - - expectStdOutContains " - ::set-output name=tag::none_12169e" - - expectMockCalled "/usr/local/bin/docker build -t gwas/generate_gwas_plots:none_12169e . - /usr/local/bin/docker push gwas/generate_gwas_plots:none_12169e" -} - - - -function expectStdOutContains() { - local expected=$(echo "${1}" | tr -d '\n') - local got=$(echo "${output}" | tr -d '\n') - echo "Expected: |${expected}| - Got: |${got}|" - echo "${got}" | grep "${expected}" -} - -function expectMockCalled() { - local expected=$(echo "${1}" | tr -d '\n') - local got=$(cat mockCalledWith | tr -d '\n') - echo "Expected: |${expected}| - Got: |${got}|" - echo "${got}" | grep "${expected}" -} From c5840a13e4bce1b03b9da03911b2c1db47193d1b Mon Sep 17 00:00:00 2001 From: Jesse Marks <32715488+jaamarks@users.noreply.github.com> Date: Mon, 26 Feb 2024 12:18:03 -0500 Subject: [PATCH 05/27] Delete .github/workflows directory --- .github/workflows/dockerimage.yml | 39 ------------------------------- 1 file changed, 39 deletions(-) delete mode 100644 .github/workflows/dockerimage.yml diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml deleted file mode 100644 index 5ce21b1..0000000 --- a/.github/workflows/dockerimage.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Build and Publish Docker Images - -on: - push: - branches: - - master - paths-ignore: - - '.github/**' - - '.gitignore' - - 'README.md' - - '*/*/README.md' - pull_request: - branches: - - master - paths-ignore: - - '.github/**' - - '.gitignore' - - 'README.md' - - '*/*/README.md' - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - run: | - git fetch --prune --unshallow - - name: get changed files - id: getfile - run: | - echo "::set-output name=files::$(git diff-tree --no-commit-id --name-only -r ${{ github.sha }} | xargs)" - - name: Build, Tag, Publish Docker - uses: ./.github/actions/build-image - with: - organization: jessemarks - changed_files: ${{ steps.getfile.outputs.files }} - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} From f0bb7e024de3a3b6b0bc64de95e4bf2eb2f9bde2 Mon Sep 17 00:00:00 2001 From: Jesse Marks <32715488+jaamarks@users.noreply.github.com> Date: Mon, 26 Feb 2024 13:57:10 -0500 Subject: [PATCH 06/27] Revert "create a new tool to test our github actions" --- .github/actions/build-image/Dockerfile | 18 ++ .github/actions/build-image/action.yml | 27 +++ .github/actions/build-image/entrypoint.sh | 182 ++++++++++++++++++ .github/actions/build-image/mock.sh | 22 +++ .github/actions/build-image/test.bats | 57 ++++++ .github/workflows/dockerimage.yml | 39 ++++ .../v1/Dockerfile | 29 --- s3_presigned_url_generator_jesse/v1/README.md | 179 ----------------- .../v1/requirements.txt | 1 - .../v1/s3_presigned_upload.py | 99 ---------- 10 files changed, 345 insertions(+), 308 deletions(-) create mode 100644 .github/actions/build-image/Dockerfile create mode 100644 .github/actions/build-image/action.yml create mode 100755 .github/actions/build-image/entrypoint.sh create mode 100755 .github/actions/build-image/mock.sh create mode 100755 .github/actions/build-image/test.bats create mode 100644 .github/workflows/dockerimage.yml delete mode 100644 s3_presigned_url_generator_jesse/v1/Dockerfile delete mode 100644 s3_presigned_url_generator_jesse/v1/README.md delete mode 100644 s3_presigned_url_generator_jesse/v1/requirements.txt delete mode 100644 s3_presigned_url_generator_jesse/v1/s3_presigned_upload.py diff --git a/.github/actions/build-image/Dockerfile b/.github/actions/build-image/Dockerfile new file mode 100644 index 0000000..f3b9b51 --- /dev/null +++ b/.github/actions/build-image/Dockerfile @@ -0,0 +1,18 @@ +FROM docker:19.03.2 as runtime + +RUN apk update \ + && apk upgrade \ + && apk add --no-cache git \ + bash + +ADD entrypoint.sh /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] + +# FROM runtime as testEnv +# RUN apk add --no-cache coreutils bats +# ADD test.bats /test.bats +# ADD mock.sh /usr/local/bin/docker +# ADD mock.sh /usr/bin/date +# RUN /test.bats + +FROM runtime diff --git a/.github/actions/build-image/action.yml b/.github/actions/build-image/action.yml new file mode 100644 index 0000000..405dcda --- /dev/null +++ b/.github/actions/build-image/action.yml @@ -0,0 +1,27 @@ +name: 'Build and Publish Docker' +author: 'Chris Ball' +branding: + icon: 'code' + color: 'green' +description: 'Builds the specified Dockerfile and pushes the image to Docker Hub.' +inputs: + changed_files: + description: 'The files changed in the triggering commit.' + required: true + username: + description: 'The login username for the registry' + required: true + password: + description: 'The login password for the registry' + required: true + organization: + description: 'The organization to push the image to' + required: true +outputs: + tag: + description: 'Is the tag, which was pushed' + digest: + description: 'Is the digest of the image, which was pushed' +runs: + using: 'docker' + image: 'Dockerfile' diff --git a/.github/actions/build-image/entrypoint.sh b/.github/actions/build-image/entrypoint.sh new file mode 100755 index 0000000..ed227db --- /dev/null +++ b/.github/actions/build-image/entrypoint.sh @@ -0,0 +1,182 @@ +#!/bin/bash +set -e + +function main() { + echo "" # see https://github.com/actions/toolkit/issues/168 + cd /github/workspace + + sanitize "${INPUT_USERNAME}" "username" + sanitize "${INPUT_PASSWORD}" "password" + sanitize "${INPUT_ORGANIZATION}" "organization" + sanitize "${INPUT_CHANGED_FILES}" "changed_files" + + # CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r ${GITHUB_SHA}) # dfe37af2c9a8c753fcd6392ea2f5e711a04b38e1 + CHANGED_FILES="${INPUT_CHANGED_FILES}" + + # Can only build 1 Docker image in 1 actions run/commit + if [[ $(echo $CHANGED_FILES | tr " " "\n" | grep -c "Dockerfile") -gt 1 ]]; then + echo "Only one changed Dockerfile is allowed per commit." + exit 1 + fi + + # Only changes to 1 Docker image directory allowed per commit + BASE_DIR_ARR=() + for FILE in ${CHANGED_FILES} + do + IFS='/'; arrFILE=($FILE); unset IFS; + BASE_DIR_ARR+=(${arrFILE[0]}) + done + UNIQUE_DIRS=($(echo "${BASE_DIR_ARR[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')) + if [[ ${#UNIQUE_DIRS[@]} -gt 1 ]]; then + echo "Only 1 Docker image directory allowed per commit" + exit 1 + fi + + CFS_arr=($(echo "$CHANGED_FILES" | tr " " "\n")) + FIRST_FILE=${CFS_arr[0]} + + IFS='/'; arrFILE=($FIRST_FILE); unset IFS; + if [[ ${#arrFILE[@]} -eq 3 ]]; then + REGISTRY_NO_PROTOCOL=${arrFILE[0]} + SOFTWARE_VERSION=${arrFILE[1]} + INPUT_WORKDIR=${arrFILE[0]}/${arrFILE[1]} + fi + if [[ ${#arrFILE[@]} -eq 2 ]]; then + REGISTRY_NO_PROTOCOL=${arrFILE[0]} + SOFTWARE_VERSION="none" + INPUT_WORKDIR=${arrFILE[0]} + fi + if [[ ${#arrFILE[@]} -eq 1 ]]; then + echo "File is not in a directory." + exit 0 + fi + if [[ ${#arrFILE[@]} -eq 0 ]]; then + echo "No changed files found." + exit 0 + fi + + # INPUT_REGISTRY="${ORGANIZATION}/${REGISTRY_NO_PROTOCOL}" + # INPUT_NAME="${INPUT_REGISTRY}" + INPUT_NAME="${INPUT_ORGANIZATION}/${REGISTRY_NO_PROTOCOL}" + + if uses "${INPUT_WORKDIR}"; then + changeWorkingDirectory + fi + + # echo ${INPUT_PASSWORD} | docker login -u ${INPUT_USERNAME} --password-stdin ${INPUT_REGISTRY} + echo ${INPUT_PASSWORD} | docker login -u ${INPUT_USERNAME} --password-stdin + + SHA_TAG="${SOFTWARE_VERSION}"_$(echo "${GITHUB_SHA}" | cut -c1-7) + + TAGS=("${SHA_TAG}") + FIRST_TAG=$(echo $TAGS | cut -d ' ' -f1) + DOCKERNAME="${INPUT_NAME}:${FIRST_TAG}" + BUILDPARAMS="" + CONTEXT="." + + # if uses "${INPUT_DOCKERFILE}"; then + # useCustomDockerfile + # fi + # if uses "${INPUT_BUILDARGS}"; then + # addBuildArgs + # fi + # if uses "${INPUT_CONTEXT}"; then + # CONTEXT="${INPUT_CONTEXT}" + # fi + # if usesBoolean "${INPUT_CACHE}"; then + # useBuildCache + # fi + # if usesBoolean "${INPUT_SNAPSHOT}"; then + # useSnapshot + # fi + + push + + echo "::set-output name=tag::${FIRST_TAG}" + DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' ${DOCKERNAME}) + echo "::set-output name=digest::${DIGEST}" + + docker logout +} + +function sanitize() { + if [ -z "${1}" ]; then + >&2 echo "Unable to find the ${2}. Did you set with.${2}?" + exit 1 + fi +} + +# function isPartOfTheName() { +# [ $(echo "${INPUT_NAME}" | sed -e "s/${1}//g") != "${INPUT_NAME}" ] +# } + + +# function hasCustomTag() { +# [ $(echo "${INPUT_NAME}" | sed -e "s/://g") != "${INPUT_NAME}" ] +# } + +# function isOnMaster() { +# [ "${BRANCH}" = "master" ] +# } + +# function isGitTag() { +# [ $(echo "${GITHUB_REF}" | sed -e "s/refs\/tags\///g") != "${GITHUB_REF}" ] +# } + +# function isPullRequest() { +# [ $(echo "${GITHUB_REF}" | sed -e "s/refs\/pull\///g") != "${GITHUB_REF}" ] +# } + +function changeWorkingDirectory() { + cd "${INPUT_WORKDIR}" +} + +# function useCustomDockerfile() { +# BUILDPARAMS="${BUILDPARAMS} -f ${INPUT_DOCKERFILE}" +# } + +# function addBuildArgs() { +# for ARG in $(echo "${INPUT_BUILDARGS}" | tr ',' '\n'); do +# BUILDPARAMS="${BUILDPARAMS} --build-arg ${ARG}" +# echo "::add-mask::${ARG}" +# done +# } + +# function useBuildCache() { +# if docker pull ${DOCKERNAME} 2>/dev/null; then +# BUILDPARAMS="$BUILDPARAMS --cache-from ${DOCKERNAME}" +# fi +# } + +function uses() { + [ ! -z "${1}" ] +} + +# function usesBoolean() { +# [ ! -z "${1}" ] && [ "${1}" = "true" ] +# } + +# function useSnapshot() { +# local TIMESTAMP=`date +%Y%m%d%H%M%S` +# local SHORT_SHA=$(echo "${GITHUB_SHA}" | cut -c1-6) +# local SNAPSHOT_TAG="${TIMESTAMP}${SHORT_SHA}" +# TAGS="${TAGS} ${SNAPSHOT_TAG}" +# echo ::set-output name=snapshot-tag::"${SNAPSHOT_TAG}" +# } + +function push() { + local BUILD_TAGS="" + for TAG in ${TAGS} + do + BUILD_TAGS="${BUILD_TAGS}-t ${INPUT_NAME}:${TAG} " + done + # docker build ${INPUT_BUILDOPTIONS} ${BUILDPARAMS} ${BUILD_TAGS} ${CONTEXT} + docker build ${BUILDPARAMS} ${BUILD_TAGS} ${CONTEXT} + + for TAG in ${TAGS} + do + docker push "${INPUT_NAME}:${TAG}" + done +} + +main diff --git a/.github/actions/build-image/mock.sh b/.github/actions/build-image/mock.sh new file mode 100755 index 0000000..9b195ad --- /dev/null +++ b/.github/actions/build-image/mock.sh @@ -0,0 +1,22 @@ +#!/bin/bash +binary="$0" +parameters="$@" +echo "${binary} ${parameters}" >> mockCalledWith + +function mockShouldFail() { + [ "${MOCK_RETURNS[${binary}]}" = "_${parameters}" ] +} + +source mockReturns +if [ ! -z "${MOCK_RETURNS[${binary}]}" ] || [ ! -z "${MOCK_RETURNS[${binary} $1]}" ]; then + if mockShouldFail ; then + exit 1 + fi + if [ ! -z "${MOCK_RETURNS[${binary} $1]}" ]; then + echo ${MOCK_RETURNS[${binary} $1]} + exit 0 + fi + echo ${MOCK_RETURNS[${binary}]} +fi + +exit 0 diff --git a/.github/actions/build-image/test.bats b/.github/actions/build-image/test.bats new file mode 100755 index 0000000..a2bf269 --- /dev/null +++ b/.github/actions/build-image/test.bats @@ -0,0 +1,57 @@ +#!/usr/bin/env bats + +setup(){ + cat /dev/null >| mockCalledWith + + declare -A -p MOCK_RETURNS=( + ['/usr/local/bin/docker']="" + ) > mockReturns + + export GITHUB_REF='refs/heads/master' + export INPUT_USERNAME='USERNAME' + export INPUT_PASSWORD='PASSWORD' + export ORGANIZATION='my_org' +} + +teardown() { + unset INPUT_TAG_NAMES + unset INPUT_SNAPSHOT + unset INPUT_DOCKERFILE + unset INPUT_REGISTRY + unset INPUT_CACHE + unset GITHUB_SHA + unset INPUT_PULL_REQUESTS + unset MOCK_ERROR_CONDITION +} + +@test "it builds and pushes the Dockerimage with single Dockerfile in commit" { + export GITHUB_SHA='12169ed809255604e557a82617264e9c373faca7' + export INPUT_USERNAME=username + export INPUT_PASSWORD=password + + run /entrypoint.sh + + expectStdOutContains " + ::set-output name=tag::none_12169e" + + expectMockCalled "/usr/local/bin/docker build -t gwas/generate_gwas_plots:none_12169e . + /usr/local/bin/docker push gwas/generate_gwas_plots:none_12169e" +} + + + +function expectStdOutContains() { + local expected=$(echo "${1}" | tr -d '\n') + local got=$(echo "${output}" | tr -d '\n') + echo "Expected: |${expected}| + Got: |${got}|" + echo "${got}" | grep "${expected}" +} + +function expectMockCalled() { + local expected=$(echo "${1}" | tr -d '\n') + local got=$(cat mockCalledWith | tr -d '\n') + echo "Expected: |${expected}| + Got: |${got}|" + echo "${got}" | grep "${expected}" +} diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml new file mode 100644 index 0000000..5ce21b1 --- /dev/null +++ b/.github/workflows/dockerimage.yml @@ -0,0 +1,39 @@ +name: Build and Publish Docker Images + +on: + push: + branches: + - master + paths-ignore: + - '.github/**' + - '.gitignore' + - 'README.md' + - '*/*/README.md' + pull_request: + branches: + - master + paths-ignore: + - '.github/**' + - '.gitignore' + - 'README.md' + - '*/*/README.md' + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - run: | + git fetch --prune --unshallow + - name: get changed files + id: getfile + run: | + echo "::set-output name=files::$(git diff-tree --no-commit-id --name-only -r ${{ github.sha }} | xargs)" + - name: Build, Tag, Publish Docker + uses: ./.github/actions/build-image + with: + organization: jessemarks + changed_files: ${{ steps.getfile.outputs.files }} + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} diff --git a/s3_presigned_url_generator_jesse/v1/Dockerfile b/s3_presigned_url_generator_jesse/v1/Dockerfile deleted file mode 100644 index 3d928b3..0000000 --- a/s3_presigned_url_generator_jesse/v1/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -# Use an official Python runtime as the base image -FROM python:3.12-alpine - -# Add Container Labels -LABEL maintainer="Jesse Marks " -LABEL description="A script to generate presigned URLs to upload to S3." -LABEL base-image="python:3.12-alpine" - -# Install System Dependencies -RUN apt-get update && apt-get install -y \ - vim \ - less \ - curl \ - && rm -rf /var/lib/apt/lists/* - -# Set the working directory in the container -WORKDIR /opt/ - -# Copy the script and requirements file to the container -COPY s3_presigned_upload.py requirements.txt ./ - -# Install the required dependencies -RUN pip install --no-cache-dir -r requirements.txt - -# Set the entry point command -ENTRYPOINT ["python", "s3_presigned_upload.py"] - -# Set the default command arguments -CMD ["--help"] diff --git a/s3_presigned_url_generator_jesse/v1/README.md b/s3_presigned_url_generator_jesse/v1/README.md deleted file mode 100644 index d0fb2e5..0000000 --- a/s3_presigned_url_generator_jesse/v1/README.md +++ /dev/null @@ -1,179 +0,0 @@ -# S3 Presigned URL Generator - -A command-line interface (CLI) tool to generate a bash script containing `curl` commands with presigned URLs for uploading files to Amazon S3. This tool enables external collaborators to upload their files to your S3 bucket securely using presigned URLs, eliminating the need for separate AWS accounts. - -
- - -[Click here to go to the recommended docker usage example](#docker-anchor) - -
- - - - - - -## Usage - -```shell -python s3_presigned_upload.py \ - --infile \ - --outfile \ - --bucket \ - --key-prefix \ - --expiration-days \ - --aws-access-key \ - --aws-secret-access-key -``` - - -Replace the following placeholders with the appropriate values: - -- ``: Path to the input file containing a list of file names to generate presigned URLs for. -- ``: Path to the output bash script file that will contain the generated curl commands. -- ``: Name of the S3 bucket where the files will be uploaded. -- ``: Prefix to be prepended to each file name as the S3 object key. -- ``: Expiration duration in days for the generated presigned URLs. -- ``: AWS access key ID for authentication. -- ``: AWS secret access key for authentication. - -* _Note_: you can typically find your access keys in your AWS CLI Configuration Files (`~/.aws/credentials`) - -Example: - -Let's assume you have an input file named `file_list.txt` containing the following filenames: - -``` -file1.txt -file2.jpg -file3.pdf -``` - -You want to generate a bash script named `upload_script.sh` that will contain the curl commands with presigned URLs for uploading these files to the S3 bucket `my-bucket` with the key prefix `uploads/` and a URL expiration of 7 days. - -You can execute the script as follows: - -```shell -python s3_presigned_upload.py \ - --infile file_list.txt \ - --outfile upload_script.sh \ - --bucket my-bucket \ - --key-prefix uploads/ \ - --expiration-days 7 \ - --aws-access-key YOUR_ACCESS_KEY \ - --aws-secret-access-key YOUR_SECRET_ACCESS_KEY -``` - -The generated `upload_script.sh` will contain the curl commands necessary to upload the files using presigned URLs. Share the `upload_script.sh` with the external collaborators, and they can execute it in the same folder as their files to upload them to your S3 account. - -
- - - - - - -## Docker usage -**This is the recommended approach.**
-Here is a toy example of how you can use this script with just a docker command. -``` -docker run --rm -v $PWD/:/data/ rtibiocloud/s3_presigned_url_generator:v1_23c8ea4 \ - --infile /data/file_list.txt \ - --outfile /data/upload_script3.sh \ - --bucket rti-cool-project \ - --key-prefix scratch/some_rti_user/ \ - --expiration-days 7 \ - --aws-access-key AKIACCESSkeyEXAMPLE \ - --aws-secret-access-key qFyQSECRECTaccessKEYexample -``` -* _Note_: check the DockerHub rtibiocloud repository for the latest tag (i.e., replace `v1_23c8ea4` if necessary), and don't forget to change the access keys in this toy example. - -
- - - - - - -## Using the Upload Script - -The generated `upload_script.sh` contains the necessary `curl` commands to upload files to the S3 location using presigned URLs. To use the script, follow these steps: - -1. Ensure that you have the `upload_script.sh` and the files you want to upload in the same directory. -2. Open a terminal and navigate to the directory containing the `upload_script.sh` and the files. -3. Make the `upload_script.sh` file executable.`chmod +x upload_script.sh` -4. Execute the `upload_script.sh` script. `./upload_script.sh` - -The script will start executing the `curl` commands, uploading each file to the specified S3 location using the presigned URLs. - -_Note_: Depending on the number and size of the files, the upload process may take some time. Monitor the progress in the terminal. -Once the script finishes executing, all the files should be successfully uploaded to the S3 bucket and location specified in the script. - -
- - - - - -## Communicating with Collaborators - -To ensure the successful upload of files by external collaborators, it is recommended to communicate with them and provide necessary instructions. Here's a template for an email you can send to collaborators: - -
- mock email - -
- - **Subject**: Uploading files to [Your Project Name] - Action Required - -Dear Collaborator, - -We are excited to work with you on [Your Project Name]. As part of our collaboration, we kindly request you to upload your files to our Amazon S3 bucket using the provided presigned URLs. This process ensures secure and efficient file transfers without requiring separate AWS accounts. - -Here are the steps to upload your files: - -1. Place the attached `upload_script.sh` file in the same directory as the files you want to upload. - -2. Open a terminal and navigate to the directory containing the `upload_script.sh` and your files. - -3. Execute the `upload_script.sh` script: - ```shell - bash upload_script.sh - ``` - -This will start the upload process. The script will automatically upload your files to our S3 bucket using presigned URLs. -Once the upload is complete, please reply to this email with the MD5 checksum for each uploaded file. This will allow us to verify the integrity of the transferred files. - -If you encounter any issues or have any questions during the upload process, please feel free to reach out to us. We are here to assist you. - -Thank you for your collaboration! - -Best regards,
-[Your Name]
-[Your Organization] -
- - -
- - - - - - -## Limitations -When using AWS presigned URLs, there is a limitation of 5GB for file uploads ([reference](https://docs.aws.amazon.com/AmazonS3/latest/userguide/upload-objects.html)). If your file size exceeds this limit, you will need to consider alternative methods or break down the file into smaller parts to accommodate the restriction. - - - -

-___ - - - - - - -## Support -For support or any questions, please reach out to Jesse Marks (jmarks@rti.org) diff --git a/s3_presigned_url_generator_jesse/v1/requirements.txt b/s3_presigned_url_generator_jesse/v1/requirements.txt deleted file mode 100644 index bf892f6..0000000 --- a/s3_presigned_url_generator_jesse/v1/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -boto3==1.24.28 diff --git a/s3_presigned_url_generator_jesse/v1/s3_presigned_upload.py b/s3_presigned_url_generator_jesse/v1/s3_presigned_upload.py deleted file mode 100644 index 410704e..0000000 --- a/s3_presigned_url_generator_jesse/v1/s3_presigned_upload.py +++ /dev/null @@ -1,99 +0,0 @@ -import argparse -import boto3 - - -def generate_presigned_urls(infile, outfile, bucket, key_prefix, expiration_days, access_key, secret_access_key): - """ - Generate a bash script containing curl commands with presigned URLs for uploading files to S3. - - This script takes an input file containing a list of file names, and for each file, it generates a presigned URL - using the provided AWS credentials. The presigned URL allows external collaborators to upload their files to the - specified S3 bucket using curl commands. The generated curl commands are written to the output file as a bash script. - - Args: - infile (str): Path to the input file containing the list of file names to generate presigned URLs for. - outfile (str): Path to the output bash script file that will contain the generated curl commands. - bucket (str): Name of the S3 bucket where the files will be uploaded. - key_prefix (str): Prefix to be prepended to each file name as the S3 object key. - expiration_days (int): Expiration duration in days for the generated presigned URLs. - access_key (str): AWS access key to be used for authentication. - secret_access_key (str): AWS secret access key to be used for authentication. - - Example: - Let's assume you have an input file named 'file_list.txt' containing the following filenames: - ``` - file1.txt - file2.jpg - file3.pdf - ``` - - You want to generate a bash script named 'upload_script.sh' that will contain the curl commands with presigned - URLs for uploading these files to the S3 bucket 'my-bucket' with the key prefix 'uploads/' and a URL expiration - of 7 days. - - You can execute the script as follows: - ``` - python s3_presigned_upload.py \ - --infile file_list.txt \ - --outfile upload_script.sh \ - --bucket my-bucket \ - --key-prefix uploads/ \ - --expiration-days 7 \ - --aws-access-key YOUR_ACCESS_KEY \ - --aws-secret-access-key YOUR_SECRET_ACCESS_KEY - ``` - - The generated 'upload_script.sh' will contain the curl commands to upload the files using presigned URLs. - You can share the 'upload_script.sh' with the external collaborators, and they can execute it in the same - folder as their files to upload them to your S3 account. - """ - - session = boto3.Session(aws_access_key_id=access_key, aws_secret_access_key=secret_access_key) - s3 = session.client("s3") - - with open(infile) as inF, open(outfile, "w") as outF: - outF.write("#!/bin/bash\n\n") - line = inF.readline() - - while line: - seconds = expiration_days * 60 * 60 * 24 - - key = "{}{}".format(key_prefix, line.strip()) - outurl = s3.generate_presigned_url( - 'put_object', - Params={'Bucket': bucket, 'Key': key}, - ExpiresIn=seconds, - HttpMethod='PUT' - ) - - outline1 = "##{}".format(line) # comment line - outline2 = "curl --request PUT --upload-file {} '{}'\n\n".format(line.strip(), outurl) - - outF.write(outline1) - outF.write(outline2) - line = inF.readline() - - outF.write("echo 'File(s) successfully uploaded to S3!'") - print(f"\n\nSuccess!\nCreated the bash script '{outfile}' for uploading files to S3 via presigned URLs.") - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Generate presigned URLs for S3 objects") - parser.add_argument("--infile", required=True, help="Input file path") - parser.add_argument("--outfile", required=True, help="Output file path") - parser.add_argument("--bucket", required=True, help="S3 bucket name") - parser.add_argument("--key-prefix", "-k", dest="key_prefix", required=True, help="S3 key prefix") - parser.add_argument("--expiration-days", "-e", dest="expiration_days", type=int, help="URL expiration in days") - parser.add_argument("--aws-access-key","-a", dest="access_key", required=True, type=str, help="AWS access key ID") - parser.add_argument("--aws-secret-access-key", "-s", dest="secret_access_key", required=True, type=str, help="AWS secret access key") - - args = parser.parse_args() - - generate_presigned_urls( - args.infile, - args.outfile, - args.bucket, - args.key_prefix, - args.expiration_days, - args.access_key, - args.secret_access_key - ) From eb48297c7882429d1085d7ad3e8dcca9a27ba88d Mon Sep 17 00:00:00 2001 From: jaamarks Date: Mon, 26 Feb 2024 14:25:32 -0500 Subject: [PATCH 07/27] test github actions with this copied docker tool --- .../v1/Dockerfile | 29 +++ s3_presigned_url_generator_jesse/v1/README.md | 179 ++++++++++++++++++ .../v1/requirements.txt | 1 + .../v1/s3_presigned_upload.py | 99 ++++++++++ 4 files changed, 308 insertions(+) create mode 100644 s3_presigned_url_generator_jesse/v1/Dockerfile create mode 100644 s3_presigned_url_generator_jesse/v1/README.md create mode 100644 s3_presigned_url_generator_jesse/v1/requirements.txt create mode 100644 s3_presigned_url_generator_jesse/v1/s3_presigned_upload.py diff --git a/s3_presigned_url_generator_jesse/v1/Dockerfile b/s3_presigned_url_generator_jesse/v1/Dockerfile new file mode 100644 index 0000000..3d928b3 --- /dev/null +++ b/s3_presigned_url_generator_jesse/v1/Dockerfile @@ -0,0 +1,29 @@ +# Use an official Python runtime as the base image +FROM python:3.12-alpine + +# Add Container Labels +LABEL maintainer="Jesse Marks " +LABEL description="A script to generate presigned URLs to upload to S3." +LABEL base-image="python:3.12-alpine" + +# Install System Dependencies +RUN apt-get update && apt-get install -y \ + vim \ + less \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Set the working directory in the container +WORKDIR /opt/ + +# Copy the script and requirements file to the container +COPY s3_presigned_upload.py requirements.txt ./ + +# Install the required dependencies +RUN pip install --no-cache-dir -r requirements.txt + +# Set the entry point command +ENTRYPOINT ["python", "s3_presigned_upload.py"] + +# Set the default command arguments +CMD ["--help"] diff --git a/s3_presigned_url_generator_jesse/v1/README.md b/s3_presigned_url_generator_jesse/v1/README.md new file mode 100644 index 0000000..d0fb2e5 --- /dev/null +++ b/s3_presigned_url_generator_jesse/v1/README.md @@ -0,0 +1,179 @@ +# S3 Presigned URL Generator + +A command-line interface (CLI) tool to generate a bash script containing `curl` commands with presigned URLs for uploading files to Amazon S3. This tool enables external collaborators to upload their files to your S3 bucket securely using presigned URLs, eliminating the need for separate AWS accounts. + +
+ + +[Click here to go to the recommended docker usage example](#docker-anchor) + +
+ + + + + + +## Usage + +```shell +python s3_presigned_upload.py \ + --infile \ + --outfile \ + --bucket \ + --key-prefix \ + --expiration-days \ + --aws-access-key \ + --aws-secret-access-key +``` + + +Replace the following placeholders with the appropriate values: + +- ``: Path to the input file containing a list of file names to generate presigned URLs for. +- ``: Path to the output bash script file that will contain the generated curl commands. +- ``: Name of the S3 bucket where the files will be uploaded. +- ``: Prefix to be prepended to each file name as the S3 object key. +- ``: Expiration duration in days for the generated presigned URLs. +- ``: AWS access key ID for authentication. +- ``: AWS secret access key for authentication. + +* _Note_: you can typically find your access keys in your AWS CLI Configuration Files (`~/.aws/credentials`) + +Example: + +Let's assume you have an input file named `file_list.txt` containing the following filenames: + +``` +file1.txt +file2.jpg +file3.pdf +``` + +You want to generate a bash script named `upload_script.sh` that will contain the curl commands with presigned URLs for uploading these files to the S3 bucket `my-bucket` with the key prefix `uploads/` and a URL expiration of 7 days. + +You can execute the script as follows: + +```shell +python s3_presigned_upload.py \ + --infile file_list.txt \ + --outfile upload_script.sh \ + --bucket my-bucket \ + --key-prefix uploads/ \ + --expiration-days 7 \ + --aws-access-key YOUR_ACCESS_KEY \ + --aws-secret-access-key YOUR_SECRET_ACCESS_KEY +``` + +The generated `upload_script.sh` will contain the curl commands necessary to upload the files using presigned URLs. Share the `upload_script.sh` with the external collaborators, and they can execute it in the same folder as their files to upload them to your S3 account. + +
+ + + + + + +## Docker usage +**This is the recommended approach.**
+Here is a toy example of how you can use this script with just a docker command. +``` +docker run --rm -v $PWD/:/data/ rtibiocloud/s3_presigned_url_generator:v1_23c8ea4 \ + --infile /data/file_list.txt \ + --outfile /data/upload_script3.sh \ + --bucket rti-cool-project \ + --key-prefix scratch/some_rti_user/ \ + --expiration-days 7 \ + --aws-access-key AKIACCESSkeyEXAMPLE \ + --aws-secret-access-key qFyQSECRECTaccessKEYexample +``` +* _Note_: check the DockerHub rtibiocloud repository for the latest tag (i.e., replace `v1_23c8ea4` if necessary), and don't forget to change the access keys in this toy example. + +
+ + + + + + +## Using the Upload Script + +The generated `upload_script.sh` contains the necessary `curl` commands to upload files to the S3 location using presigned URLs. To use the script, follow these steps: + +1. Ensure that you have the `upload_script.sh` and the files you want to upload in the same directory. +2. Open a terminal and navigate to the directory containing the `upload_script.sh` and the files. +3. Make the `upload_script.sh` file executable.`chmod +x upload_script.sh` +4. Execute the `upload_script.sh` script. `./upload_script.sh` + +The script will start executing the `curl` commands, uploading each file to the specified S3 location using the presigned URLs. + +_Note_: Depending on the number and size of the files, the upload process may take some time. Monitor the progress in the terminal. +Once the script finishes executing, all the files should be successfully uploaded to the S3 bucket and location specified in the script. + +
+ + + + + +## Communicating with Collaborators + +To ensure the successful upload of files by external collaborators, it is recommended to communicate with them and provide necessary instructions. Here's a template for an email you can send to collaborators: + +
+ mock email + +
+ + **Subject**: Uploading files to [Your Project Name] - Action Required + +Dear Collaborator, + +We are excited to work with you on [Your Project Name]. As part of our collaboration, we kindly request you to upload your files to our Amazon S3 bucket using the provided presigned URLs. This process ensures secure and efficient file transfers without requiring separate AWS accounts. + +Here are the steps to upload your files: + +1. Place the attached `upload_script.sh` file in the same directory as the files you want to upload. + +2. Open a terminal and navigate to the directory containing the `upload_script.sh` and your files. + +3. Execute the `upload_script.sh` script: + ```shell + bash upload_script.sh + ``` + +This will start the upload process. The script will automatically upload your files to our S3 bucket using presigned URLs. +Once the upload is complete, please reply to this email with the MD5 checksum for each uploaded file. This will allow us to verify the integrity of the transferred files. + +If you encounter any issues or have any questions during the upload process, please feel free to reach out to us. We are here to assist you. + +Thank you for your collaboration! + +Best regards,
+[Your Name]
+[Your Organization] +
+ + +
+ + + + + + +## Limitations +When using AWS presigned URLs, there is a limitation of 5GB for file uploads ([reference](https://docs.aws.amazon.com/AmazonS3/latest/userguide/upload-objects.html)). If your file size exceeds this limit, you will need to consider alternative methods or break down the file into smaller parts to accommodate the restriction. + + + +

+___ + + + + + + +## Support +For support or any questions, please reach out to Jesse Marks (jmarks@rti.org) diff --git a/s3_presigned_url_generator_jesse/v1/requirements.txt b/s3_presigned_url_generator_jesse/v1/requirements.txt new file mode 100644 index 0000000..bf892f6 --- /dev/null +++ b/s3_presigned_url_generator_jesse/v1/requirements.txt @@ -0,0 +1 @@ +boto3==1.24.28 diff --git a/s3_presigned_url_generator_jesse/v1/s3_presigned_upload.py b/s3_presigned_url_generator_jesse/v1/s3_presigned_upload.py new file mode 100644 index 0000000..410704e --- /dev/null +++ b/s3_presigned_url_generator_jesse/v1/s3_presigned_upload.py @@ -0,0 +1,99 @@ +import argparse +import boto3 + + +def generate_presigned_urls(infile, outfile, bucket, key_prefix, expiration_days, access_key, secret_access_key): + """ + Generate a bash script containing curl commands with presigned URLs for uploading files to S3. + + This script takes an input file containing a list of file names, and for each file, it generates a presigned URL + using the provided AWS credentials. The presigned URL allows external collaborators to upload their files to the + specified S3 bucket using curl commands. The generated curl commands are written to the output file as a bash script. + + Args: + infile (str): Path to the input file containing the list of file names to generate presigned URLs for. + outfile (str): Path to the output bash script file that will contain the generated curl commands. + bucket (str): Name of the S3 bucket where the files will be uploaded. + key_prefix (str): Prefix to be prepended to each file name as the S3 object key. + expiration_days (int): Expiration duration in days for the generated presigned URLs. + access_key (str): AWS access key to be used for authentication. + secret_access_key (str): AWS secret access key to be used for authentication. + + Example: + Let's assume you have an input file named 'file_list.txt' containing the following filenames: + ``` + file1.txt + file2.jpg + file3.pdf + ``` + + You want to generate a bash script named 'upload_script.sh' that will contain the curl commands with presigned + URLs for uploading these files to the S3 bucket 'my-bucket' with the key prefix 'uploads/' and a URL expiration + of 7 days. + + You can execute the script as follows: + ``` + python s3_presigned_upload.py \ + --infile file_list.txt \ + --outfile upload_script.sh \ + --bucket my-bucket \ + --key-prefix uploads/ \ + --expiration-days 7 \ + --aws-access-key YOUR_ACCESS_KEY \ + --aws-secret-access-key YOUR_SECRET_ACCESS_KEY + ``` + + The generated 'upload_script.sh' will contain the curl commands to upload the files using presigned URLs. + You can share the 'upload_script.sh' with the external collaborators, and they can execute it in the same + folder as their files to upload them to your S3 account. + """ + + session = boto3.Session(aws_access_key_id=access_key, aws_secret_access_key=secret_access_key) + s3 = session.client("s3") + + with open(infile) as inF, open(outfile, "w") as outF: + outF.write("#!/bin/bash\n\n") + line = inF.readline() + + while line: + seconds = expiration_days * 60 * 60 * 24 + + key = "{}{}".format(key_prefix, line.strip()) + outurl = s3.generate_presigned_url( + 'put_object', + Params={'Bucket': bucket, 'Key': key}, + ExpiresIn=seconds, + HttpMethod='PUT' + ) + + outline1 = "##{}".format(line) # comment line + outline2 = "curl --request PUT --upload-file {} '{}'\n\n".format(line.strip(), outurl) + + outF.write(outline1) + outF.write(outline2) + line = inF.readline() + + outF.write("echo 'File(s) successfully uploaded to S3!'") + print(f"\n\nSuccess!\nCreated the bash script '{outfile}' for uploading files to S3 via presigned URLs.") + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Generate presigned URLs for S3 objects") + parser.add_argument("--infile", required=True, help="Input file path") + parser.add_argument("--outfile", required=True, help="Output file path") + parser.add_argument("--bucket", required=True, help="S3 bucket name") + parser.add_argument("--key-prefix", "-k", dest="key_prefix", required=True, help="S3 key prefix") + parser.add_argument("--expiration-days", "-e", dest="expiration_days", type=int, help="URL expiration in days") + parser.add_argument("--aws-access-key","-a", dest="access_key", required=True, type=str, help="AWS access key ID") + parser.add_argument("--aws-secret-access-key", "-s", dest="secret_access_key", required=True, type=str, help="AWS secret access key") + + args = parser.parse_args() + + generate_presigned_urls( + args.infile, + args.outfile, + args.bucket, + args.key_prefix, + args.expiration_days, + args.access_key, + args.secret_access_key + ) From 2f6cea00830b37f6e5212e5dda37507a3ebd4d44 Mon Sep 17 00:00:00 2001 From: jaamarks Date: Mon, 26 Feb 2024 14:30:41 -0500 Subject: [PATCH 08/27] cleanup empty directory in cov-ldsc --- cov_ldsc/v1/cov-LDSC/cov-ldsc | 1 - 1 file changed, 1 deletion(-) delete mode 160000 cov_ldsc/v1/cov-LDSC/cov-ldsc diff --git a/cov_ldsc/v1/cov-LDSC/cov-ldsc b/cov_ldsc/v1/cov-LDSC/cov-ldsc deleted file mode 160000 index 8cd5abe..0000000 --- a/cov_ldsc/v1/cov-LDSC/cov-ldsc +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8cd5abe2bcb479cceae25ba2e4387eb6428ca6b1 From 128e9f13e91287a273b9f1f040b28ca41d0ce24e Mon Sep 17 00:00:00 2001 From: jaamarks Date: Mon, 26 Feb 2024 14:32:43 -0500 Subject: [PATCH 09/27] change from alpine 12 to 11 for testing purposes --- s3_presigned_url_generator_jesse/v1/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/s3_presigned_url_generator_jesse/v1/Dockerfile b/s3_presigned_url_generator_jesse/v1/Dockerfile index 3d928b3..b344622 100644 --- a/s3_presigned_url_generator_jesse/v1/Dockerfile +++ b/s3_presigned_url_generator_jesse/v1/Dockerfile @@ -1,10 +1,10 @@ # Use an official Python runtime as the base image -FROM python:3.12-alpine +FROM python:3.11-alpine # Add Container Labels LABEL maintainer="Jesse Marks " LABEL description="A script to generate presigned URLs to upload to S3." -LABEL base-image="python:3.12-alpine" +LABEL base-image="python:3.11-alpine" # Install System Dependencies RUN apt-get update && apt-get install -y \ From 90e12d882b129cc91e5c6c3acd06cbed42f51090 Mon Sep 17 00:00:00 2001 From: jaamarks Date: Mon, 26 Feb 2024 14:35:51 -0500 Subject: [PATCH 10/27] alpine to slim. apt install fails with alpine --- s3_presigned_url_generator_jesse/v1/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/s3_presigned_url_generator_jesse/v1/Dockerfile b/s3_presigned_url_generator_jesse/v1/Dockerfile index b344622..0dc0d5b 100644 --- a/s3_presigned_url_generator_jesse/v1/Dockerfile +++ b/s3_presigned_url_generator_jesse/v1/Dockerfile @@ -1,10 +1,10 @@ # Use an official Python runtime as the base image -FROM python:3.11-alpine +FROM python:3.12-slim # Add Container Labels LABEL maintainer="Jesse Marks " LABEL description="A script to generate presigned URLs to upload to S3." -LABEL base-image="python:3.11-alpine" +LABEL base-image="python:3.12-slim" # Install System Dependencies RUN apt-get update && apt-get install -y \ From 21ae8aa2897a8555ed6d8db0afbf4ac6cf47cd72 Mon Sep 17 00:00:00 2001 From: jaamarks Date: Mon, 26 Feb 2024 15:00:35 -0500 Subject: [PATCH 11/27] set env var instead of deprecated save-output: and test --- .github/workflows/dockerimage.yml | 2 +- s3_presigned_url_generator_jesse/v1/Dockerfile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index 5ce21b1..0154e95 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -29,7 +29,7 @@ jobs: - name: get changed files id: getfile run: | - echo "::set-output name=files::$(git diff-tree --no-commit-id --name-only -r ${{ github.sha }} | xargs)" + echo "files=$(git diff-tree --no-commit-id --name-only -r ${{ github.sha }} | xargs)" >> "$GITHUB_ENV" - name: Build, Tag, Publish Docker uses: ./.github/actions/build-image with: diff --git a/s3_presigned_url_generator_jesse/v1/Dockerfile b/s3_presigned_url_generator_jesse/v1/Dockerfile index 0dc0d5b..b32053e 100644 --- a/s3_presigned_url_generator_jesse/v1/Dockerfile +++ b/s3_presigned_url_generator_jesse/v1/Dockerfile @@ -1,10 +1,10 @@ # Use an official Python runtime as the base image -FROM python:3.12-slim +FROM python:3.11-slim # Add Container Labels LABEL maintainer="Jesse Marks " LABEL description="A script to generate presigned URLs to upload to S3." -LABEL base-image="python:3.12-slim" +LABEL base-image="python:3.11-slim" # Install System Dependencies RUN apt-get update && apt-get install -y \ From 4fe9f067e2523055ed2f75f92dbe0dbb01052dc8 Mon Sep 17 00:00:00 2001 From: jaamarks Date: Mon, 26 Feb 2024 15:05:21 -0500 Subject: [PATCH 12/27] try with env.files --- .github/workflows/dockerimage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index 0154e95..2a0b09e 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -34,6 +34,6 @@ jobs: uses: ./.github/actions/build-image with: organization: jessemarks - changed_files: ${{ steps.getfile.outputs.files }} + changed_files: ${{ env.files }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} From 45a104be7e270c1be24a943584307b138c846928 Mon Sep 17 00:00:00 2001 From: jaamarks Date: Mon, 26 Feb 2024 15:06:13 -0500 Subject: [PATCH 13/27] test 001 --- s3_presigned_url_generator_jesse/v1/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/s3_presigned_url_generator_jesse/v1/Dockerfile b/s3_presigned_url_generator_jesse/v1/Dockerfile index b32053e..fed730b 100644 --- a/s3_presigned_url_generator_jesse/v1/Dockerfile +++ b/s3_presigned_url_generator_jesse/v1/Dockerfile @@ -1,10 +1,9 @@ # Use an official Python runtime as the base image -FROM python:3.11-slim +FROM python:3.12-slim # Add Container Labels LABEL maintainer="Jesse Marks " LABEL description="A script to generate presigned URLs to upload to S3." -LABEL base-image="python:3.11-slim" # Install System Dependencies RUN apt-get update && apt-get install -y \ From 0af97815f0a04ad4338a2b818668e06038253d30 Mon Sep 17 00:00:00 2001 From: jaamarks Date: Mon, 26 Feb 2024 15:12:43 -0500 Subject: [PATCH 14/27] test 002 --- s3_presigned_url_generator_jesse/v1/Dockerfile | 2 +- s3_presigned_url_generator_jesse/v1/s3_presigned_upload.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/s3_presigned_url_generator_jesse/v1/Dockerfile b/s3_presigned_url_generator_jesse/v1/Dockerfile index fed730b..c3934b0 100644 --- a/s3_presigned_url_generator_jesse/v1/Dockerfile +++ b/s3_presigned_url_generator_jesse/v1/Dockerfile @@ -1,5 +1,5 @@ # Use an official Python runtime as the base image -FROM python:3.12-slim +FROM python:3.11-slim # Add Container Labels LABEL maintainer="Jesse Marks " diff --git a/s3_presigned_url_generator_jesse/v1/s3_presigned_upload.py b/s3_presigned_url_generator_jesse/v1/s3_presigned_upload.py index 410704e..a0a6b4e 100644 --- a/s3_presigned_url_generator_jesse/v1/s3_presigned_upload.py +++ b/s3_presigned_url_generator_jesse/v1/s3_presigned_upload.py @@ -1,7 +1,6 @@ import argparse import boto3 - def generate_presigned_urls(infile, outfile, bucket, key_prefix, expiration_days, access_key, secret_access_key): """ Generate a bash script containing curl commands with presigned URLs for uploading files to S3. @@ -88,6 +87,7 @@ def generate_presigned_urls(infile, outfile, bucket, key_prefix, expiration_days args = parser.parse_args() + # comment for testing generate_presigned_urls( args.infile, args.outfile, From 3065a5708087c8d36ee3c4f0e0194d4dbb297862 Mon Sep 17 00:00:00 2001 From: jaamarks Date: Mon, 26 Feb 2024 16:30:47 -0500 Subject: [PATCH 15/27] convert to environmental variables --- .github/actions/build-image/entrypoint.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/actions/build-image/entrypoint.sh b/.github/actions/build-image/entrypoint.sh index ed227db..c06bff5 100755 --- a/.github/actions/build-image/entrypoint.sh +++ b/.github/actions/build-image/entrypoint.sh @@ -92,9 +92,10 @@ function main() { push - echo "::set-output name=tag::${FIRST_TAG}" + # Write the outputs to environment variables + echo "tag=${FIRST_TAG}" >> "$GITHUB_ENV" DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' ${DOCKERNAME}) - echo "::set-output name=digest::${DIGEST}" + echo "digest=${DIGEST}" >> "$GITHUB_ENV" docker logout } From f4f7fa8c6766370ab3dee3b665b98d07709978c3 Mon Sep 17 00:00:00 2001 From: jaamarks Date: Mon, 26 Feb 2024 16:31:35 -0500 Subject: [PATCH 16/27] test 003 --- s3_presigned_url_generator_jesse/v1/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3_presigned_url_generator_jesse/v1/Dockerfile b/s3_presigned_url_generator_jesse/v1/Dockerfile index c3934b0..fed730b 100644 --- a/s3_presigned_url_generator_jesse/v1/Dockerfile +++ b/s3_presigned_url_generator_jesse/v1/Dockerfile @@ -1,5 +1,5 @@ # Use an official Python runtime as the base image -FROM python:3.11-slim +FROM python:3.12-slim # Add Container Labels LABEL maintainer="Jesse Marks " From 11ba035d3f4e585ef6ab5d88bca17e4c5c90c785 Mon Sep 17 00:00:00 2001 From: jaamarks Date: Mon, 26 Feb 2024 16:36:58 -0500 Subject: [PATCH 17/27] use env vars test --- .github/actions/build-image/action.yml | 3 --- .github/actions/build-image/entrypoint.sh | 4 ++-- .github/workflows/dockerimage.yml | 1 - 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/actions/build-image/action.yml b/.github/actions/build-image/action.yml index 405dcda..6973a3c 100644 --- a/.github/actions/build-image/action.yml +++ b/.github/actions/build-image/action.yml @@ -5,9 +5,6 @@ branding: color: 'green' description: 'Builds the specified Dockerfile and pushes the image to Docker Hub.' inputs: - changed_files: - description: 'The files changed in the triggering commit.' - required: true username: description: 'The login username for the registry' required: true diff --git a/.github/actions/build-image/entrypoint.sh b/.github/actions/build-image/entrypoint.sh index c06bff5..bad77d6 100755 --- a/.github/actions/build-image/entrypoint.sh +++ b/.github/actions/build-image/entrypoint.sh @@ -8,10 +8,10 @@ function main() { sanitize "${INPUT_USERNAME}" "username" sanitize "${INPUT_PASSWORD}" "password" sanitize "${INPUT_ORGANIZATION}" "organization" - sanitize "${INPUT_CHANGED_FILES}" "changed_files" + sanitize "${files}" "changed_files" # CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r ${GITHUB_SHA}) # dfe37af2c9a8c753fcd6392ea2f5e711a04b38e1 - CHANGED_FILES="${INPUT_CHANGED_FILES}" + CHANGED_FILES="${files}" # Can only build 1 Docker image in 1 actions run/commit if [[ $(echo $CHANGED_FILES | tr " " "\n" | grep -c "Dockerfile") -gt 1 ]]; then diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index 2a0b09e..801a492 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -34,6 +34,5 @@ jobs: uses: ./.github/actions/build-image with: organization: jessemarks - changed_files: ${{ env.files }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} From 98c54a2e172f8d793c1dfbaf3fb89aaaed25ac5a Mon Sep 17 00:00:00 2001 From: jaamarks Date: Mon, 26 Feb 2024 16:38:45 -0500 Subject: [PATCH 18/27] test004 --- s3_presigned_url_generator_jesse/v1/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3_presigned_url_generator_jesse/v1/Dockerfile b/s3_presigned_url_generator_jesse/v1/Dockerfile index fed730b..c3934b0 100644 --- a/s3_presigned_url_generator_jesse/v1/Dockerfile +++ b/s3_presigned_url_generator_jesse/v1/Dockerfile @@ -1,5 +1,5 @@ # Use an official Python runtime as the base image -FROM python:3.12-slim +FROM python:3.11-slim # Add Container Labels LABEL maintainer="Jesse Marks " From 25b56d53135722bb3aa3e5ac1a5ecd2fab31a612 Mon Sep 17 00:00:00 2001 From: jaamarks Date: Mon, 26 Feb 2024 16:42:51 -0500 Subject: [PATCH 19/27] test 005 --- s3_presigned_url_generator_jesse/v1/Dockerfile | 4 +--- s3_presigned_url_generator_jesse/v1/s3_presigned_upload.py | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/s3_presigned_url_generator_jesse/v1/Dockerfile b/s3_presigned_url_generator_jesse/v1/Dockerfile index c3934b0..b25d6d6 100644 --- a/s3_presigned_url_generator_jesse/v1/Dockerfile +++ b/s3_presigned_url_generator_jesse/v1/Dockerfile @@ -1,5 +1,5 @@ # Use an official Python runtime as the base image -FROM python:3.11-slim +FROM python:3.12-slim # Add Container Labels LABEL maintainer="Jesse Marks " @@ -8,8 +8,6 @@ LABEL description="A script to generate presigned URLs to upload to S3." # Install System Dependencies RUN apt-get update && apt-get install -y \ vim \ - less \ - curl \ && rm -rf /var/lib/apt/lists/* # Set the working directory in the container diff --git a/s3_presigned_url_generator_jesse/v1/s3_presigned_upload.py b/s3_presigned_url_generator_jesse/v1/s3_presigned_upload.py index a0a6b4e..d2a9e6c 100644 --- a/s3_presigned_url_generator_jesse/v1/s3_presigned_upload.py +++ b/s3_presigned_url_generator_jesse/v1/s3_presigned_upload.py @@ -87,7 +87,6 @@ def generate_presigned_urls(infile, outfile, bucket, key_prefix, expiration_days args = parser.parse_args() - # comment for testing generate_presigned_urls( args.infile, args.outfile, From 5cf9c90345c558cd7df68cc5e1815c9950f8cff6 Mon Sep 17 00:00:00 2001 From: jaamarks Date: Mon, 26 Feb 2024 17:17:01 -0500 Subject: [PATCH 20/27] change diff-tree to diff --- .github/workflows/dockerimage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index 801a492..9d78a34 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -29,7 +29,7 @@ jobs: - name: get changed files id: getfile run: | - echo "files=$(git diff-tree --no-commit-id --name-only -r ${{ github.sha }} | xargs)" >> "$GITHUB_ENV" + echo "files=$(git diff --no-commit-id --name-only ${{ github.sha }} | xargs)" >> "$GITHUB_ENV" - name: Build, Tag, Publish Docker uses: ./.github/actions/build-image with: From f4e7c3fc893e3a873376ab9d626c28bd2cd976ec Mon Sep 17 00:00:00 2001 From: jaamarks Date: Mon, 26 Feb 2024 17:17:43 -0500 Subject: [PATCH 21/27] Revert "change diff-tree to diff" This reverts commit 5cf9c90345c558cd7df68cc5e1815c9950f8cff6. --- .github/workflows/dockerimage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index 9d78a34..801a492 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -29,7 +29,7 @@ jobs: - name: get changed files id: getfile run: | - echo "files=$(git diff --no-commit-id --name-only ${{ github.sha }} | xargs)" >> "$GITHUB_ENV" + echo "files=$(git diff-tree --no-commit-id --name-only -r ${{ github.sha }} | xargs)" >> "$GITHUB_ENV" - name: Build, Tag, Publish Docker uses: ./.github/actions/build-image with: From 7397fe93e5b5c5f80a54a959121ac32da3956432 Mon Sep 17 00:00:00 2001 From: jaamarks Date: Mon, 26 Feb 2024 17:19:56 -0500 Subject: [PATCH 22/27] test 006 --- .github/workflows/dockerimage.yml | 2 +- s3_presigned_url_generator_jesse/v1/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index 801a492..9d78a34 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -29,7 +29,7 @@ jobs: - name: get changed files id: getfile run: | - echo "files=$(git diff-tree --no-commit-id --name-only -r ${{ github.sha }} | xargs)" >> "$GITHUB_ENV" + echo "files=$(git diff --no-commit-id --name-only ${{ github.sha }} | xargs)" >> "$GITHUB_ENV" - name: Build, Tag, Publish Docker uses: ./.github/actions/build-image with: diff --git a/s3_presigned_url_generator_jesse/v1/Dockerfile b/s3_presigned_url_generator_jesse/v1/Dockerfile index b25d6d6..a643acc 100644 --- a/s3_presigned_url_generator_jesse/v1/Dockerfile +++ b/s3_presigned_url_generator_jesse/v1/Dockerfile @@ -1,5 +1,5 @@ # Use an official Python runtime as the base image -FROM python:3.12-slim +FROM python:3.11-slim # Add Container Labels LABEL maintainer="Jesse Marks " From 76ddec88f05338dc2295e2f01b95ef5dca2cdd8e Mon Sep 17 00:00:00 2001 From: jaamarks Date: Mon, 26 Feb 2024 17:24:28 -0500 Subject: [PATCH 23/27] test 007 --- .github/workflows/dockerimage.yml | 2 +- s3_presigned_url_generator_jesse/v1/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index 9d78a34..e9cebba 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -29,7 +29,7 @@ jobs: - name: get changed files id: getfile run: | - echo "files=$(git diff --no-commit-id --name-only ${{ github.sha }} | xargs)" >> "$GITHUB_ENV" + echo "files=$(git diff --no-commit-id --name-only ${{ github.event.before }} ${{ github.sha }} | xargs)" >> "$GITHUB_ENV" - name: Build, Tag, Publish Docker uses: ./.github/actions/build-image with: diff --git a/s3_presigned_url_generator_jesse/v1/Dockerfile b/s3_presigned_url_generator_jesse/v1/Dockerfile index a643acc..b25d6d6 100644 --- a/s3_presigned_url_generator_jesse/v1/Dockerfile +++ b/s3_presigned_url_generator_jesse/v1/Dockerfile @@ -1,5 +1,5 @@ # Use an official Python runtime as the base image -FROM python:3.11-slim +FROM python:3.12-slim # Add Container Labels LABEL maintainer="Jesse Marks " From 5ecca1d3f68d392b31bf1f7c9efdd7a7d806c2e2 Mon Sep 17 00:00:00 2001 From: jaamarks Date: Mon, 26 Feb 2024 17:27:57 -0500 Subject: [PATCH 24/27] test008 --- .github/workflows/dockerimage.yml | 2 +- s3_presigned_url_generator_jesse/v1/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index e9cebba..801a492 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -29,7 +29,7 @@ jobs: - name: get changed files id: getfile run: | - echo "files=$(git diff --no-commit-id --name-only ${{ github.event.before }} ${{ github.sha }} | xargs)" >> "$GITHUB_ENV" + echo "files=$(git diff-tree --no-commit-id --name-only -r ${{ github.sha }} | xargs)" >> "$GITHUB_ENV" - name: Build, Tag, Publish Docker uses: ./.github/actions/build-image with: diff --git a/s3_presigned_url_generator_jesse/v1/Dockerfile b/s3_presigned_url_generator_jesse/v1/Dockerfile index b25d6d6..a643acc 100644 --- a/s3_presigned_url_generator_jesse/v1/Dockerfile +++ b/s3_presigned_url_generator_jesse/v1/Dockerfile @@ -1,5 +1,5 @@ # Use an official Python runtime as the base image -FROM python:3.12-slim +FROM python:3.11-slim # Add Container Labels LABEL maintainer="Jesse Marks " From 212d0f873c33cb9f0bec61e1d6c102d5e36813ab Mon Sep 17 00:00:00 2001 From: jaamarks Date: Mon, 26 Feb 2024 17:29:03 -0500 Subject: [PATCH 25/27] 09 --- s3_presigned_url_generator_jesse/v1/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3_presigned_url_generator_jesse/v1/Dockerfile b/s3_presigned_url_generator_jesse/v1/Dockerfile index a643acc..b25d6d6 100644 --- a/s3_presigned_url_generator_jesse/v1/Dockerfile +++ b/s3_presigned_url_generator_jesse/v1/Dockerfile @@ -1,5 +1,5 @@ # Use an official Python runtime as the base image -FROM python:3.11-slim +FROM python:3.12-slim # Add Container Labels LABEL maintainer="Jesse Marks " From d3899e0e8b99564b8c009b6da51c0c6015b5d880 Mon Sep 17 00:00:00 2001 From: jaamarks Date: Mon, 26 Feb 2024 17:30:28 -0500 Subject: [PATCH 26/27] test010 --- .github/workflows/dockerimage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml index 801a492..0a1049b 100644 --- a/.github/workflows/dockerimage.yml +++ b/.github/workflows/dockerimage.yml @@ -29,7 +29,7 @@ jobs: - name: get changed files id: getfile run: | - echo "files=$(git diff-tree --no-commit-id --name-only -r ${{ github.sha }} | xargs)" >> "$GITHUB_ENV" + echo "files=$(git diff --no-commit-id --name-only ${{ github.event.before }} ${{ github.sha }} | xargs)" >> "$GITHUB_ENV" - name: Build, Tag, Publish Docker uses: ./.github/actions/build-image with: From d29dce94f9d83345fb857bb1af46c609a2f9e699 Mon Sep 17 00:00:00 2001 From: jaamarks Date: Mon, 26 Feb 2024 17:31:00 -0500 Subject: [PATCH 27/27] test 011 --- s3_presigned_url_generator_jesse/v1/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3_presigned_url_generator_jesse/v1/Dockerfile b/s3_presigned_url_generator_jesse/v1/Dockerfile index b25d6d6..a643acc 100644 --- a/s3_presigned_url_generator_jesse/v1/Dockerfile +++ b/s3_presigned_url_generator_jesse/v1/Dockerfile @@ -1,5 +1,5 @@ # Use an official Python runtime as the base image -FROM python:3.12-slim +FROM python:3.11-slim # Add Container Labels LABEL maintainer="Jesse Marks "