Merge pull request #1890 from cBioPortal/remove_methylation450k #266
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Zip studies on push to master branch | |
| on: | |
| # Trigger the workflow on push, | |
| # but only for the master branch | |
| push: | |
| branches: | |
| - master | |
| paths: | |
| - 'public/**' | |
| jobs: | |
| zip_studies: | |
| if: github.repository == 'cbioportal/datahub' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - id: files | |
| uses: jitterbit/get-changed-files@v1 | |
| - run: | | |
| echo "##[set-output name=unique_studies_list;]$(echo "${{ steps.files.outputs.all }}" | awk 'BEGIN { RS=" "; FS="/"; ORS=" " } { set[$2]++ } END { for (dir in set) { print dir } }')" | |
| exit 0 | |
| id: unique_studies | |
| - uses: actions/checkout@v2 | |
| - run: | | |
| # detecting modified studies and archiving them | |
| # create a list of all studies under 'public' | |
| echo creating ${STUDY_LIST_FILENAME} | |
| echo "[" > ${STUDY_LIST_FILENAME} | |
| first_study=1 | |
| find public -type d -mindepth 1 -maxdepth 1 | sort > ${STUDY_PATH_FILENAME} | |
| while read studypath ; do | |
| if [ ${first_study} -ne 1 ] ; then | |
| echo "," >> ${STUDY_LIST_FILENAME} | |
| fi | |
| studyname="${studypath:7}" | |
| echo -n "\"${studyname}\"" >> ${STUDY_LIST_FILENAME} | |
| first_study=0 | |
| done < ${STUDY_PATH_FILENAME} | |
| rm -f ${STUDY_PATH_FILENAME} | |
| echo "" >> ${STUDY_LIST_FILENAME} | |
| echo "]" >> ${STUDY_LIST_FILENAME} | |
| # copy study list file to S3 | |
| echo copying ${STUDY_LIST_FILENAME} to S3 | |
| aws s3 cp --acl public-read ${STUDY_LIST_FILENAME} s3://${AWS_S3_BUCKET}/ | |
| rm -f ${STUDY_LIST_FILENAME} | |
| # first save .git | |
| tar -cvf ${SAVED_GIT_REPO_FILENAME} .git > /dev/null | |
| mkdir studies_to_upload | |
| cd public | |
| for changed_study in ${{ steps.unique_studies.outputs.unique_studies_list }}; do | |
| if [ ! -d "${changed_study}" ]; then | |
| continue | |
| fi | |
| echo "Pulling study: ${changed_study}." | |
| git lfs pull --include="${changed_study}" | |
| echo "Compressing this study: ${changed_study}." | |
| tar -czvf ../studies_to_upload/${changed_study}.tar.gz ${changed_study} | |
| echo "Copy file to S3: ../studies_to_upload/${changed_study}.tar.gz" | |
| aws s3 cp --acl public-read ../studies_to_upload/${changed_study}.tar.gz s3://${AWS_S3_BUCKET}/ | |
| echo "Remove this directory: ${changed_study}." | |
| rm -rf "${changed_study}" | |
| echo "Remove this file: ../studies_to_upload/${changed_study}.tar.gz." | |
| rm ../studies_to_upload/${changed_study}.tar.gz | |
| echo "Remove .git" | |
| cd .. | |
| rm -rf .git | |
| # add original .git back | |
| tar -xf ${SAVED_GIT_REPO_FILENAME} > /dev/null | |
| cd public | |
| done | |
| cd .. | |
| env: | |
| STUDY_PATH_FILENAME: study_paths.txt | |
| STUDY_LIST_FILENAME: study_list.json | |
| SAVED_GIT_REPO_FILENAME: save_git.tar | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} | |
| AWS_EC2_METADATA_DISABLED: "true" # fixes aws cli error (code 255); needed since ubuntu-latest v2.262.1 |