From eaaf5885b4c6e8e04b41c78101f993e5bdb1429b Mon Sep 17 00:00:00 2001 From: Robin Getz Date: Fri, 12 Jun 2020 15:37:15 -0400 Subject: [PATCH] travis-ci: remove old branches, so things don't fill up this will go through the files on the server, and remove old artifacts that aren't associated with an active branch or tag. Signed-off-by: Robin Getz --- CI/travis/deploy | 2 +- CI/travis/lib.sh | 131 +++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 111 insertions(+), 22 deletions(-) diff --git a/CI/travis/deploy b/CI/travis/deploy index 63a3f60cd..aa4d086ba 100755 --- a/CI/travis/deploy +++ b/CI/travis/deploy @@ -7,4 +7,4 @@ upload_file_to_swdownloads libiio ${RELEASE_PKG_FILE_DEB} ${TARGET_DEB} .deb upload_file_to_swdownloads libiio ${RELEASE_PKG_FILE_RPM} ${TARGET_RPM} .rpm upload_file_to_swdownloads libiio ${RELEASE_PKG_FILE_TGZ} ${TARGET_TGZ} .tar.gz upload_file_to_swdownloads libiio ${RELEASE_PKG_FILE_PKG} ${TARGET_PKG} .pkg - +remove_old_pkgs libiio diff --git a/CI/travis/lib.sh b/CI/travis/lib.sh index 53c8cd13e..450177372 100644 --- a/CI/travis/lib.sh +++ b/CI/travis/lib.sh @@ -225,11 +225,27 @@ sftp_cmd_pipe() { sftp -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" } +sftp_run_cmds() { + local x=5 + while [ "${x}" -gt "0" ] ; do + sftp -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" 0< "$1" && break; + echo_red "failed to ssh, trying again" + x=$((x - 1)) + sleep 10 + done + if [ "${x}" -eq "0" ] ; then + echo_red "failed to upload files" + return 1; + fi + return 0 +} + sftp_rm_artifact() { local artifact="$1" sftp_cmd_pipe <<-EOF cd ${DEPLOY_TO} rm ${artifact} + bye EOF } @@ -297,33 +313,106 @@ upload_file_to_swdownloads() { echo and "${branch}_${LIBNAME}${LDIST}${EXT}" ssh -V - for rmf in ${TO} ${LATE} ; do - sftp_rm_artifact "${rmf}" || \ - echo_blue "Could not delete ${rmf}" - done + local tmpfl=$(mktemp) + echo "cd ${DEPLOY_TO}" > "${tmpfl}" + echo "rm ${TO}" >> "${tmpfl}" + echo "rm ${LATE}" >> "${tmpfl}" + echo "put ${FROM} ${TO}" >> "${tmpfl}" + echo "symlink ${TO} ${LATE}" >> "${tmpfl}" + echo "ls -l ${TO}" >> "${tmpfl}" + echo "ls -l ${LATE}" >> "${tmpfl}" + echo "bye" >> "${tmpfl}" - sftp_upload "${FROM}" "${TO}" "${LATE}" || { - echo_red "Failed to upload artifact from '${FROM}', to '${TO}', symlink '${LATE}'" - return 1 - } + sftp_run_cmds "${tmpfl}" + rm "${tmpfl}" + + return 0 +} +remove_old_pkgs() { # limit things to a few files, so things don't grow forever # we only do this on one build so simultaneous builds don't clobber each other - if [ -n "${GH_DOC_TOKEN}" ] ; then - for files in $(ssh -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" \ + if [ -z "${GH_DOC_TOKEN}" ] ; then + return 0 + fi + + if [ -z "${TRAVIS_BUILD_DIR}" ] ; then + echo "TRAVIS_BUILD_DIR not set" + return 0 + fi + + if [ ! -d "${TRAVIS_BUILD_DIR}/.git" ] ; then + echo "No ${TRAVIS_BUILD_DIR}/.git to operate git on" + return 0 + fi + + local LIBNAME=$1 + local old= + + echo "Remove old packages from ${LIBNAME}" + + if [ -n "$TRAVIS_PULL_REQUEST_BRANCH" ] ; then + local branch="$TRAVIS_PULL_REQUEST_BRANCH" + else + local branch="$TRAVIS_BRANCH" + fi + + local GLOB=${DEPLOY_TO}/${branch}_${LIBNAME}-* + + # putting everything into a file, and connecting once decreases the chances + # for ssh issues, connections happen once, not every single file + local tmpfl=$(mktemp) + echo "cd ${DEPLOY_TO}" > "${tmpfl}" + for files in $(ssh -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" \ "ls -lt ${GLOB}" | tail -n +100 | awk '{print $NF}') - do - ssh -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" \ - "rm ${DEPLOY_TO}/${files}" || \ - return 1 - done - # provide an index so people can find files. - ssh -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" \ - "ls -lt ${DEPLOY_TO}" | grep ${LIBNAME} > ${LIBNAME}_index.html - sftp_rm_artifact "${LIBNAME}_index.html" || \ - echo_blue "Could not delete ${LIBNAME}_index.html" - sftp_upload "${LIBNAME}_index.html" "${LIBNAME}_index.html" + do + echo "rm ${files}" >> "${tmpfl}" + done + echo "bye" >> "${tmpfl}" + # if it is only cd & bye, skip it + if [ "$(wc -l "${tmpfl}" | awk '{print $1}')" -gt "2" ] ; then + sftp_run_cmds "${tmpfl}" fi + rm "${tmpfl}" + # provide an index so people can find files. + ssh -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" \ + "ls -lt ${DEPLOY_TO}" | grep "${LIBNAME}" > "${LIBNAME}_index.html" + echo "ls captured" + + echo "cd ${DEPLOY_TO}" > "${tmpfl}" + # prune old / removed branches, leave things are are tags/branches + for old in $(sed 's/-> .*$//' libiio_index.html | \ + awk '{print $NF}' | grep -v master | sort | \ + sed "s/_libiio-0.[0-9][0-9].g[a-z0-9]*-/ %% /" | \ + grep "%%" | awk '{print $1}' | sort -u) + do + if [ "$(git --git-dir "${TRAVIS_BUILD_DIR}/.git" ls-remote --heads origin "${old}" | wc -l)" -ne "0" ] ; then + echo "${old} is a branch" + else + if [ "$(git --git-dir "${TRAVIS_BUILD_DIR}/.git" ls-remote --tags origin "${old}" | wc -l)" -ne "0" ] ; then + echo "${old} is a tag" + else + echo "${old} can be removed" + echo "rm ${old}_${LIBNAME}-*" >> "${tmpfl}" + echo "rm ${old}_latest_${LIBNAME}-*" >> "${tmpfl}" + echo "rm ${old}_lastest_${LIBNAME}-*" >> "${tmpfl}" + fi + fi + done + # cap things at 15, so we don't exceed the time + sed -i 16q "${tmpfl}" + echo "bye" >> "${tmpfl}" + # if it is only cd & bye, skip it + if [ "$(wc -l "${tmpfl}" | awk '{print $1}')" -gt "2" ] ; then + sftp_run_cmds "${tmpfl}" + fi + rm "${tmpfl}" + + #Now that we removed things, do it again + rm "${LIBNAME}_index.html" + ssh -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" \ + "ls -lt ${DEPLOY_TO}" | grep "${LIBNAME}" > "${LIBNAME}_index.html" + sftp_upload "${LIBNAME}_index.html" "${LIBNAME}_index.html" return 0 }