travis-ci: remove old branches, so things don't fill up

this will go through the files on the server, and remove old artifacts that aren't associated with an active branch or tag. Signed-off-by: Robin Getz <[email protected]>
ledoune · Jun 15, 2020 · eaaf588 · eaaf588
1 parent f9abc28
commit eaaf588
Show file tree

Hide file tree

Showing 2 changed files with 111 additions and 22 deletions.
diff --git a/CI/travis/deploy b/CI/travis/deploy
@@ -7,4 +7,4 @@ upload_file_to_swdownloads libiio ${RELEASE_PKG_FILE_DEB} ${TARGET_DEB} .deb
 upload_file_to_swdownloads libiio ${RELEASE_PKG_FILE_RPM} ${TARGET_RPM} .rpm
 upload_file_to_swdownloads libiio ${RELEASE_PKG_FILE_TGZ} ${TARGET_TGZ} .tar.gz
 upload_file_to_swdownloads libiio ${RELEASE_PKG_FILE_PKG} ${TARGET_PKG} .pkg
-
+remove_old_pkgs            libiio
diff --git a/CI/travis/lib.sh b/CI/travis/lib.sh
@@ -225,11 +225,27 @@ sftp_cmd_pipe() {
 	sftp -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}"
 }
 
+sftp_run_cmds() {
+	local x=5
+	while [ "${x}" -gt "0" ] ; do
+		sftp -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" 0< "$1" && break;
+		echo_red "failed to ssh, trying again"
+		x=$((x  - 1))
+		sleep 10
+	done
+	if [ "${x}" -eq "0" ] ; then
+		echo_red "failed to upload files"
+		return 1;
+	fi
+	return 0
+}
+
 sftp_rm_artifact() {
 	local artifact="$1"
 	sftp_cmd_pipe <<-EOF
 		cd ${DEPLOY_TO}
 		rm ${artifact}
+		bye
 	EOF
 }
 
@@ -297,33 +313,106 @@ upload_file_to_swdownloads() {
 	echo and "${branch}_${LIBNAME}${LDIST}${EXT}"
 	ssh -V
 
-	for rmf in ${TO} ${LATE} ; do
-		sftp_rm_artifact "${rmf}" || \
-			echo_blue "Could not delete ${rmf}"
-	done
+	local tmpfl=$(mktemp)
+	echo "cd ${DEPLOY_TO}" > "${tmpfl}"
+	echo "rm ${TO}" >> "${tmpfl}"
+	echo "rm ${LATE}" >> "${tmpfl}"
+	echo "put ${FROM} ${TO}" >> "${tmpfl}"
+	echo "symlink ${TO} ${LATE}" >> "${tmpfl}"
+	echo "ls -l ${TO}" >> "${tmpfl}"
+	echo "ls -l ${LATE}" >> "${tmpfl}"
+	echo "bye"  >> "${tmpfl}"
 
-	sftp_upload "${FROM}" "${TO}" "${LATE}" || {
-		echo_red "Failed to upload artifact from '${FROM}', to '${TO}', symlink '${LATE}'"
-		return 1
-	}
+	sftp_run_cmds "${tmpfl}"
+	rm "${tmpfl}"
+
+	return 0
+}
 
+remove_old_pkgs() {
 	# limit things to a few files, so things don't grow forever
 	# we only do this on one build so simultaneous builds don't clobber each other
-	if [ -n "${GH_DOC_TOKEN}" ] ; then
-		for files in $(ssh -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" \
+	if [ -z "${GH_DOC_TOKEN}" ] ; then
+		return 0
+	fi
+
+	if [ -z "${TRAVIS_BUILD_DIR}" ] ; then
+		echo "TRAVIS_BUILD_DIR not set"
+		return 0
+	fi
+
+	if [ ! -d "${TRAVIS_BUILD_DIR}/.git" ] ; then
+		echo "No ${TRAVIS_BUILD_DIR}/.git to operate git on"
+		return 0
+	fi
+
+	local LIBNAME=$1
+	local old=
+
+	echo "Remove old packages from ${LIBNAME}"
+
+	if [ -n "$TRAVIS_PULL_REQUEST_BRANCH" ] ; then
+		local branch="$TRAVIS_PULL_REQUEST_BRANCH"
+	else
+		local branch="$TRAVIS_BRANCH"
+	fi
+
+	local GLOB=${DEPLOY_TO}/${branch}_${LIBNAME}-*
+
+	# putting everything into a file, and connecting once decreases the chances
+	# for ssh issues, connections happen once, not every single file
+	local tmpfl=$(mktemp)
+	echo "cd ${DEPLOY_TO}" > "${tmpfl}"
+	for files in $(ssh -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" \
 			"ls -lt ${GLOB}" | tail -n +100 | awk '{print $NF}')
-		do
-			ssh -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" \
-				"rm ${DEPLOY_TO}/${files}" || \
-				return 1
-		done
-		# provide an index so people can find files.
-		ssh -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" \
-			"ls -lt ${DEPLOY_TO}" | grep ${LIBNAME} > ${LIBNAME}_index.html
-		sftp_rm_artifact "${LIBNAME}_index.html" || \
-			echo_blue "Could not delete ${LIBNAME}_index.html"
-		sftp_upload "${LIBNAME}_index.html" "${LIBNAME}_index.html"
+	do
+		echo "rm ${files}" >> "${tmpfl}"
+	done
+	echo "bye" >> "${tmpfl}"
+	# if it is only cd & bye, skip it
+	if [ "$(wc -l "${tmpfl}" | awk '{print $1}')" -gt "2" ] ; then
+		sftp_run_cmds "${tmpfl}"
 	fi
+	rm "${tmpfl}"
+	# provide an index so people can find files.
+	ssh -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" \
+			"ls -lt ${DEPLOY_TO}" | grep "${LIBNAME}" > "${LIBNAME}_index.html"
+	echo "ls captured"
+
+	echo "cd ${DEPLOY_TO}" > "${tmpfl}"
+	# prune old / removed branches, leave things are are tags/branches
+	for old in $(sed 's/-> .*$//' libiio_index.html | \
+			awk  '{print $NF}' | grep -v master | sort | \
+			sed "s/_libiio-0.[0-9][0-9].g[a-z0-9]*-/ %% /" | \
+			grep "%%" | awk '{print $1}' | sort -u)
+	do
+		if [ "$(git --git-dir "${TRAVIS_BUILD_DIR}/.git" ls-remote --heads origin "${old}" | wc -l)" -ne "0" ] ; then
+			echo "${old} is a branch"
+		else
+			if [ "$(git --git-dir "${TRAVIS_BUILD_DIR}/.git" ls-remote --tags origin "${old}" | wc -l)" -ne "0" ] ; then
+				echo "${old} is a tag"
+			else
+				echo "${old} can be removed"
+				echo "rm ${old}_${LIBNAME}-*" >> "${tmpfl}"
+				echo "rm ${old}_latest_${LIBNAME}-*" >> "${tmpfl}"
+				echo "rm ${old}_lastest_${LIBNAME}-*" >> "${tmpfl}"
+			fi
+		fi
+	done
+	# cap things at 15, so we don't exceed the time
+	sed -i 16q "${tmpfl}"
+	echo "bye" >> "${tmpfl}"
+	# if it is only cd & bye, skip it
+	if [ "$(wc -l "${tmpfl}" | awk '{print $1}')" -gt "2" ] ; then
+		sftp_run_cmds "${tmpfl}"
+	fi
+	rm "${tmpfl}"
+
+	#Now that we removed things, do it again
+	rm "${LIBNAME}_index.html"
+	ssh -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" \
+			"ls -lt ${DEPLOY_TO}" | grep "${LIBNAME}" > "${LIBNAME}_index.html"
+	sftp_upload "${LIBNAME}_index.html" "${LIBNAME}_index.html"
 
 	return 0
 }