Merge pull request #57 from matyasselmeci/pr/rsync-link-dest.SOFTWARE…

…-5728

Tell rsync to hardlink to already downloaded condor RPMs instead of downloading twice (SOFTWARE-5728)

bin/pull_condor_rpms.sh

@@ -2,18 +2,20 @@
 RSYNC_ROOT="rsync://rsync.cs.wisc.edu/htcondor"
 
 TAG=$1
-REPO_DIR=$2
-SOURCE_SET=$3
+NEW_REPO_DIR=$2
+CURRENT_REPO_DIR=$3
+SOURCE_SET=$4
 
 usage () {
-  echo "Usage: $(basename "$0") TAG REPO_DIR SOURCE_SET"
+  echo "Usage: $(basename "$0") TAG NEW_REPO_DIR CURRENT_REPO_DIR SOURCE_SET"
   echo "Where:"
   echo "  TAG is osg-SERIES-BRANCH-DVER-REPO"
   echo "  SERIES is: 23, 24, etc."
   echo "  DVER is: el8, el9, etc."
   echo "  BRANCH is: main or upcoming"
   echo "  REPO is: development, testing, or release"
-  echo "  REPO_DIR is the directory to output condor RPMs into"
+  echo "  NEW_REPO_DIR is the directory to output condor RPMs into"
+  echo "  CURRENT_REPO_DIR is the directory that may contain condor RPMs from the most recent successful run"
   echo "  SOURCE_SET is '' for rpms and 'SRPMS/' for srpms"
   exit 1
 }
@@ -33,7 +35,7 @@ repo_not_supported() {
     exit 1
 }
 
-[[ $# -eq 3 ]] || usage
+[[ $# -eq 4 ]] || usage
 
 # read series, branch, dver, and repo from the osg tag
 case $TAG in
@@ -67,7 +69,7 @@ esac
 
 # get every build available for that package from every applicable condor repo
 RSYNC_URL="$RSYNC_ROOT/$CONDOR_SERIES/$DVER/x86_64/$CONDOR_REPO/$SOURCE_SET*.rpm"
-echo "rsyncing $RSYNC_URL to $REPO_DIR"
-if ! rsync $RSYNC_URL $REPO_DIR ; then
+echo "rsyncing $RSYNC_URL to $NEW_REPO_DIR"
+if ! rsync --times $RSYNC_URL $NEW_REPO_DIR --link-dest $CURRENT_REPO_DIR; then
     echo "Warning: No packages found for $RSYNC_URL. Skipping"
 fi

bin/update_repo.sh

@@ -46,7 +46,9 @@ working_path="/usr/local/repo.working/osg/$SERIES/$DVER/$REPO"
 previous_path="/usr/local/repo.previous/osg/$SERIES/$DVER/$REPO"
 reponame=$TAG
 repo_working_path="$working_path/$reponame/$arch"
+repo_release_path="$release_path/$arch"
 repo_working_srpm_path="$working_path/$reponame/source/SRPMS"
+repo_release_srpm_path="$release_path/source/SRPMS"
 
 if test -d $release_path && grep -q $TAG $OSGTAGS.create-only ; then
   echo "Tag $TAG is create-only and already exists. Skipping"
@@ -62,14 +64,14 @@ if [ "$?" -ne "0" ]; then
 fi
 
 # Copy relevant htcondor rpms to the working directory, if any
-if pull_condor_rpms.sh $TAG $repo_working_path '' ; then
+if pull_condor_rpms.sh $TAG $repo_working_path $repo_release_path '' ; then
         # if htcondor rpms were copied, we need to regenerate the repo files
         createrepo --update $repo_working_path
         repoview $repo_working_path
 fi
 
 # Copy relevant htcondor srpms to the working directory, if any
-if pull_condor_rpms.sh $TAG $repo_working_srpm_path 'SRPMS/' ; then
+if pull_condor_rpms.sh $TAG $repo_working_srpm_path $repo_release_srpm_path 'SRPMS/' ; then
         # if htcondor srpms were copied, we need to regenerate the repo files
         createrepo --update $repo_working_srpm_path
         repoview $repo_working_srpm_path