From dfd9a5f296cd45762afd9b9e098d293b12dc5193 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Thu, 20 Jun 2024 10:35:28 -0700 Subject: [PATCH] Switch to upstream mirror for scraping version numbers Currently, we have a 12.4 release that's published, but not (yet) on the kernel.org mirror, so our version scraping fails to pick it up. While I feel less bad about scraping the kernel.org mirror load-wise, I think it's probably reasonable for our only-a-couple-times-daily version scraping script to poke the official upstream server to find the official upstream releases without relying on them having propagated to the mirrors yet (especially since it's not downloading the large bits -- just the HTML file listing). (Also add it as a very low priority fallback during download just in case it's the literal only site which has the version we're updating to.) --- 10/Dockerfile | 2 ++ 11/Dockerfile | 2 ++ 12/Dockerfile | 2 ++ 13/Dockerfile | 2 ++ 14/Dockerfile | 2 ++ Dockerfile.template | 2 ++ versions.sh | 7 +++---- 7 files changed, 15 insertions(+), 4 deletions(-) diff --git a/10/Dockerfile b/10/Dockerfile index 7d70e2001e0..48ff42699ae 100644 --- a/10/Dockerfile +++ b/10/Dockerfile @@ -38,6 +38,8 @@ ENV GCC_MIRRORS \ https://bigsearcher.com/mirrors/gcc/releases \ http://www.netgull.com/gcc/releases \ https://ftpmirror.gnu.org/gcc \ +# "sourceware.org" is the canonical upstream release host (the host of "gcc.gnu.org") + https://sourceware.org/pub/gcc/releases \ # only attempt the origin FTP as a mirror of last resort ftp://ftp.gnu.org/gnu/gcc diff --git a/11/Dockerfile b/11/Dockerfile index 38edef25d66..5a7f9c42b95 100644 --- a/11/Dockerfile +++ b/11/Dockerfile @@ -38,6 +38,8 @@ ENV GCC_MIRRORS \ https://bigsearcher.com/mirrors/gcc/releases \ http://www.netgull.com/gcc/releases \ https://ftpmirror.gnu.org/gcc \ +# "sourceware.org" is the canonical upstream release host (the host of "gcc.gnu.org") + https://sourceware.org/pub/gcc/releases \ # only attempt the origin FTP as a mirror of last resort ftp://ftp.gnu.org/gnu/gcc diff --git a/12/Dockerfile b/12/Dockerfile index 9a2a008c3be..2815d1e427c 100644 --- a/12/Dockerfile +++ b/12/Dockerfile @@ -38,6 +38,8 @@ ENV GCC_MIRRORS \ https://bigsearcher.com/mirrors/gcc/releases \ http://www.netgull.com/gcc/releases \ https://ftpmirror.gnu.org/gcc \ +# "sourceware.org" is the canonical upstream release host (the host of "gcc.gnu.org") + https://sourceware.org/pub/gcc/releases \ # only attempt the origin FTP as a mirror of last resort ftp://ftp.gnu.org/gnu/gcc diff --git a/13/Dockerfile b/13/Dockerfile index df9ad6e20eb..ee8540b1a45 100644 --- a/13/Dockerfile +++ b/13/Dockerfile @@ -38,6 +38,8 @@ ENV GCC_MIRRORS \ https://bigsearcher.com/mirrors/gcc/releases \ http://www.netgull.com/gcc/releases \ https://ftpmirror.gnu.org/gcc \ +# "sourceware.org" is the canonical upstream release host (the host of "gcc.gnu.org") + https://sourceware.org/pub/gcc/releases \ # only attempt the origin FTP as a mirror of last resort ftp://ftp.gnu.org/gnu/gcc diff --git a/14/Dockerfile b/14/Dockerfile index a6ee3fa3ffa..37882cc8227 100644 --- a/14/Dockerfile +++ b/14/Dockerfile @@ -38,6 +38,8 @@ ENV GCC_MIRRORS \ https://bigsearcher.com/mirrors/gcc/releases \ http://www.netgull.com/gcc/releases \ https://ftpmirror.gnu.org/gcc \ +# "sourceware.org" is the canonical upstream release host (the host of "gcc.gnu.org") + https://sourceware.org/pub/gcc/releases \ # only attempt the origin FTP as a mirror of last resort ftp://ftp.gnu.org/gnu/gcc diff --git a/Dockerfile.template b/Dockerfile.template index e8032e6e279..1f3703c84aa 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -32,6 +32,8 @@ ENV GCC_MIRRORS \ https://bigsearcher.com/mirrors/gcc/releases \ http://www.netgull.com/gcc/releases \ https://ftpmirror.gnu.org/gcc \ +# "sourceware.org" is the canonical upstream release host (the host of "gcc.gnu.org") + https://sourceware.org/pub/gcc/releases \ # only attempt the origin FTP as a mirror of last resort ftp://ftp.gnu.org/gnu/gcc diff --git a/versions.sh b/versions.sh index 12d8a9ce33a..24272de7fa1 100755 --- a/versions.sh +++ b/versions.sh @@ -34,8 +34,7 @@ debianStable="$(awk <<<"$debianStable" '$1 == "Codename:" { print $2; exit }')" [ -n "$debianStable" ] defaultDebianSuite="$debianStable" -#packagesUrl='https://ftpmirror.gnu.org/gcc/' -packagesUrl='https://mirrors.kernel.org/gnu/gcc/' # the actual HTML of the page changes based on which mirror we end up hitting, so let's hit a specific one for now... :'( +packagesUrl='https://sourceware.org/pub/gcc/releases/?C=M;O=D' # the actual HTML of the page changes based on which mirror we end up hitting, *and* sometimes specific mirrors are missing versions, so let's hit the original canonical host for version scraping packages="$(wget -qO- "$packagesUrl")" # our own "supported" window is 18 months from the most recent release because upstream doesn't have a good guideline, but appears to only release maintenance updates for 2-3 years after the initial release @@ -50,8 +49,8 @@ eols=() dateFormat='%Y-%m-%d' for version in "${versions[@]}"; do - fullVersion="$(grep -E '