From 8f03259376b98d0fc64c41a9797a7cf3f9ed6099 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 19 Oct 2022 13:36:24 +0900 Subject: [PATCH] ARROW-17353: [Release][R] Validate binaries version (#14396) We generate version in Crossbow not CI job. We can use file name validation feature by using version generated by Crossbow. We use "X.Y.Z" version for RC version such as "10.0.0" for "10.0.0-rc1". Authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- dev/archery/archery/crossbow/core.py | 43 +++++++++++++++++-- .../fixtures/crossbow-job-no-failure.yaml | 4 +- .../crossbow/tests/fixtures/crossbow-job.yaml | 4 +- dev/tasks/macros.jinja | 12 ++++-- dev/tasks/r/github.packages.yml | 8 +--- dev/tasks/tasks.yml | 22 +++++----- r/tools/nixlibs.R | 2 +- r/tools/winlibs.R | 5 ++- 8 files changed, 72 insertions(+), 28 deletions(-) diff --git a/dev/archery/archery/crossbow/core.py b/dev/archery/archery/crossbow/core.py index c8ea8a13a4bb2..a83c190d121ef 100644 --- a/dev/archery/archery/crossbow/core.py +++ b/dev/archery/archery/crossbow/core.py @@ -738,14 +738,16 @@ class Target(Serializable): (currently only an email address where the notification should be sent). """ - def __init__(self, head, branch, remote, version, email=None): + def __init__(self, head, branch, remote, version, r_version, email=None): self.head = head self.email = email self.branch = branch self.remote = remote self.github_repo = "/".join(_parse_github_user_repo(remote)) self.version = version + self.r_version = r_version self.no_rc_version = re.sub(r'-rc\d+\Z', '', version) + self.no_rc_r_version = re.sub(r'-rc\d+\Z', '', r_version) # TODO(ARROW-17552): Remove "master" from default_branch after # migration to "main". self.default_branch = ['main', 'master'] @@ -791,8 +793,39 @@ def from_repo(cls, repo, head=None, branch=None, remote=None, version=None, if email is None: email = repo.user_email + version_dev_match = re.match(r".*\.dev(\d+)$", version) + if version_dev_match: + with open(f"{repo.path}/r/DESCRIPTION") as description_file: + description = description_file.read() + r_version_pattern = re.compile(r"^Version:\s*(.*)$", + re.MULTILINE) + r_version = re.findall(r_version_pattern, description)[0] + if r_version: + version_dev = int(version_dev_match[1]) + # "1_0000_00_00 +" is for generating a greater version + # than YYYYMMDD. For example, 1_0000_00_01 + # (version_dev == 1 case) is greater than 2022_10_16. + # + # Why do we need a greater version than YYYYMMDD? It's + # for keeping backward compatibility. We used + # MAJOR.MINOR.PATCH.YYYYMMDD as our nightly package + # version. (See also ARROW-16403). If we use "9000 + + # version_dev" here, a developer that used + # 9.0.0.20221016 can't upgrade to the later nightly + # package unless we release 10.0.0. Because 9.0.0.9234 + # or something is less than 9.0.0.20221016. + r_version_dev = 1_0000_00_00 + version_dev + # version: 10.0.0.dev234 + # r_version: 9.0.0.9000 + # -> 9.0.0.100000234 + r_version = re.sub(r"\.9000\Z", f".{r_version_dev}", r_version) + else: + r_version = version + else: + r_version = version + return cls(head=head, email=email, branch=branch, remote=remote, - version=version) + version=version, r_version=r_version) def is_default_branch(self): # TODO(ARROW-17552): Switch the condition to "is" instead of "in" @@ -1105,7 +1138,10 @@ def from_config(cls, config, target, tasks=None, groups=None, params=None): 'version': target.version, 'no_rc_version': target.no_rc_version, 'no_rc_semver_version': target.no_rc_semver_version, - 'no_rc_snapshot_version': target.no_rc_snapshot_version} + 'no_rc_snapshot_version': target.no_rc_snapshot_version, + 'r_version': target.r_version, + 'no_rc_r_version': target.no_rc_r_version, + } for task_name, task in task_definitions.items(): task = task.copy() artifacts = task.pop('artifacts', None) or [] # because of yaml @@ -1260,6 +1296,7 @@ def validate(self): branch='master', remote='https://github.com/apache/arrow', version='1.0.0dev123', + r_version='0.13.0.100000123', email='dummy@example.ltd' ) job = Job.from_config(config=self, diff --git a/dev/archery/archery/crossbow/tests/fixtures/crossbow-job-no-failure.yaml b/dev/archery/archery/crossbow/tests/fixtures/crossbow-job-no-failure.yaml index 15e8ca3ff5ee3..eb03bbee0bd6d 100644 --- a/dev/archery/archery/crossbow/tests/fixtures/crossbow-job-no-failure.yaml +++ b/dev/archery/archery/crossbow/tests/fixtures/crossbow-job-no-failure.yaml @@ -5,7 +5,9 @@ target: !Target branch: refs/pull/4435/merge remote: https://github.com/apache/arrow version: 0.13.0.dev306 + r_version: 0.12.0.100000306 no_rc_version: 0.13.0.dev306 + no_rc_r_version: 0.12.0.100000306 tasks: docker-cpp-cmake32: !Task ci: circle @@ -64,4 +66,4 @@ branch: ursabot-1 _queue: !Queue path: the_path github_token: xxxxxxxxx - _remote_url: https://github.com/apache/crossbow \ No newline at end of file + _remote_url: https://github.com/apache/crossbow diff --git a/dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml b/dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml index 90eab7049884b..f6de07dd456c8 100644 --- a/dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml +++ b/dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml @@ -5,7 +5,9 @@ target: !Target branch: refs/pull/4435/merge remote: https://github.com/apache/arrow version: 0.13.0.dev306 + r_version: 0.12.0.100000306 no_rc_version: 0.13.0.dev306 + no_rc_r_version: 0.12.0.100000306 tasks: docker-cpp-cmake32: !Task ci: circle @@ -64,4 +66,4 @@ branch: ursabot-1 _queue: !Queue path: the_path github_token: xxxxxxxxx - _remote_url: https://github.com/apache/crossbow \ No newline at end of file + _remote_url: https://github.com/apache/crossbow diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 3bec472bcf60f..bd3358e0733eb 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -269,7 +269,7 @@ on: rm -f apache-arrow*.rb.bak {% endmacro %} -{%- macro github_change_r_pkg_version(is_fork, version = '\\2.\'\"$(date +%Y%m%d)\"\'' ) -%} +{%- macro github_change_r_pkg_version(is_fork, version) -%} - name: Modify version shell: bash run: | @@ -339,12 +339,16 @@ on: # tree not available in git-bash on windows run: | ls -R repo - - name: Add dev repo to .Rprofile + - name: Add repos to .Rprofile shell: Rscript {0} run: | - str <- paste0("options(arrow.dev_repo ='file://", getwd(), "/repo' )") - print(str) profile_path <- file.path(getwd(), ".Rprofile") + repo <- paste0("file://", getwd(), "/repo") + str <- paste0("options(arrow.repo = '", repo, "' )") + print(str) + write(str, file = profile_path, append = TRUE) + str <- paste0("options(arrow.dev_repo = '", repo, "' )") + print(str) write(str, file = profile_path, append = TRUE) # Set envvar for later steps by appending to $GITHUB_ENV write(paste0("R_PROFILE_USER=", profile_path), file = Sys.getenv("GITHUB_ENV"), append = TRUE) diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index 504592d52a566..2ee76a1efba33 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -17,10 +17,6 @@ {% import 'macros.jinja' as macros with context %} -# This allows us to set a custom version via param: -# crossbow submit --param custom_version=8.5.3 r-binary-packages -# if the param is unset defaults to the usual Ymd naming scheme -{% set package_version = custom_version|replace("Unset", "\\2.\'\"$(date +%Y%m%d)\"\'") %} {% set is_fork = macros.is_fork %} {{ macros.github_header() }} @@ -35,7 +31,7 @@ jobs: pkg_version: {{ '${{ steps.save-version.outputs.pkg_version }}' }} steps: {{ macros.github_checkout_arrow()|indent }} - {{ macros.github_change_r_pkg_version(is_fork, package_version)|indent }} + {{ macros.github_change_r_pkg_version(is_fork, arrow.no_rc_r_version)|indent }} - name: Save Version id: save-version shell: bash @@ -281,7 +277,7 @@ jobs: {{ macros.github_test_r_src_pkg()|indent(8) }} ' - name: Upload binary artifact - if: matrix.config.devtoolset + if: matrix.config.devtoolset uses: actions/upload-artifact@v3 with: name: r-pkg_centos7 diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 81a2c24bba1e9..bdf53ff1dac4a 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -957,17 +957,17 @@ tasks: params: custom_version: Unset artifacts: - - r-lib__libarrow__bin__windows__arrow-[0-9\.]+\.zip - - r-lib__libarrow__bin__centos-7__arrow-[0-9\.]+\.zip - - r-lib__libarrow__bin__ubuntu-18.04__arrow-[0-9\.]+\.zip - - r-lib__libarrow__bin__ubuntu-22.04__arrow-[0-9\.]+\.zip - - r-pkg__bin__windows__contrib__4.1__arrow_[0-9\.]+\.zip - - r-pkg__bin__windows__contrib__4.2__arrow_[0-9\.]+\.zip - - r-pkg__bin__macosx__contrib__4.1__arrow_[0-9\.]+\.tgz - - r-pkg__bin__macosx__contrib__4.2__arrow_[0-9\.]+\.tgz - - r-pkg__bin__macosx__big-sur-arm64__contrib__4.1__arrow_[0-9\.]+\.tgz - - r-pkg__bin__macosx__big-sur-arm64__contrib__4.2__arrow_[0-9\.]+\.tgz - - r-pkg__src__contrib__arrow_[0-9\.]+\.tar\.gz + - r-lib__libarrow__bin__windows__arrow-{no_rc_r_version}\.zip + - r-lib__libarrow__bin__centos-7__arrow-{no_rc_r_version}\.zip + - r-lib__libarrow__bin__ubuntu-18.04__arrow-{no_rc_r_version}\.zip + - r-lib__libarrow__bin__ubuntu-22.04__arrow-{no_rc_r_version}\.zip + - r-pkg__bin__windows__contrib__4.1__arrow_{no_rc_r_version}\.zip + - r-pkg__bin__windows__contrib__4.2__arrow_{no_rc_r_version}\.zip + - r-pkg__bin__macosx__contrib__4.1__arrow_{no_rc_r_version}\.tgz + - r-pkg__bin__macosx__contrib__4.2__arrow_{no_rc_r_version}\.tgz + - r-pkg__bin__macosx__big-sur-arm64__contrib__4.1__arrow_{no_rc_r_version}\.tgz + - r-pkg__bin__macosx__big-sur-arm64__contrib__4.2__arrow_{no_rc_r_version}\.tgz + - r-pkg__src__contrib__arrow_{no_rc_r_version}\.tar\.gz ########################### Release verification ############################ diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index f066e4494d3a8..817563b85ccc2 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -32,7 +32,7 @@ dev_version <- package_version(VERSION)[1, 4] # Small dev versions are added for R-only changes during CRAN submission. if (is.na(dev_version) || dev_version < 100) { VERSION <- package_version(VERSION)[1, 1:3] - arrow_repo <- sprintf("https://apache.jfrog.io/artifactory/arrow/r/%s/libarrow/", VERSION) + arrow_repo <- paste0(getOption("arrow.repo", sprintf("https://apache.jfrog.io/artifactory/arrow/r/%s", VERSION)), "/libarrow/") } else { arrow_repo <- paste0(getOption("arrow.dev_repo", "https://nightlies.apache.org/arrow/r"), "/libarrow/") } diff --git a/r/tools/winlibs.R b/r/tools/winlibs.R index 165c98da5ea95..d941da4baa61f 100644 --- a/r/tools/winlibs.R +++ b/r/tools/winlibs.R @@ -44,7 +44,10 @@ if (!file.exists(sprintf("windows/arrow-%s/include/arrow/api.h", VERSION))) { "/libarrow/bin/windows/arrow-%s.zip" ) # %1$s uses the first variable for both substitutions - artifactory <- "https://apache.jfrog.io/artifactory/arrow/r/%1$s/libarrow/bin/windows/arrow-%1$s.zip" + artifactory <- paste0( + getOption("arrow.repo", "https://apache.jfrog.io/artifactory/arrow/r/%1$s"), + "/libarrow/bin/windows/arrow-%1$s.zip" + ) rwinlib <- "https://github.com/rwinlib/arrow/archive/v%s.zip" dev_version <- package_version(VERSION)[1, 4]