From c2bfb2e98d133c62be328e6e93c4b4bdf766d82c Mon Sep 17 00:00:00 2001 From: kshyanashree <109167932+kshyanashree@users.noreply.github.com> Date: Wed, 18 Jan 2023 22:10:54 -0800 Subject: [PATCH] Enable git_submodules tests and merge git_repository and starlark_git_repository (#17247) - Added necessary flags to allow clone submodule from local repository. - We only have the starlark version of git repository rule and most test cases in two integration tests are duplicated. Fixes https://github.com/bazelbuild/bazel/issues/17040 RELNOTES: None PiperOrigin-RevId: 498989149 Change-Id: I1eec3efa17382ddd0a608c0984e2f6c9f29c5c7c Co-authored-by: Googler --- src/test/shell/bazel/BUILD | 11 - src/test/shell/bazel/git_repository_test.sh | 515 ------------------ .../bazel/starlark_git_repository_test.sh | 33 +- tools/build_defs/repo/git_worker.bzl | 7 +- 4 files changed, 37 insertions(+), 529 deletions(-) delete mode 100755 src/test/shell/bazel/git_repository_test.sh diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD index ee97c61db6d2d8..5d0af9dbfb6953 100644 --- a/src/test/shell/bazel/BUILD +++ b/src/test/shell/bazel/BUILD @@ -716,17 +716,6 @@ sh_test( tags = ["no_windows"], ) -sh_test( - name = "git_repository_test", - size = "large", - srcs = ["git_repository_test.sh"], - data = [ - ":test-deps", - "//src/test/shell/bazel/testdata:git-repos", - "@bazel_tools//tools/bash/runfiles", - ], -) - sh_test( name = "starlark_git_repository_test", size = "large", diff --git a/src/test/shell/bazel/git_repository_test.sh b/src/test/shell/bazel/git_repository_test.sh deleted file mode 100755 index 4d352a201d61b8..00000000000000 --- a/src/test/shell/bazel/git_repository_test.sh +++ /dev/null @@ -1,515 +0,0 @@ -#!/bin/bash -# -# Copyright 2015 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Test git_repository and new_git_repository workspace rules. -# - -set -euo pipefail -# --- begin runfiles.bash initialization --- -if [[ ! -d "${RUNFILES_DIR:-/dev/null}" && ! -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then - if [[ -f "$0.runfiles_manifest" ]]; then - export RUNFILES_MANIFEST_FILE="$0.runfiles_manifest" - elif [[ -f "$0.runfiles/MANIFEST" ]]; then - export RUNFILES_MANIFEST_FILE="$0.runfiles/MANIFEST" - elif [[ -f "$0.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then - export RUNFILES_DIR="$0.runfiles" - fi -fi -if [[ -f "${RUNFILES_DIR:-/dev/null}/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then - source "${RUNFILES_DIR}/bazel_tools/tools/bash/runfiles/runfiles.bash" -elif [[ -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then - source "$(grep -m1 "^bazel_tools/tools/bash/runfiles/runfiles.bash " \ - "$RUNFILES_MANIFEST_FILE" | cut -d ' ' -f 2-)" -else - echo >&2 "ERROR: cannot find @bazel_tools//tools/bash/runfiles:runfiles.bash" - exit 1 -fi -# --- end runfiles.bash initialization --- - -source "$(rlocation "io_bazel/src/test/shell/integration_test_setup.sh")" \ - || { echo "integration_test_setup.sh not found!" >&2; exit 1; } - -# `uname` returns the current platform, e.g "MSYS_NT-10.0" or "Linux". -# `tr` converts all upper case letters to lower case. -# `case` matches the result if the `uname | tr` expression to string prefixes -# that use the same wildcards as names do in Bash, i.e. "msys*" matches strings -# starting with "msys", and "*" matches everything (it's the default case). -case "$(uname -s | tr [:upper:] [:lower:])" in -msys*) - # As of 2019-01-15, Bazel on Windows only supports MSYS Bash. - declare -r is_windows=true - ;; -*) - declare -r is_windows=false - ;; -esac - -if $is_windows; then - export MSYS_NO_PATHCONV=1 - export MSYS2_ARG_CONV_EXCL="*" - # Enable symlink runfiles tree to make bazel run work - add_to_bazelrc "build --enable_runfiles" -fi - -# Global test setup. -# -# Unpacks the test Git repositories in the test temporary directory. -function set_up() { - bazel clean --expunge - local repos_dir=$TEST_TMPDIR/repos - if [ -e "$repos_dir" ]; then - rm -rf $repos_dir - fi - - mkdir -p $repos_dir - cp "$(rlocation io_bazel/src/test/shell/bazel/testdata/pluto-repo.tar.gz)" $repos_dir - cp "$(rlocation io_bazel/src/test/shell/bazel/testdata/outer-planets-repo.tar.gz)" $repos_dir - cp "$(rlocation io_bazel/src/test/shell/bazel/testdata/refetch-repo.tar.gz)" $repos_dir - cp "$(rlocation io_bazel/src/test/shell/bazel/testdata/strip-prefix-repo.tar.gz)" $repos_dir - cd $repos_dir - tar zxf pluto-repo.tar.gz - tar zxf outer-planets-repo.tar.gz - tar zxf refetch-repo.tar.gz - tar zxf strip-prefix-repo.tar.gz -} - -# Test cloning a Git repository using the git_repository rule. -# -# This test uses the pluto Git repository at tag 1-build, which contains the -# following files: -# -# pluto/ -# WORKSPACE -# BUILD -# info -# -# Then, set up workspace with the following files: -# -# $WORKSPACE_DIR/ -# WORKSPACE -# planets/ -# BUILD -# planet_info.sh -# -# //planets has a dependency on a target in the pluto Git repository. -function test_git_repository() { - local pluto_repo_dir=$TEST_TMPDIR/repos/pluto - # Commit corresponds to tag 1-build. See testdata/pluto.git_log. - local commit_hash="52f9a3f87a2dd17ae0e5847bbae9734f09354afd" - - # Create a workspace that clones the repository at the first commit. - cd $WORKSPACE_DIR - cat >> $(create_workspace_with_default_repos WORKSPACE) < planets/BUILD < planets/planet_info.sh <& $TEST_log \ - || echo "Expected build/run to succeed" - expect_log "Pluto is a dwarf planet" -} - -function test_new_git_repository_with_build_file() { - do_new_git_repository_test "build_file" -} - -function test_new_git_repository_with_build_file_content() { - do_new_git_repository_test "build_file_content" -} - -# Test cloning a Git repository using the new_git_repository rule. -# -# This test uses the pluto Git repository at tag 0-initial, which contains the -# following files: -# -# pluto/ -# info -# -# Set up workspace with the following files: -# -# $WORKSPACE_DIR/ -# WORKSPACE -# pluto.BUILD -# planets/ -# BUILD -# planet_info.sh -# -# //planets has a dependency on a target in the $TEST_TMPDIR/pluto Git -# repository. -function do_new_git_repository_test() { - local pluto_repo_dir=$TEST_TMPDIR/repos/pluto - - # Create a workspace that clones the repository at the first commit. - cd $WORKSPACE_DIR - - if [ "$1" == "build_file" ] ; then - cat >> $(create_workspace_with_default_repos WORKSPACE) < BUILD < pluto.BUILD <> $(create_workspace_with_default_repos WORKSPACE) < planets/BUILD < planets/planet_info.sh <& $TEST_log \ - || echo "Expected build/run to succeed" - expect_log "Pluto is a planet" -} - -# Test cloning a Git repository that has a submodule using the -# new_git_repository rule. -# -# This test uses the outer-planets Git repository at revision 1-submodule, which -# contains the following files: -# -# outer_planets/ -# neptune/ -# info -# pluto/ --> submodule ../pluto -# info -# -# Set up workspace with the following files: -# -# $WORKSPACE_DIR/ -# WORKSPACE -# outer_planets.BUILD -# planets/ -# BUILD -# planet_info.sh -# -# planets has a dependency on targets in the $TEST_TMPDIR/outer_planets Git -# repository. -function test_new_git_repository_submodules() { - local outer_planets_repo_dir=$TEST_TMPDIR/repos/outer-planets - - # Create a workspace that clones the outer_planets repository. - cd $WORKSPACE_DIR - cat >> $(create_workspace_with_default_repos WORKSPACE) < BUILD < outer_planets.BUILD < planets/BUILD < planets/planet_info.sh <& $TEST_log \ - || echo "Expected build/run to succeed" - expect_log "Neptune is a planet" - expect_log "Pluto is a planet" -} - -function test_git_repository_not_refetched_on_server_restart() { - local repo_dir=$TEST_TMPDIR/repos/refetch - - cd $WORKSPACE_DIR - cat >> $(create_workspace_with_default_repos WORKSPACE) <& $TEST_log || fail "Build failed" - assert_contains "GIT 1" bazel-genfiles/external/g/go - - # Without changing anything, restart the server, which should not cause the checkout to be re-cloned. - bazel --batch build @g//:g >& $TEST_log || fail "Build failed" - assert_contains "GIT 1" bazel-genfiles/external/g/go - - # Change the commit id, which should cause the checkout to be re-cloned. - rm WORKSPACE - cat >> $(create_workspace_with_default_repos WORKSPACE) <& $TEST_log || fail "Build failed" - assert_contains "GIT 2" bazel-genfiles/external/g/go - - # Change the WORKSPACE but not the commit id, which should not cause the checkout to be re-cloned. - rm WORKSPACE - cat >> $(create_workspace_with_default_repos WORKSPACE) <& $TEST_log || fail "Build failed" - assert_contains "GIT 2" bazel-genfiles/external/g/go -} - -function test_git_repository_refetched_when_commit_changes() { - local repo_dir=$TEST_TMPDIR/repos/refetch - - cd $WORKSPACE_DIR - cat >> $(create_workspace_with_default_repos WORKSPACE) <& $TEST_log || fail "Build failed" - assert_contains "GIT 1" bazel-genfiles/external/g/go - - # Change the commit id, which should cause the checkout to be re-cloned. - rm WORKSPACE - cat >> $(create_workspace_with_default_repos WORKSPACE) <& $TEST_log || fail "Build failed" - assert_contains "GIT 2" bazel-genfiles/external/g/go -} - -function test_git_repository_and_nofetch() { - local repo_dir=$TEST_TMPDIR/repos/refetch - - cd $WORKSPACE_DIR - cat >> $(create_workspace_with_default_repos WORKSPACE) <& $TEST_log && fail "Build succeeded" - expect_log "fetching repositories is disabled" - bazel build @g//:g >& $TEST_log || fail "Build failed" - assert_contains "GIT 1" bazel-genfiles/external/g/go - - rm WORKSPACE - cat >> $(create_workspace_with_default_repos WORKSPACE) <& $TEST_log || fail "Build failed" - expect_log "External repository 'g' is not up-to-date" - assert_contains "GIT 1" bazel-genfiles/external/g/go - bazel build @g//:g >& $TEST_log || fail "Build failed" - assert_contains "GIT 2" bazel-genfiles/external/g/go -} - -# Helper function for setting up the workspace as follows -# -# $WORKSPACE_DIR/ -# WORKSPACE -# planets/ -# planet_info.sh -# BUILD -function setup_error_test() { - cd $WORKSPACE_DIR - mkdir -p planets - cat > planets/planet_info.sh < planets/BUILD <> $(create_workspace_with_default_repos WORKSPACE) <& $TEST_log \ - || echo "Expect run to fail." - expect_log "Exactly one of commit" -} - -# Verifies that rule fails if neither tag or commit are set. -# -# This test uses the pluto Git repository at tag 1-build, which contains the -# following files: -# -# pluto/ -# WORKSPACE -# BUILD -# info -function test_git_repository_no_commit_tag_error() { - setup_error_test - local pluto_repo_dir=$TEST_TMPDIR/pluto - - cd $WORKSPACE_DIR - cat >> $(create_workspace_with_default_repos WORKSPACE) <& $TEST_log \ - || echo "Expect run to fail." - expect_log "Exactly one of commit" -} - -# Verifies that load statement works while using strip_prefix. -# -# This test uses the strip-prefix Git repository, which contains the -# following files: -# -# strip-prefix -# └── prefix-foo -# ├── BUILD -# ├── WORKSPACE -# └── defs.bzl -function test_git_repository_with_strip_prefix_for_load_statement() { - setup_error_test - local strip_prefix_repo_dir=$TEST_TMPDIR/repos/strip-prefix - - cd $WORKSPACE_DIR - cat >> $(create_workspace_with_default_repos WORKSPACE) <& $TEST_log || fail "Expect bazel info to succeed." -} - -run_suite "git_repository tests" diff --git a/src/test/shell/bazel/starlark_git_repository_test.sh b/src/test/shell/bazel/starlark_git_repository_test.sh index c53f9c9e12d515..2e1e9ca23acbf9 100755 --- a/src/test/shell/bazel/starlark_git_repository_test.sh +++ b/src/test/shell/bazel/starlark_git_repository_test.sh @@ -78,10 +78,12 @@ function set_up() { cp "$(rlocation io_bazel/src/test/shell/bazel/testdata/pluto-repo.tar.gz)" $repos_dir cp "$(rlocation io_bazel/src/test/shell/bazel/testdata/outer-planets-repo.tar.gz)" $repos_dir cp "$(rlocation io_bazel/src/test/shell/bazel/testdata/refetch-repo.tar.gz)" $repos_dir + cp "$(rlocation io_bazel/src/test/shell/bazel/testdata/strip-prefix-repo.tar.gz)" $repos_dir cd $repos_dir tar zxf pluto-repo.tar.gz tar zxf outer-planets-repo.tar.gz tar zxf refetch-repo.tar.gz + tar zxf strip-prefix-repo.tar.gz # Fix environment variables for a hermetic use of git. export GIT_CONFIG_NOSYSTEM=1 @@ -371,7 +373,7 @@ EOF expect_log "Pluto is a planet" } -function test_new_git_repository_submodules() { +function test_new_git_repository_submodules_with_recursive_init_modules() { local outer_planets_repo_dir=$TEST_TMPDIR/repos/outer-planets # Create a workspace that clones the outer_planets repository. @@ -686,4 +688,33 @@ EOF expect_log "shallow_since not allowed if a tag is specified; --depth=1 will be used for tags" } +# Verifies that load statement works while using strip_prefix. +# +# This test uses the strip-prefix Git repository, which contains the +# following files: +# +# strip-prefix +# └── prefix-foo +# ├── BUILD +# ├── WORKSPACE +# └── defs.bzl +function test_git_repository_with_strip_prefix_for_load_statement() { + setup_error_test + local strip_prefix_repo_dir=$TEST_TMPDIR/repos/strip-prefix + + cd $WORKSPACE_DIR + cat >> $(create_workspace_with_default_repos WORKSPACE) <& $TEST_log || fail "Expect bazel info to succeed." +} + run_suite "Starlark git_repository tests" diff --git a/tools/build_defs/repo/git_worker.bzl b/tools/build_defs/repo/git_worker.bzl index 4983604ee9e27b..c663cdaabf8032 100644 --- a/tools/build_defs/repo/git_worker.bzl +++ b/tools/build_defs/repo/git_worker.bzl @@ -152,9 +152,12 @@ def clean(ctx, git_repo): def update_submodules(ctx, git_repo, recursive = False): if recursive: - _git(ctx, git_repo, "submodule", "update", "--init", "--recursive", "--checkout", "--force") + # "protocol.file.allow=always" allows the submodule command clone from a local directory. + # It's necessary for Git 2.38.1 and assoicated backport versions. + # See https://github.com/bazelbuild/bazel/issues/17040 + _git(ctx, git_repo, "-c", "protocol.file.allow=always", "submodule", "update", "--init", "--recursive", "--checkout", "--force") else: - _git(ctx, git_repo, "submodule", "update", "--init", "--checkout", "--force") + _git(ctx, git_repo, "-c", "protocol.file.allow=always", "submodule", "update", "--init", "--checkout", "--force") def _get_head_commit(ctx, git_repo): return _git(ctx, git_repo, "log", "-n", "1", "--pretty=format:%H")