From a4776b2635bd1bef321670624b9450acb390493c Mon Sep 17 00:00:00 2001 From: Xu Wang Date: Thu, 18 Apr 2019 13:18:04 +0800 Subject: [PATCH 1/6] CI: static check for openstack git URL Fixes: #1483 Use git ls-remote instead of curl for git URL check Signed-off-by: Xu Wang (cherry picked from commit 0d28f77dec116087f9eb9c2099177712f8de1d10) Signed-off-by: Ganesh Maharaj Mahalingam --- .ci/static-checks.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.ci/static-checks.sh b/.ci/static-checks.sh index 45d48e0f4..bb0edf99c 100755 --- a/.ci/static-checks.sh +++ b/.ci/static-checks.sh @@ -438,6 +438,12 @@ check_docs() # Google APIs typically require an auth token. echo "$url"|grep -q 'https://www.googleapis.com' && continue + # Git repo URL check + if echo "$url"|grep -q '^https.*git' + then + git ls-remote "$url" > /dev/null 2>&1 && continue + fi + # Check the URL, saving it if invalid ( curl -sLf -o /dev/null "$url" ||\ echo "$url" >> "$invalid_urls") & From 08b7a25bd1ae58b3826638cc4b68d591f8b70374 Mon Sep 17 00:00:00 2001 From: Salvador Fuentes Date: Mon, 22 Apr 2019 12:24:49 -0500 Subject: [PATCH 2/6] cri-o: tests: use devicemapper on all ubuntu versions Do not restrict ubuntu version to run with devicemapper. We have now moved most of our jobs to ubuntu 18.04 and we were not testing with devicemapper. With this change, we will test cri-o with devicemapper on all ubuntu versions. Fixes: #1493. Signed-off-by: Salvador Fuentes (cherry picked from commit 77a8cd531aa5b9889fb109d5f6f7d81c54c877ab) Signed-off-by: Ganesh Maharaj Mahalingam --- integration/cri-o/cri-o.sh | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/integration/cri-o/cri-o.sh b/integration/cri-o/cri-o.sh index b76cf2987..b04e1e98e 100755 --- a/integration/cri-o/cri-o.sh +++ b/integration/cri-o/cri-o.sh @@ -60,9 +60,8 @@ done IFS=$OLD_IFS -# run CRI-O tests using devicemapper on ubuntu 16.04 -MAJOR=$(echo "$VERSION_ID"|cut -d\. -f1) -if [ "$ID" == "ubuntu" ] && [ "$MAJOR" -eq 16 ]; then +# run CRI-O tests using devicemapper on ubuntu +if [ "$ID" == "ubuntu" ]; then # Block device attached to the VM where we run the CI # If the block device has a partition, cri-o will not be able to use it. export LVM_DEVICE @@ -72,13 +71,9 @@ if [ "$ID" == "ubuntu" ] && [ "$MAJOR" -eq 16 ]; then export STORAGE_OPTIONS="$DM_STORAGE_OPTIONS" fi -# But if on ubuntu 17.10 or newer, test using overlay +# On other distros or on ZUUL, use overlay. # This will allow us to run tests with at least 2 different # storage drivers. -if [ "$ID" == "ubuntu" ] && [ "$MAJOR" -ge 17 ]; then - export STORAGE_OPTIONS="$OVERLAY_STORAGE_OPTIONS" -fi - if [ "$ID" == "fedora" ] || [ "$ID" == "centos" ]; then export STORAGE_OPTIONS="$OVERLAY_STORAGE_OPTIONS" fi From 44f6e3995d360fda6723816fcb7ef34f9ed3ca2d Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Fri, 26 Apr 2019 13:53:24 +0100 Subject: [PATCH 3/6] kata-manager: Check for dependencies earlier The `kata-manager` script needs to download `git` repos. But if `git` isn't available, it can fall back to using `curl` or `wget`. However, if none of these tools are available, it should exit before any changes are made to the system. Note: The script _could_ install one of the tools of course. However, all known Linux distro "default installs" provide at least one of these tools, so this scenario should only be hit on very minimal systems. In that case, install one of the tools the error mentions and re-run `kata-manager`. Fixes #1502. Signed-off-by: James O. D. Hunt (cherry picked from commit cfa9c821d873db8d8a662243e11e8cd098313211) Signed-off-by: Ganesh Maharaj Mahalingam --- cmd/kata-manager/kata-manager.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/kata-manager/kata-manager.sh b/cmd/kata-manager/kata-manager.sh index 9e49ff747..a01f0419e 100755 --- a/cmd/kata-manager/kata-manager.sh +++ b/cmd/kata-manager/kata-manager.sh @@ -269,8 +269,6 @@ get_git_repo() local -r repo_url="https://${repo_path}" if ! command -v git >/dev/null; then - info "getting repo $1 using http downloader" - detect_downloader $downloader "${repo_url}/${tarball_suffix}" | tar xz -C "$local_dest" --strip-components=1 return fi @@ -437,6 +435,11 @@ setup() if [ -z "$kata_repos_base" ]; then kata_repos_base="$HOME/go" fi + + if ! command -v git >/dev/null; then + info "git not installed - trying to use a downloader instead" + detect_downloader + fi } parse_args() From 46b09ea76681a7cbdb51d55831ca8c207950da9b Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Fri, 26 Apr 2019 09:39:15 -0500 Subject: [PATCH 4/6] integration/docker: improve memory test Consume twice the memory hot added to the container to guarantee the `out of memory` error. fixes #1504 Signed-off-by: Julio Montes (cherry picked from commit bb49514364c5b46d594cf771ce315db546348686) Signed-off-by: Ganesh Maharaj Mahalingam --- integration/docker/mem_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/integration/docker/mem_test.go b/integration/docker/mem_test.go index 8fa41f755..f0ceb29d7 100644 --- a/integration/docker/mem_test.go +++ b/integration/docker/mem_test.go @@ -113,6 +113,7 @@ var _ = Describe("memory constraints", func() { useKmem bool err error defaultMemSz int + hotMemSz int ) BeforeEach(func() { @@ -143,8 +144,9 @@ var _ = Describe("memory constraints", func() { Context("run container exceeding memory constraints", func() { It("should ran out of memory", func() { - memSize = "256MB" - limSize = strconv.Itoa(260+defaultMemSz) + "M" + hotMemSz = 256 + memSize = fmt.Sprintf("%dMB", hotMemSz) + limSize = fmt.Sprintf("%dM", (hotMemSz*2)+defaultMemSz) args = []string{"--name", id, "--rm", "-m", memSize, StressImage, "-mem-total", limSize, "-mem-alloc-size", limSize} _, stderr, exitCode = dockerRun(args...) Expect(exitCode).NotTo(Equal(0)) From 862319852412d09cb6760d0b6ffefc068a83aa74 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Mon, 29 Apr 2019 08:59:05 +0100 Subject: [PATCH 5/6] CI: Refactor static check vendor test Modify the `check_vendor()` function in the static check script. Rather than exiting the function as early as possible, change the logic to allow follow on future checks to be added. Signed-off-by: James O. D. Hunt (cherry picked from commit 50f3b035079e7c38b797edb11ac4c4df1afa58b9) Signed-off-by: Ganesh Maharaj Mahalingam --- .ci/static-checks.sh | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/.ci/static-checks.sh b/.ci/static-checks.sh index bb0edf99c..185c0d822 100755 --- a/.ci/static-checks.sh +++ b/.ci/static-checks.sh @@ -567,22 +567,26 @@ check_vendor() # All vendor operations should modify this file local vendor_ctl_file="Gopkg.lock" + [ -e "$vendor_ctl_file" ] || { info "No vendoring in this repository" && return; } + + info "Checking vendored code is pristine" + files=$(get_pr_changed_file_details_full || true) # Strip off status files=$(echo "$files"|awk '{print $NF}') - # No files were changed - [ -z "$files" ] && info "No files found" && return - - vendor_files=$(echo "$files" | grep "vendor/" || true) - - # No vendor files modified - [ -z "$vendor_files" ] && return - - result=$(echo "$files" | egrep "\<${vendor_ctl_file}\>" || true) + if [ -n "$files" ] + then + # PR changed files so check if it changed any vendored files + vendor_files=$(echo "$files" | grep "vendor/" || true) - [ -n "$result" ] || die "PR changes vendor files, but does not update ${vendor_ctl_file}" + if [ -n "$vendor_files" ] + then + result=$(echo "$files" | egrep "\<${vendor_ctl_file}\>" || true) + [ -n "$result" ] || die "PR changes vendor files, but does not update ${vendor_ctl_file}" + fi + fi } main() From b9e6d5b391cdcb532711b5b357a517112465266d Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Mon, 29 Apr 2019 09:02:40 +0100 Subject: [PATCH 6/6] CI: Add explicit static check script vendor test Update the `check_vendor()` function in the static check script to run our current vendoring tool (`dep`) in check mode. This will ensure our vendoring does not get silently broken [1]. Fixes #1506. [1] - See for example https://github.com/kata-containers/runtime/pull/1442. Signed-off-by: James O. D. Hunt (cherry picked from commit 830983f62b49703e4c052b38ab59947e98722a6f) Signed-off-by: Ganesh Maharaj Mahalingam --- .ci/static-checks.sh | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.ci/static-checks.sh b/.ci/static-checks.sh index 185c0d822..fcbd5e505 100755 --- a/.ci/static-checks.sh +++ b/.ci/static-checks.sh @@ -553,11 +553,15 @@ check_files() exit 1 } -# Ensure that changes to vendored code are accompanied by an update to the -# vendor tooling config file. If not, the user simply hacked the vendor files -# rather than following the correct process: +# Perform vendor checks: # -# - https://github.com/kata-containers/community/blob/master/VENDORING.md +# - Ensure that changes to vendored code are accompanied by an update to the +# vendor tooling config file. If not, the user simply hacked the vendor files +# rather than following the correct process: +# +# https://github.com/kata-containers/community/blob/master/VENDORING.md +# +# - Ensure vendor metadata is valid. check_vendor() { local files @@ -587,6 +591,14 @@ check_vendor() [ -n "$result" ] || die "PR changes vendor files, but does not update ${vendor_ctl_file}" fi fi + + info "Checking vendoring metadata" + + # Get the vendoring tool + go get github.com/golang/dep/cmd/dep + + # Check, but don't touch! + dep ensure -no-vendor -dry-run } main()