From e505a3bab23bbd26cf2aaa23a8ccaee43a1a10b6 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Mon, 3 May 2021 10:31:20 -0700 Subject: [PATCH 01/13] Fix build on Plaza where user PKG_CONFIG_PATH is needed --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ce2fcc4a3fe..120be01571c 100644 --- a/Makefile +++ b/Makefile @@ -554,7 +554,7 @@ $(LIB_DIR)/libtabixpp.a: $(LIB_DIR)/libhts.a $(TABIXPP_DIR)/*.cpp $(TABIXPP_DIR) # Build vcflib. Install the library and headers but not binaries or man pages. $(LIB_DIR)/libvcflib.a: $(LIB_DIR)/libhts.a $(LIB_DIR)/libtabixpp.a $(VCFLIB_DIR)/src/*.cpp $(VCFLIB_DIR)/src/*.hpp $(VCFLIB_DIR)/intervaltree/*.cpp $(VCFLIB_DIR)/intervaltree/*.h - +. ./source_me.sh && cd $(VCFLIB_DIR) && rm -Rf build && mkdir build && cd build && PKG_CONFIG_PATH="$(CWD)/$(LIB_DIR)/pkgconfig" cmake .. && cmake --build . + +. ./source_me.sh && cd $(VCFLIB_DIR) && rm -Rf build && mkdir build && cd build && PKG_CONFIG_PATH="$(CWD)/$(LIB_DIR)/pkgconfig:$(PKG_CONFIG_PATH)" cmake -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON .. && cmake --build . +cp $(VCFLIB_DIR)/filevercmp/*.h* $(INC_DIR) +cp $(VCFLIB_DIR)/fastahack/*.h* $(INC_DIR) +cp $(VCFLIB_DIR)/smithwaterman/*.h* $(INC_DIR) From 6bea975cfad0b587922b644de51f93c60e55a46e Mon Sep 17 00:00:00 2001 From: Jordan Eizenga Date: Sat, 15 May 2021 16:07:59 -0700 Subject: [PATCH 02/13] update to fastahack to handle tabs correctly --- deps/fastahack | 2 +- src/index_registry.cpp | 6 +++--- src/subcommand/autoindex_main.cpp | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deps/fastahack b/deps/fastahack index 8e886558959..75f12d25df9 160000 --- a/deps/fastahack +++ b/deps/fastahack @@ -1 +1 @@ -Subproject commit 8e886558959b4283dde8d31144f277716157884a +Subproject commit 75f12d25df9416b9d49b84c70dcc58406afce11a diff --git a/src/index_registry.cpp b/src/index_registry.cpp index a350cf88c6f..6d0537a9b37 100644 --- a/src/index_registry.cpp +++ b/src/index_registry.cpp @@ -631,9 +631,9 @@ IndexRegistry VGIndexes::get_vg_index_registry() { FastaReference ref; ref.open(fasta_filenames[i]); for (const auto& idx_entry : *ref.index) { - seq_files[idx_entry.second.name] = i; - seq_lengths[idx_entry.second.name] = idx_entry.second.length; - seq_queue.emplace(idx_entry.second.length, idx_entry.second.name); + seq_files[idx_entry.first] = i; + seq_lengths[idx_entry.first] = idx_entry.second.length; + seq_queue.emplace(idx_entry.second.length, idx_entry.first); } } diff --git a/src/subcommand/autoindex_main.cpp b/src/subcommand/autoindex_main.cpp index b51aebb3be9..96b4bf4b4c4 100644 --- a/src/subcommand/autoindex_main.cpp +++ b/src/subcommand/autoindex_main.cpp @@ -265,7 +265,7 @@ int main_autoindex(int argc, char** argv) { }; int option_index = 0; - c = getopt_long (argc, argv, "p:w:r:v:i:g:x:a:P:R:f:M:T:t:dVh", + c = getopt_long (argc, argv, "p:w:r:v:i:g:x:a:P:R:f:M:T:t:dV:h", long_options, &option_index); // Detect the end of the options. From bad95b514b317b82b86b7c07674439af9c9524c1 Mon Sep 17 00:00:00 2001 From: Jordan Eizenga Date: Sat, 15 May 2021 16:26:11 -0700 Subject: [PATCH 03/13] add best practice options for gbwt --- src/index_registry.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/index_registry.cpp b/src/index_registry.cpp index 6d0537a9b37..6a2124ef88f 100644 --- a/src/index_registry.cpp +++ b/src/index_registry.cpp @@ -2328,6 +2328,10 @@ IndexRegistry VGIndexes::get_vg_index_registry() { } haplotype_indexer->show_progress = IndexingParameters::verbosity >= IndexingParameters::Debug; + // from the toil-vg best practices + haplotype_indexer->force_phasing = true; + haplotype_indexer->discard_overlaps = true; + vector parse_files = haplotype_indexer->parse_vcf(vcf_filenames[i], *graph); From 87c75bf79c8095f06a5b871579dd422a8ef6d612 Mon Sep 17 00:00:00 2001 From: Jordan Eizenga Date: Sat, 15 May 2021 17:21:23 -0700 Subject: [PATCH 04/13] add dummy sample to placate haplotype indexer --- src/haplotype_indexer.cpp | 2 +- src/index_registry.cpp | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/haplotype_indexer.cpp b/src/haplotype_indexer.cpp index dcaf19ce18a..422bb7cd006 100644 --- a/src/haplotype_indexer.cpp +++ b/src/haplotype_indexer.cpp @@ -57,7 +57,7 @@ std::vector HaplotypeIndexer::parse_vcf(const std::string& filename // How many samples are there? size_t num_samples = variant_file.sampleNames.size(); if (num_samples == 0) { - std::cerr << "error: [HaplotypeIndexer::parse_vcf] the variant file does not contain phasings" << std::endl; + std::cerr << "error: [HaplotypeIndexer::parse_vcf] variant file '" << filename << "' does not contain phasings" << std::endl; std::exit(EXIT_FAILURE); } diff --git a/src/index_registry.cpp b/src/index_registry.cpp index 6a2124ef88f..feda2e60f49 100644 --- a/src/index_registry.cpp +++ b/src/index_registry.cpp @@ -955,6 +955,12 @@ IndexRegistry VGIndexes::get_vg_index_registry() { auto output_vcf_name = plan->output_filepath(output_vcf, i, buckets.size()); htsFile* vcf = bcf_open(output_vcf_name.c_str(), "wz"); bcf_hdr_t* header = bcf_hdr_init("w"); + // this is to satisfy HaplotypeIndexer, which doesn't like sample-less VCFs + int sample_add_code = bcf_hdr_add_sample(header, "dummy"); + if (sample_add_code != 0) { + cerr << "error:[IndexRegistry] error initializing VCF header" << endl; + exit(1); + } int hdr_write_err_code = bcf_hdr_write(vcf, header); if (hdr_write_err_code != 0) { cerr << "error:[IndexRegistry] error writing VCF header to " << output_vcf_name << endl; From 048fd92aa218749cf8b5202115c104695f507194 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Mon, 17 May 2021 12:26:31 -0700 Subject: [PATCH 05/13] Handle Node ecosystem wanting to Move Fast and Break My Build --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b1c6359e59f..1d7e4d47d84 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,6 +8,8 @@ image: quay.io/vgteam/vg_ci_prebake:latest before_script: + # txm markdown test runner depends on yargs which requires Node newer than current Ubuntu LTS can ship + - curl -sL https://deb.nodesource.com/setup_14.x | sudo -E bash - - sudo apt-get -q -y update # Make sure we have some curl stuff for pycurl which we need for some Python stuff # And the CI report upload needs uuidgen from uuid-runtime From 9e892f58fc14ff73683af1bee2af7013462569a3 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Mon, 17 May 2021 13:45:44 -0700 Subject: [PATCH 06/13] Actually install the new node in the container where it is needed --- Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index cd2176ab6c5..76f6e4f9404 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,8 @@ ARG TARGETARCH RUN echo build > /stage.txt -RUN apt-get -qq -y update && \ +RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - && \ + apt-get -qq -y update && \ apt-get -qq -y upgrade && \ apt-get -qq -y install sudo @@ -76,6 +77,8 @@ ARG THREADS=8 RUN echo test > /stage.txt +RUN apt-get -qq -y install nodejs + # Fail if any non-portable instructions were used RUN /bin/bash -e -c 'if objdump -d /vg/bin/vg | grep vperm2i128 ; then exit 1 ; else exit 0 ; fi' From 3000e8789e0a69ec3f699b3703543e396741e853 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Mon, 17 May 2021 13:46:27 -0700 Subject: [PATCH 07/13] Don't install the new node in the Gitlab environment --- .gitlab-ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1d7e4d47d84..b1c6359e59f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,8 +8,6 @@ image: quay.io/vgteam/vg_ci_prebake:latest before_script: - # txm markdown test runner depends on yargs which requires Node newer than current Ubuntu LTS can ship - - curl -sL https://deb.nodesource.com/setup_14.x | sudo -E bash - - sudo apt-get -q -y update # Make sure we have some curl stuff for pycurl which we need for some Python stuff # And the CI report upload needs uuidgen from uuid-runtime From 365b379ccf632ca507b7afea2db011e15ebfdfce Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Mon, 17 May 2021 15:08:19 -0700 Subject: [PATCH 08/13] Adjust builder deployment so it updates and maybe learns to reauth with Quay --- vgci/buildkit-deployment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vgci/buildkit-deployment.yml b/vgci/buildkit-deployment.yml index 1db768665bd..1bd1a045ca8 100644 --- a/vgci/buildkit-deployment.yml +++ b/vgci/buildkit-deployment.yml @@ -34,7 +34,7 @@ spec: initContainers: - name: configure image: moby/buildkit:buildx-stable-1 - imagePullPolicy: IfNotPresent + imagePullPolicy: Always # Set up to GC any container data over 80 GB. # In the background? See https://github.com/moby/buildkit/issues/1962 command: @@ -90,7 +90,7 @@ spec: privileged: true containers: - image: moby/buildkit:buildx-stable-1 - imagePullPolicy: IfNotPresent + imagePullPolicy: Always command: [buildkitd, '--debug'] name: buildkitd readinessProbe: From 3654fec4b54befeb506e7d379cb5f8ff0c61e7f1 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Mon, 17 May 2021 15:55:58 -0700 Subject: [PATCH 09/13] Actually install the node we need in the test container where we actually need it --- Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 76f6e4f9404..6cf7e060955 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,8 +16,7 @@ ARG TARGETARCH RUN echo build > /stage.txt -RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - && \ - apt-get -qq -y update && \ +RUN apt-get -qq -y update && \ apt-get -qq -y upgrade && \ apt-get -qq -y install sudo @@ -77,7 +76,7 @@ ARG THREADS=8 RUN echo test > /stage.txt -RUN apt-get -qq -y install nodejs +RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - && apt-get -qq -y install nodejs # Fail if any non-portable instructions were used RUN /bin/bash -e -c 'if objdump -d /vg/bin/vg | grep vperm2i128 ; then exit 1 ; else exit 0 ; fi' From 962b0b68248be99405e1d8a1496b836e72d0ac9e Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Mon, 17 May 2021 16:32:33 -0700 Subject: [PATCH 10/13] Do the upload after testing and bang on it until Quay takes it --- .gitlab-ci.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b1c6359e59f..6122607b039 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -89,13 +89,16 @@ build-job: - docker buildx create --use --name=buildkit --platform=linux/amd64,linux/arm64 --node=buildkit-amd64 --driver=kubernetes --driver-opt="nodeselector=kubernetes.io/arch=amd64" # Prune down build cache to make space - docker buildx prune --keep-storage 80G - # Connect so we can upload our images - - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}" script: - make include/vg_git_version.hpp - - docker buildx build --platform=linux/amd64 --build-arg THREADS=8 --target run --push -t "quay.io/vgteam/vg:ci-${CI_PIPELINE_IID}-${CI_COMMIT_SHA}" -f Dockerfile . - # Also run the tests + # Build but don't push, just fill the cache + - docker buildx build --platform=linux/amd64 --build-arg THREADS=8 --target run -f Dockerfile . + # Run the tests - docker buildx build --platform=linux/amd64 --build-arg THREADS=8 --target test -f Dockerfile . + # Connect so we can upload our images + - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}" + # Keep trying to push until it works or we time out + - while [ ! docker buildx build --platform=linux/amd64 --build-arg THREADS=8 --target run --push -t "quay.io/vgteam/vg:ci-${CI_PIPELINE_IID}-${CI_COMMIT_SHA}" -f Dockerfile . ] ; do docker logout "${CI_REGISTRY}" ; sleep 30; docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}" ; done variables: GIT_SUBMODULE_STRATEGY: recursive From 9d3d05d9b765a3b78ee938884e512e5c85c49e3b Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Mon, 17 May 2021 18:05:06 -0700 Subject: [PATCH 11/13] Try another upload method --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6122607b039..004a5c6e6ba 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -98,7 +98,9 @@ build-job: # Connect so we can upload our images - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}" # Keep trying to push until it works or we time out - - while [ ! docker buildx build --platform=linux/amd64 --build-arg THREADS=8 --target run --push -t "quay.io/vgteam/vg:ci-${CI_PIPELINE_IID}-${CI_COMMIT_SHA}" -f Dockerfile . ] ; do docker logout "${CI_REGISTRY}" ; sleep 30; docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}" ; done + - set +e + - docker buildx build --platform=linux/amd64 --build-arg THREADS=8 --target run --push -t "quay.io/vgteam/vg:ci-${CI_PIPELINE_IID}-${CI_COMMIT_SHA}" -f Dockerfile . + - while [[ "$?" != "0" ]] ; do docker logout "${CI_REGISTRY}" ; sleep 30; docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}" ; done variables: GIT_SUBMODULE_STRATEGY: recursive From 63f6b2083eec51227788f4dd981f5877bd299bce Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Mon, 17 May 2021 19:35:33 -0700 Subject: [PATCH 12/13] Hack tube map config so it can start up when IPv6 is off --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 004a5c6e6ba..ae1782e5ef5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -69,6 +69,8 @@ local-build-test-job: # Also test as a backend for the tube map - git clone https://github.com/vgteam/sequenceTubeMap.git - cd sequenceTubeMap + # Tube map expects local IPv6 but Kubernetes won't let us have it + - sed -i 's/^}$/,"serverBindAddress": "127.0.0.1"}/' src/config.json - npm install - CI=true npm run test variables: From 62b7ac953d1c5c640c401efeb1f44ca61c802c7d Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Mon, 17 May 2021 21:56:22 -0700 Subject: [PATCH 13/13] Hide colon from colonophobic YAML parser --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ae1782e5ef5..d3dbe90db25 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -70,7 +70,7 @@ local-build-test-job: - git clone https://github.com/vgteam/sequenceTubeMap.git - cd sequenceTubeMap # Tube map expects local IPv6 but Kubernetes won't let us have it - - sed -i 's/^}$/,"serverBindAddress": "127.0.0.1"}/' src/config.json + - 'sed -i "s/^}$/,\"serverBindAddress\": \"127.0.0.1\"}/" src/config.json' - npm install - CI=true npm run test variables: