refactor(proto-compiler): download protobuf definitions via https ins…

…tead of git (#18)

* feat: use submodules instead of git2 crate to get protobuf definitions

* test(proto): ensure that tenderdash version we use is correct

* build(github): enable checkout submodules

* build(github): enable checkout submodules

* Revert " build(github): enable checkout submodules"

This reverts commit 2ba1db3.

* build(github): enable checkout submodules

* chote(proto-compiler): use git checkout on tenderdash submodule

* chore(proto-compiler): fix git handling

* chore(proto-compiler): git fetch update shallow

* chore(proto-compiler): fetch tags

* chore(proto-compiler): update submodule as needed

* chore(proto-compiler): download tenderdash sources zip instead of cloning git repo

* chore(proto-compiler): create cache (/target) dir if not exist

* build(github): don't checkout submodules anymore

.github/workflows/audit.yml

@@ -10,7 +10,7 @@ jobs:
   security_audit:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Cache cargo bin
         uses: actions/cache@v1
         with:

.github/workflows/build.yml

@@ -22,7 +22,7 @@ jobs:
   tenderdash:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: ./.github/actions/deps
       - name: Build source code
         shell: bash

.github/workflows/rust-clippy.yml

@@ -25,8 +25,7 @@ jobs:
       actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
-
+        uses: actions/checkout@v3
       - name: Install Rust toolchain and deps
         uses: ./.github/actions/deps
 

.github/workflows/rust.yml

@@ -23,7 +23,7 @@ jobs:
   fmt:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: ./.github/actions/deps
       - name: Check code formatting
         shell: bash
@@ -32,7 +32,7 @@ jobs:
   docs:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: ./.github/actions/deps
       - name: Check documentation generation
         shell: bash

.github/workflows/test.yml

@@ -28,8 +28,7 @@ jobs:
       RUST_TEST_TIME_INTEGRATION: "3000,6000"
       RUST_TEST_TIME_DOCTEST: "3000,6000"
     steps:
-      - uses: actions/checkout@v2
-
+      - uses: actions/checkout@v3
       - uses: ./.github/actions/deps
         with:
           toolchain: nightly

proto-compiler/Cargo.toml

@@ -11,7 +11,9 @@ publish = false
 [dependencies]
 walkdir = { version = "2.3" }
 prost-build = { version = "0.11" }
-git2 = { version = "0.16" }
 tempfile = { version = "3.2.0" }
 subtle-encoding = { version = "0.5" }
 regex = { "version" = "1.7.1" }
+reqwest = { "version" = "0.11.16", features = ["blocking"] }
+zip = { version = "0.6.4", default-features = false, features = ["deflate"] }
+fs_extra = { version = "1.3.0" }

proto-compiler/src/functions.rs

@@ -5,163 +5,87 @@ use std::{
     path::{Path, PathBuf},
 };
 
-use git2::{
-    build::{CheckoutBuilder, RepoBuilder},
-    AutotagOption, Commit, FetchOptions, Oid, Reference, Repository,
-};
-use subtle_encoding::hex;
 use walkdir::WalkDir;
 
 use crate::constants::DEFAULT_TENDERDASH_COMMITISH;
 
-/// Clone or open+fetch a repository and check out a specific commitish
-/// In case of an existing repository, the origin remote will be set to `url`.
-pub fn fetch_commitish(dir: &Path, url: &str, commitish: &str) {
-    let mut dotgit = dir.to_path_buf();
-    dotgit.push(".git");
-    let repo = if dotgit.is_dir() {
-        fetch_existing(dir, url)
-    } else {
-        clone_new(dir, url)
-    };
-    checkout_commitish(&repo, commitish)
-}
-
-fn clone_new(dir: &Path, url: &str) -> Repository {
-    println!(
-        "  [info] => Cloning {} into {} folder",
-        url,
-        dir.to_string_lossy()
-    );
-
-    let mut fo = FetchOptions::new();
-    fo.download_tags(AutotagOption::All);
-    fo.update_fetchhead(true);
+/// Check out a specific commitish of the tenderdash repository.
+///
+/// As this tool is mainly used by build.rs script, we rely
+/// on cargo to decide wherther or not to call it. It means
+/// we will not be called too frequently, so the fetch will
+/// not happen too often.
+pub fn fetch_commitish(tenderdash_dir: &Path, cache_dir: &Path, url: &str, commitish: &str) {
+    let url = format!("{url}/archive/{commitish}.zip");
 
-    let mut builder = RepoBuilder::new();
-    builder.fetch_options(fo);
-
-    builder.clone(url, dir).unwrap()
-}
-
-fn fetch_existing(dir: &Path, url: &str) -> Repository {
     println!(
-        "  [info] => Fetching from {} into existing {} folder",
+        "  [info] => Downloading and extracting {} into {}",
         url,
-        dir.to_string_lossy()
+        tenderdash_dir.to_string_lossy()
     );
-    let repo = Repository::open(dir).unwrap();
-
-    let mut fo = git2::FetchOptions::new();
-    fo.download_tags(git2::AutotagOption::All);
-    fo.update_fetchhead(true);
 
-    let mut remote = repo
-        .find_remote("origin")
-        .unwrap_or_else(|_| repo.remote("origin", url).unwrap());
-    if remote.url().is_none() || remote.url().unwrap() != url {
-        repo.remote_set_url("origin", url).unwrap();
-    }
-    println!("  [info] => Fetching repo using remote `origin`");
-    let specs: &[&str] = &[];
-    remote.fetch(specs, Some(&mut fo), None).unwrap();
-
-    let stats = remote.stats();
-    if stats.local_objects() > 0 {
-        println!(
-            "  [info] => Received {}/{} objects in {} bytes (used {} local objects)",
-            stats.indexed_objects(),
-            stats.total_objects(),
-            stats.received_bytes(),
-            stats.local_objects()
-        );
-    } else {
-        println!(
-            "  [info] => Received {}/{} objects in {} bytes",
-            stats.indexed_objects(),
-            stats.total_objects(),
-            stats.received_bytes()
-        );
+    // ensure cache dir exists
+    if !cache_dir.is_dir() {
+        std::fs::create_dir_all(cache_dir).expect("cannot create cache directory");
     }
 
-    Repository::open(dir).unwrap()
-}
+    let archive_file = cache_dir.join(format!("tenderdash-{}.zip", commitish));
+    // Unzip Tenderdash sources to tmpdir and move to target/tenderdash
+    let tmpdir = tempfile::tempdir().expect("cannot create temporary dir to extract archive");
+    download_and_unzip(&url, archive_file.as_path(), tmpdir.path());
 
-fn checkout_commitish(repo: &Repository, commitish: &str) {
-    let (reference, commit) = find_reference_or_commit(repo, commitish);
+    // Downloaded zip contains subdirectory like tenderdash-0.12.0-dev.1. We need to
+    // move its contents to target/tederdash, so that we get correct paths like
+    // target/tenderdash/version/version.go
+    let src_dir = find_subdir(tmpdir.path(), "tenderdash-");
 
-    println!(
-        "  [info] => Checking out repo in detached HEAD mode:\n    \
-             [info] => id: {},\n    \
-             [info] => author: {},\n    \
-             [info] => committer: {},\n    \
-             [info] => summary: {}",
-        commit.id(),
-        commit.author(),
-        commit.committer(),
-        commit.summary().unwrap_or(""),
-    );
+    let options = fs_extra::dir::CopyOptions::new().content_only(true);
 
-    match reference {
-        None => repo.set_head_detached(commit.id()).unwrap(),
-        Some(reference) => {
-            println!("    [info] => name: {}", reference.shorthand().unwrap());
-            repo.set_head(reference.name().unwrap()).unwrap();
-        },
-    }
-
-    let mut checkout_options = CheckoutBuilder::new();
-    checkout_options
-        .force()
-        .remove_untracked(true)
-        .remove_ignored(true)
-        .use_theirs(true);
-    repo.checkout_head(Some(&mut checkout_options)).unwrap();
+    fs_extra::dir::create(tenderdash_dir, true).expect("cannot create destination directory");
+    fs_extra::dir::move_dir(src_dir, tenderdash_dir, &options)
+        .expect("cannot move tenderdash directory");
 }
 
-fn find_reference_or_commit<'a>(
-    repo: &'a Repository,
-    commitish: &str,
-) -> (Option<Reference<'a>>, Commit<'a>) {
-    let mut tried_origin = false; // we tried adding 'origin/' to the commitish
-
-    let mut try_reference = repo.resolve_reference_from_short_name(commitish);
-    if try_reference.is_err() {
-        // Local branch might be missing, try the remote branch
-        try_reference = repo.resolve_reference_from_short_name(&format!("origin/{commitish}"));
-        tried_origin = true;
-        if try_reference.is_err() {
-            // Remote branch not found, last chance: try as a commit ID
-            // Note: Oid::from_str() currently does an incorrect conversion and cuts the
-            // second half of the ID. We are falling back on Oid::from_bytes()
-            // for now.
-            let commitish_vec = hex::decode(commitish).unwrap_or_else(|_| {
-                hex::decode_upper(commitish).expect(
-                    "TENDERDASH_COMMITISH refers to non-existing or invalid git branch/tag/commit",
-                )
-            });
-            return (
-                None,
-                repo.find_commit(Oid::from_bytes(commitish_vec.as_slice()).unwrap())
-                    .unwrap(),
-            );
+/// Download file from URL and unzip it to `dest_dir`
+fn download_and_unzip(url: &str, archive_file: &Path, dest_dir: &Path) {
+    // We download only if the file does not exist
+    if !archive_file.is_file() {
+        let mut file = File::create(archive_file).expect("cannot create file");
+
+        let mut rb = reqwest::blocking::get(url).expect("cannot download archive");
+        if !rb.status().is_success() {
+            panic!(
+                "cannot download tenderdash sources from {url}: {:?}",
+                rb.status()
+            )
         }
+        rb.copy_to(&mut file).expect("cannot save downloaded data");
+        file.flush().expect("flush of archive file failed");
     }
 
-    let mut reference = try_reference.unwrap();
-    if reference.is_branch() {
-        if tried_origin {
-            panic!("[error] => local branch names with 'origin/' prefix not supported");
-        }
-        try_reference = repo.resolve_reference_from_short_name(&format!("origin/{commitish}"));
-        reference = try_reference.unwrap();
-        if reference.is_branch() {
-            panic!("[error] => local branch names with 'origin/' prefix not supported");
-        }
-    }
+    let file = File::open(archive_file).expect("cannot open downloaded zip");
+    let mut archive = zip::ZipArchive::new(&file).expect("cannot open zip archive");
 
-    let commit = reference.peel_to_commit().unwrap();
-    (Some(reference), commit)
+    archive.extract(dest_dir).expect("cannot extract archive");
+}
+/// Find a subdirectory of a parent path which has provided name prefix
+fn find_subdir(parent: &Path, name_prefix: &str) -> PathBuf {
+    let dir_content = fs_extra::dir::get_dir_content(parent).expect("cannot ls tmp dir");
+    let mut src_dir = String::new();
+    for directory in dir_content.directories {
+        let directory = Path::new(&directory)
+            .file_name()
+            .expect("cannot extract dir name");
+        println!("{:?}", directory);
+        if directory.to_string_lossy().starts_with(name_prefix) {
+            src_dir = directory.to_string_lossy().into();
+            break;
+        };
+    }
+    if src_dir.is_empty() {
+        panic!("cannot find extracted Tenderdash sources")
+    }
+    parent.join(src_dir)
 }
 
 /// Copy generated files to target folder

proto-compiler/src/lib.rs

@@ -1,7 +1,4 @@
-use std::{
-    env::{self, var},
-    path::PathBuf,
-};
+use std::{env::var, path::PathBuf};
 
 use tempfile::tempdir;
 
@@ -21,18 +18,21 @@ use constants::{CUSTOM_FIELD_ATTRIBUTES, CUSTOM_TYPE_ATTRIBUTES, TENDERDASH_REPO
 /// ../proto/src/tenderdash.rs
 pub fn proto_compile() {
     let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+
     let tenderdash_lib_target = root
         .join("..")
         .join("proto")
         .join("src")
         .join("tenderdash.rs");
+
     let target_dir = root.join("..").join("proto").join("src").join("prost");
+
     let out_dir = var("OUT_DIR")
         .map(PathBuf::from)
         .or_else(|_| tempdir().map(|d| d.into_path()))
         .unwrap();
 
-    let cargo_target_dir = match env::var("CARGO_TARGET_DIR") {
+    let cargo_target_dir = match std::env::var("CARGO_TARGET_DIR") {
         Ok(s) => PathBuf::from(s),
         Err(_) => root.join("..").join("target"),
     };
@@ -48,7 +48,12 @@ pub fn proto_compile() {
 
     let commitish = tenderdash_commitish();
     println!("[info] => Fetching {TENDERDASH_REPO} at {commitish} into {tenderdash_dir:?}");
-    fetch_commitish(&PathBuf::from(&tenderdash_dir), TENDERDASH_REPO, &commitish); // This panics if it fails.
+    fetch_commitish(
+        &PathBuf::from(&tenderdash_dir),
+        &cargo_target_dir,
+        TENDERDASH_REPO,
+        &commitish,
+    ); // This panics if it fails.
 
     let proto_paths = vec![tenderdash_dir.join("proto").join("tendermint").join("abci")];
     let proto_includes_paths = vec![tenderdash_dir.join("proto"), thirdparty_dir];

proto/tests/unit.rs

@@ -133,3 +133,9 @@ pub fn test_response_exception_from() {
         "string"
     );
 }
+
+#[test]
+pub fn test_tenderdash_version() {
+    let version = env!("CARGO_PKG_VERSION");
+    assert_eq!(version, tenderdash_proto::meta::TENDERDASH_VERSION)
+}