Add mbtiles integration tests

.github/workflows/ci.yml

@@ -62,14 +62,18 @@ jobs:
             rustup target add aarch64-apple-darwin
             # compile without debug symbols
             RUSTFLAGS='-C link-arg=-s' cargo build --release --target ${{ matrix.target }} --features=vendored-openssl
+            RUSTFLAGS='-C link-arg=-s' cargo build --release --target ${{ matrix.target }} --features=tools --package mbtiles
           else
             cargo build --release --target ${{ matrix.target }} --features=ssl
+            cargo build --release --target ${{ matrix.target }} --features=tools --package mbtiles
           fi
           mkdir target_releases
           if [[ "${{ runner.os }}" == "Windows" ]]; then
             mv target/${{ matrix.target }}/release/martin.exe target_releases
+            mv target/${{ matrix.target }}/release/mbtiles.exe target_releases
           else
             mv target/${{ matrix.target }}/release/martin target_releases
+            mv target/${{ matrix.target }}/release/mbtiles target_releases
           fi
       - name: Save build artifact build-${{ matrix.target }}
         uses: actions/upload-artifact@v3
@@ -86,19 +90,19 @@ jobs:
       matrix:
         include:
           - os: ubuntu-latest
-            bin: martin
+            ext: ''
             target: x86_64-unknown-linux-gnu
             name: martin-Linux-x86_64.tar.gz
           - os: windows-latest
-            bin: martin.exe
+            ext: '.exe'
             target: x86_64-pc-windows-msvc
             name: martin-Windows-x86_64.zip
           - os: macOS-latest
-            bin: martin
+            ext: ''
             target: x86_64-apple-darwin
             name: martin-Darwin-x86_64.tar.gz
           - os: ubuntu-latest
-            bin: martin
+            ext: ''
             target: aarch64-apple-darwin
             name: martin-Darwin-aarch64.tar.gz
     runs-on: ${{ matrix.os }}
@@ -147,13 +151,17 @@ jobs:
         shell: bash
         run: |
           if [[ "${{ runner.os }}" != "Windows" ]]; then
-            chmod +x target/${{ matrix.bin }}
+            chmod +x target/martin${{ matrix.ext }}
+            chmod +x target/mbtiles${{ matrix.ext }}
           fi
           tests/test.sh
         env:
           DATABASE_URL: ${{ steps.pg.outputs.connection-uri }}
           MARTIN_BUILD: "-"
-          MARTIN_BIN: target/${{ matrix.bin }}
+          MARTIN_BIN: target/martin${{ matrix.ext }}
+          MBTILES_BUILD: "-"
+          MBTILES_BIN: target/mbtiles${{ matrix.ext }}
+
       - name: Compare test output results (Linux)
         if: matrix.target == 'x86_64-unknown-linux-gnu'
         run: diff --brief --recursive --new-file tests/output tests/expected
@@ -170,12 +178,13 @@ jobs:
           cd target/
           # Symbol stripping does not work cross-platform
           if [[ "${{ matrix.target }}" != "aarch64-apple-darwin" ]]; then
-            strip ${{ matrix.bin }}
+            strip martin${{ matrix.ext }}
+            strip mbtiles${{ matrix.ext }}
           fi
           if [[ "${{ runner.os }}" == "Windows" ]]; then
-            7z a ../${{ matrix.name }} ${{ matrix.bin }}
+            7z a ../${{ matrix.name }} martin${{ matrix.ext }} mbtiles${{ matrix.ext }}
           else
-            tar czvf ../${{ matrix.name }} ${{ matrix.bin }}
+            tar czvf ../${{ matrix.name }} martin${{ matrix.ext }} mbtiles${{ matrix.ext }}
           fi
           cd -
       - name: Generate SHA-256 (MacOS)
@@ -276,9 +285,11 @@ jobs:
         path: target/
     - name: Integration Tests
       run: |
-        chmod +x target/martin
+        chmod +x target/martin target/mbtiles
         tests/test.sh
       env:
         DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=${{ matrix.sslmode }}
         MARTIN_BUILD: "-"
         MARTIN_BIN: target/martin
+        MBTILES_BUILD: "-"
+        MBTILES_BIN: target/mbtiles

.github/workflows/docker.yml

@@ -94,6 +94,8 @@ jobs:
         if: matrix.platform == 'linux/amd64'
         run: |
           TAG=$(echo '${{ steps.docker_meta.outputs.json }}' | jq -r '.tags[0]')
+          export MBTILES_BUILD=-
+          export MBTILES_BIN=-
           export MARTIN_BUILD=-
           export MARTIN_BIN="docker run --rm --net host -e DATABASE_URL -v $PWD/tests:/tests $TAG"
           echo "MARTIN_BIN=$MARTIN_BIN"

Cargo.toml

@@ -27,11 +27,11 @@ ssl = ["openssl", "postgres-openssl"]
 vendored-openssl = ["ssl", "openssl/vendored"]
 
 [dependencies]
-actix.workspace = true
 actix-cors.workspace = true
 actix-http.workspace = true
 actix-rt.workspace = true
 actix-web.workspace = true
+actix.workspace = true
 async-trait.workspace = true
 brotli.workspace = true
 clap.workspace = true
@@ -44,12 +44,10 @@ log.workspace = true
 martin-mbtiles.workspace = true
 martin-tile-utils.workspace = true
 num_cpus.workspace = true
-openssl = { workspace = true, optional = true }
 pmtiles.workspace = true
 postgis.workspace = true
-postgres.workspace = true
-postgres-openssl = { workspace = true, optional = true }
 postgres-protocol.workspace = true
+postgres.workspace = true
 regex.workspace = true
 semver.workspace = true
 serde.workspace = true
@@ -59,6 +57,10 @@ subst.workspace = true
 thiserror.workspace = true
 tilejson.workspace = true
 
+# Optional dependencies for openssl support
+openssl = { workspace = true, optional = true }
+postgres-openssl = { workspace = true, optional = true }
+
 [dev-dependencies]
 cargo-husky.workspace = true
 criterion.workspace = true

martin-mbtiles/Cargo.toml

@@ -13,6 +13,11 @@ rust-version.workspace = true
 repository.workspace = true
 license.workspace = true
 
+[features]
+# TODO: Disable "tools" feature in default builds
+default = ["tools"]
+tools = ["anyhow", "clap", "tokio"]
+
 [dependencies]
 futures.workspace = true
 log.workspace = true
@@ -23,10 +28,9 @@ thiserror.workspace = true
 tilejson.workspace = true
 
 # Bin dependencies
-# TODO: Make these only enabled for the bin target, i.e. by using a feature, and add a just task to run the bin
-anyhow.workspace = true
-clap.workspace = true
-tokio.workspace = true
+anyhow = { workspace = true, optional = true }
+clap = { workspace = true, optional = true }
+tokio = { workspace = true, optional = true }
 
 [dev-dependencies]
 # For testing, might as well use the same async framework as the Martin itself
@@ -38,3 +42,4 @@ path = "src/lib.rs"
 [[bin]]
 name = "mbtiles"
 path = "src/bin/main.rs"
+required-features = ["tools"]

tests/expected/mbtiles/help.txt

@@ -0,0 +1,11 @@
+A utility to work with .mbtiles file content
+
+Usage: mbtiles <COMMAND>
+
+Commands:
+  meta-get  Gets a single value from metadata table
+  help      Print this message or the help of the given subcommand(s)
+
+Options:
+  -h, --help     Print help
+  -V, --version  Print version

tests/expected/mbtiles/meta-get_help.txt

@@ -0,0 +1,10 @@
+Gets a single value from metadata table
+
+Usage: mbtiles meta-get <FILE> <KEY>
+
+Arguments:
+  <FILE>  MBTiles file to read a value from
+  <KEY>   Value to read
+
+Options:
+  -h, --help  Print help

tests/expected/mbtiles/meta-get_name.txt

@@ -0,0 +1 @@
+Major cities from Natural Earth data

tests/test.sh

@@ -10,6 +10,9 @@ MARTIN_URL="http://localhost:${MARTIN_PORT}"
 MARTIN_ARGS="${MARTIN_ARGS:---listen-addresses localhost:${MARTIN_PORT}}"
 MARTIN_BIN="${MARTIN_BIN:-cargo run --all-features --} ${MARTIN_ARGS}"
 
+MBTILES_BUILD="${MBTILES_BUILD:-cargo build -p martin-mbtiles}"
+MBTILES_BIN="${MBTILES_BIN:-target/debug/mbtiles}"
+
 function wait_for_martin {
     # Seems the --retry-all-errors option is not available on older curl versions, but maybe in the future we can just use this:
     # timeout -k 20s 20s curl --retry 10 --retry-all-errors --retry-delay 1 -sS "$MARTIN_URL/health"
@@ -130,11 +133,14 @@ validate_log()
 
 curl --version
 
-# Make sure martin is built - this way it won't timeout while waiting for it to start
-# If MARTIN_BUILD is set to "-", don't build
+# Make sure martin and mbtiles are built - this way it won't timeout while waiting for it to start
+# If set to "-", don't build
 if [[ "$MARTIN_BUILD" != "-" ]]; then
   $MARTIN_BUILD
 fi
+if [[ "$MBTILES_BUILD" != "-" ]]; then
+  $MBTILES_BUILD
+fi
 
 
 echo "------------------------------------------------------------------------------------------------------------------------"
@@ -251,4 +257,23 @@ validate_log test_log_2.txt
 remove_line "$(dirname "$0")/output/given_config.yaml"       " connection_string: "
 remove_line "$(dirname "$0")/output/generated_config.yaml"   " connection_string: "
 
+
+echo "------------------------------------------------------------------------------------------------------------------------"
+echo "Test mbtiles utility"
+if [[ "$MBTILES_BIN" != "-" ]]; then
+  TEST_OUT_DIR="$(dirname "$0")/output/mbtiles"
+  mkdir -p "$TEST_OUT_DIR"
+
+  set -x
+
+  $MBTILES_BIN --help 2>&1 | tee "$TEST_OUT_DIR/help.txt"
+  $MBTILES_BIN meta-get --help 2>&1 | tee "$TEST_OUT_DIR/meta-get_help.txt"
+  $MBTILES_BIN meta-get ./tests/fixtures/files/world_cities.mbtiles name 2>&1 | tee "$TEST_OUT_DIR/meta-get_name.txt"
+  $MBTILES_BIN meta-get ./tests/fixtures/files/world_cities.mbtiles missing_value 2>&1 | tee "$TEST_OUT_DIR/meta-get_missing_value.txt"
+
+  { set +x; } 2> /dev/null
+else
+  echo "Skipping mbtiles utility tests"
+fi
+
 >&2 echo "All integration tests have passed"