diff --git a/.github/workflows/ql-for-ql-build.yml b/.github/workflows/ql-for-ql-build.yml index e8ac1fa0f173..62d0620b13b4 100644 --- a/.github/workflows/ql-for-ql-build.yml +++ b/.github/workflows/ql-for-ql-build.yml @@ -32,7 +32,7 @@ jobs: path: | ql/extractor-pack/ ql/target/release/buramu - key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('shared/tree-sitter-extractor') }}-${{ hashFiles('ql/**/*.rs') }} + key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ruby/extractor/codeql-extractor') }}-${{ hashFiles('ql/**/*.rs') }} - name: Cache cargo if: steps.cache-extractor.outputs.cache-hit != 'true' uses: actions/cache@v3 @@ -44,20 +44,20 @@ jobs: key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-rust-cargo-${{ hashFiles('ql/**/Cargo.lock') }} - name: Release build if: steps.cache-extractor.outputs.cache-hit != 'true' - run: cd ql; ./scripts/create-extractor-pack.sh + run: cd ql; ./scripts/create-extractor-pack.sh env: - GH_TOKEN: ${{ github.token }} + GH_TOKEN: ${{ github.token }} - name: Cache compilation cache id: query-cache uses: ./.github/actions/cache-query-compilation - with: + with: key: run-ql-for-ql - name: Make database and analyze run: | ./ql/target/release/buramu | tee deprecated.blame # Add a blame file for the extractor to parse. ${CODEQL} database create -l=ql --search-path ql/extractor-pack ${DB} ${CODEQL} database analyze -j0 --format=sarif-latest --output=ql-for-ql.sarif ${DB} ql/ql/src/codeql-suites/ql-code-scanning.qls --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" - env: + env: CODEQL: ${{ steps.find-codeql.outputs.codeql-path }} DB: ${{ runner.temp }}/DB LGTM_INDEX_FILTERS: | diff --git a/.github/workflows/ruby-build.yml b/.github/workflows/ruby-build.yml index 392c6ff83026..12f2b0d7c21e 100644 --- a/.github/workflows/ruby-build.yml +++ b/.github/workflows/ruby-build.yml @@ -61,7 +61,7 @@ jobs: ruby/extractor/target/release/codeql-extractor-ruby ruby/extractor/target/release/codeql-extractor-ruby.exe ruby/extractor/ql/lib/codeql/ruby/ast/internal/TreeSitter.qll - key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-extractor-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/extractor/Cargo.lock') }}-${{ hashFiles('shared/tree-sitter-extractor') }}-${{ hashFiles('ruby/extractor/**/*.rs') }} + key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-extractor-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/extractor/Cargo.lock') }}-${{ hashFiles('ruby/extractor/codeql-extractor') }}-${{ hashFiles('ruby/extractor/**/*.rs') }} - uses: actions/cache@v3 if: steps.cache-extractor.outputs.cache-hit != 'true' with: @@ -119,7 +119,7 @@ jobs: - name: Cache compilation cache id: query-cache uses: ./.github/actions/cache-query-compilation - with: + with: key: ruby-build - name: Build Query Pack run: | diff --git a/.github/workflows/tree-sitter-extractor-test.yml b/.github/workflows/tree-sitter-extractor-test.yml index 5d13b25466d3..6ff47d041787 100644 --- a/.github/workflows/tree-sitter-extractor-test.yml +++ b/.github/workflows/tree-sitter-extractor-test.yml @@ -3,14 +3,14 @@ name: Test tree-sitter-extractor on: push: paths: - - "shared/tree-sitter-extractor/**" + - "ruby/extractor/codeql-extractor/**" - .github/workflows/tree-sitter-extractor-test.yml branches: - main - "rc/*" pull_request: paths: - - "shared/tree-sitter-extractor/**" + - "ruby/extractor/codeql-extractor/**" - .github/workflows/tree-sitter-extractor-test.yml branches: - main @@ -21,7 +21,7 @@ env: defaults: run: - working-directory: shared/tree-sitter-extractor + working-directory: ruby/extractor/codeql-extractor jobs: test: @@ -33,13 +33,13 @@ jobs: - name: Run tests run: cargo test --verbose fmt: - runs-on: ubuntu-latest + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Check formatting run: cargo fmt --check clippy: - runs-on: ubuntu-latest + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Run clippy diff --git a/ql/extractor/Cargo.toml b/ql/extractor/Cargo.toml index 301abf60afbf..0a9bf2c8aa5a 100644 --- a/ql/extractor/Cargo.toml +++ b/ql/extractor/Cargo.toml @@ -17,4 +17,4 @@ tracing = "0.1" tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } rayon = "1.8.1" regex = "1.10.3" -codeql-extractor = { path = "../../shared/tree-sitter-extractor" } +codeql-extractor = { path = "../../ruby/extractor/codeql-extractor" } diff --git a/ruby/actions/create-extractor-pack/action.yml b/ruby/actions/create-extractor-pack/action.yml index 84f67b812af3..d3cc92df55fb 100644 --- a/ruby/actions/create-extractor-pack/action.yml +++ b/ruby/actions/create-extractor-pack/action.yml @@ -10,7 +10,7 @@ runs: uses: actions/cache@v3 with: path: ruby/extractor-pack - key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-extractor-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/scripts/create-extractor-pack.sh', 'ruby/extractor/**/Cargo.lock', 'ruby/actions/create-extractor-pack/action.yml') }}-${{ hashFiles('shared/tree-sitter-extractor') }}-${{ hashFiles('ruby/extractor/**/*.rs') }}-${{ hashFiles('ruby/codeql-extractor.yml', 'ruby/downgrades', 'ruby/tools', 'ruby/ql/lib/ruby.dbscheme', 'ruby/ql/lib/ruby.dbscheme.stats') }} + key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-extractor-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/scripts/create-extractor-pack.sh', 'ruby/extractor/**/Cargo.lock', 'ruby/actions/create-extractor-pack/action.yml') }}-${{ hashFiles('ruby/extractor/codeql-extractor') }}-${{ hashFiles('ruby/extractor/**/*.rs') }}-${{ hashFiles('ruby/codeql-extractor.yml', 'ruby/downgrades', 'ruby/tools', 'ruby/ql/lib/ruby.dbscheme', 'ruby/ql/lib/ruby.dbscheme.stats') }} - name: Cache cargo uses: actions/cache@v3 if: steps.cache-extractor.outputs.cache-hit != 'true' diff --git a/ruby/extractor/Cargo.lock b/ruby/extractor/Cargo.lock index 328e597dd3bb..4dde584d1b84 100644 --- a/ruby/extractor/Cargo.lock +++ b/ruby/extractor/Cargo.lock @@ -183,12 +183,15 @@ dependencies = [ "globset", "lazy_static", "num_cpus", + "rand", "rayon", "regex", "serde", "serde_json", "tracing", "tree-sitter", + "tree-sitter-json", + "tree-sitter-ql", ] [[package]] @@ -436,6 +439,17 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "getrandom" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + [[package]] name = "globset" version = "0.4.14" @@ -540,9 +554,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.141" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "link-cplusplus" @@ -655,6 +669,12 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + [[package]] name = "proc-macro2" version = "1.0.56" @@ -673,6 +693,36 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + [[package]] name = "rayon" version = "1.7.0" @@ -870,7 +920,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" dependencies = [ "libc", - "wasi", + "wasi 0.10.0+wasi-snapshot-preview1", "winapi", ] @@ -955,6 +1005,24 @@ dependencies = [ "tree-sitter", ] +[[package]] +name = "tree-sitter-json" +version = "0.20.0" +source = "git+https://github.com/tausbn/tree-sitter-json#745663ee997f1576fe1e7187e6347e0db36ec7a9" +dependencies = [ + "cc", + "tree-sitter", +] + +[[package]] +name = "tree-sitter-ql" +version = "0.19.0" +source = "git+https://github.com/tree-sitter/tree-sitter-ql#bd087020f0d8c183080ca615d38de0ec827aeeaf" +dependencies = [ + "cc", + "tree-sitter", +] + [[package]] name = "tree-sitter-ruby" version = "0.20.0" @@ -994,6 +1062,12 @@ version = "0.10.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + [[package]] name = "wasm-bindgen" version = "0.2.84" diff --git a/ruby/extractor/Cargo.toml b/ruby/extractor/Cargo.toml index 066505c3ef60..6ba7c578771f 100644 --- a/ruby/extractor/Cargo.toml +++ b/ruby/extractor/Cargo.toml @@ -1,3 +1,5 @@ +[workspace] + [package] name = "codeql-extractor-ruby" description = "CodeQL Ruby extractor" @@ -18,4 +20,4 @@ rayon = "1.5.0" regex = "1.7.1" encoding = "0.2" lazy_static = "1.4.0" -codeql-extractor = { path = "../../shared/tree-sitter-extractor" } +codeql-extractor = { path = "codeql-extractor" } diff --git a/ruby/extractor/Cross.toml b/ruby/extractor/Cross.toml index f3ed51aee597..17726bd5e821 100644 --- a/ruby/extractor/Cross.toml +++ b/ruby/extractor/Cross.toml @@ -1,8 +1,2 @@ [target.x86_64-unknown-linux-gnu] image = "centos/devtoolset-7-toolchain-centos7" - -[build.env] -# Provide the path to the shared extractor -# Cross mounts this directory as a volume, so builds inside the docker container -# can see it. -volumes = ["__CODEQL-EXTRACTOR=../../shared/tree-sitter-extractor"] diff --git a/shared/tree-sitter-extractor/.gitignore b/ruby/extractor/codeql-extractor/.gitignore similarity index 100% rename from shared/tree-sitter-extractor/.gitignore rename to ruby/extractor/codeql-extractor/.gitignore diff --git a/shared/tree-sitter-extractor/Cargo.toml b/ruby/extractor/codeql-extractor/Cargo.toml similarity index 100% rename from shared/tree-sitter-extractor/Cargo.toml rename to ruby/extractor/codeql-extractor/Cargo.toml diff --git a/shared/tree-sitter-extractor/rust-toolchain.toml b/ruby/extractor/codeql-extractor/rust-toolchain.toml similarity index 100% rename from shared/tree-sitter-extractor/rust-toolchain.toml rename to ruby/extractor/codeql-extractor/rust-toolchain.toml diff --git a/shared/tree-sitter-extractor/src/autobuilder.rs b/ruby/extractor/codeql-extractor/src/autobuilder.rs similarity index 100% rename from shared/tree-sitter-extractor/src/autobuilder.rs rename to ruby/extractor/codeql-extractor/src/autobuilder.rs diff --git a/shared/tree-sitter-extractor/src/diagnostics.rs b/ruby/extractor/codeql-extractor/src/diagnostics.rs similarity index 100% rename from shared/tree-sitter-extractor/src/diagnostics.rs rename to ruby/extractor/codeql-extractor/src/diagnostics.rs diff --git a/shared/tree-sitter-extractor/src/extractor/mod.rs b/ruby/extractor/codeql-extractor/src/extractor/mod.rs similarity index 100% rename from shared/tree-sitter-extractor/src/extractor/mod.rs rename to ruby/extractor/codeql-extractor/src/extractor/mod.rs diff --git a/shared/tree-sitter-extractor/src/extractor/simple.rs b/ruby/extractor/codeql-extractor/src/extractor/simple.rs similarity index 100% rename from shared/tree-sitter-extractor/src/extractor/simple.rs rename to ruby/extractor/codeql-extractor/src/extractor/simple.rs diff --git a/shared/tree-sitter-extractor/src/file_paths.rs b/ruby/extractor/codeql-extractor/src/file_paths.rs similarity index 100% rename from shared/tree-sitter-extractor/src/file_paths.rs rename to ruby/extractor/codeql-extractor/src/file_paths.rs diff --git a/shared/tree-sitter-extractor/src/generator/dbscheme.rs b/ruby/extractor/codeql-extractor/src/generator/dbscheme.rs similarity index 100% rename from shared/tree-sitter-extractor/src/generator/dbscheme.rs rename to ruby/extractor/codeql-extractor/src/generator/dbscheme.rs diff --git a/shared/tree-sitter-extractor/src/generator/language.rs b/ruby/extractor/codeql-extractor/src/generator/language.rs similarity index 100% rename from shared/tree-sitter-extractor/src/generator/language.rs rename to ruby/extractor/codeql-extractor/src/generator/language.rs diff --git a/shared/tree-sitter-extractor/src/generator/mod.rs b/ruby/extractor/codeql-extractor/src/generator/mod.rs similarity index 100% rename from shared/tree-sitter-extractor/src/generator/mod.rs rename to ruby/extractor/codeql-extractor/src/generator/mod.rs diff --git a/shared/tree-sitter-extractor/src/generator/prefix.dbscheme b/ruby/extractor/codeql-extractor/src/generator/prefix.dbscheme similarity index 100% rename from shared/tree-sitter-extractor/src/generator/prefix.dbscheme rename to ruby/extractor/codeql-extractor/src/generator/prefix.dbscheme diff --git a/shared/tree-sitter-extractor/src/generator/ql.rs b/ruby/extractor/codeql-extractor/src/generator/ql.rs similarity index 100% rename from shared/tree-sitter-extractor/src/generator/ql.rs rename to ruby/extractor/codeql-extractor/src/generator/ql.rs diff --git a/shared/tree-sitter-extractor/src/generator/ql_gen.rs b/ruby/extractor/codeql-extractor/src/generator/ql_gen.rs similarity index 100% rename from shared/tree-sitter-extractor/src/generator/ql_gen.rs rename to ruby/extractor/codeql-extractor/src/generator/ql_gen.rs diff --git a/shared/tree-sitter-extractor/src/lib.rs b/ruby/extractor/codeql-extractor/src/lib.rs similarity index 100% rename from shared/tree-sitter-extractor/src/lib.rs rename to ruby/extractor/codeql-extractor/src/lib.rs diff --git a/shared/tree-sitter-extractor/src/node_types.rs b/ruby/extractor/codeql-extractor/src/node_types.rs similarity index 100% rename from shared/tree-sitter-extractor/src/node_types.rs rename to ruby/extractor/codeql-extractor/src/node_types.rs diff --git a/shared/tree-sitter-extractor/src/options.rs b/ruby/extractor/codeql-extractor/src/options.rs similarity index 100% rename from shared/tree-sitter-extractor/src/options.rs rename to ruby/extractor/codeql-extractor/src/options.rs diff --git a/shared/tree-sitter-extractor/src/trap.rs b/ruby/extractor/codeql-extractor/src/trap.rs similarity index 100% rename from shared/tree-sitter-extractor/src/trap.rs rename to ruby/extractor/codeql-extractor/src/trap.rs diff --git a/shared/tree-sitter-extractor/tests/common/mod.rs b/ruby/extractor/codeql-extractor/tests/common/mod.rs similarity index 100% rename from shared/tree-sitter-extractor/tests/common/mod.rs rename to ruby/extractor/codeql-extractor/tests/common/mod.rs diff --git a/shared/tree-sitter-extractor/tests/integration_test.rs b/ruby/extractor/codeql-extractor/tests/integration_test.rs similarity index 100% rename from shared/tree-sitter-extractor/tests/integration_test.rs rename to ruby/extractor/codeql-extractor/tests/integration_test.rs diff --git a/shared/tree-sitter-extractor/tests/multiple_languages.rs b/ruby/extractor/codeql-extractor/tests/multiple_languages.rs similarity index 100% rename from shared/tree-sitter-extractor/tests/multiple_languages.rs rename to ruby/extractor/codeql-extractor/tests/multiple_languages.rs