From 9297bd8f6df6355d02ac43eaf1c73b1b7350ac3d Mon Sep 17 00:00:00 2001 From: peasee <98815791+peasee@users.noreply.github.com> Date: Fri, 4 Oct 2024 14:29:41 +1000 Subject: [PATCH 1/4] feat: Add user agent generation and test --- .github/workflows/build.yaml | 82 ++++++++++++++++++++++- src/main/java/ai/spice/Config.java | 24 ++++++- src/main/java/ai/spice/Version.java | 35 ++++++++++ src/test/java/ai/spice/UserAgentTest.java | 44 ++++++++++++ 4 files changed, 181 insertions(+), 4 deletions(-) create mode 100644 src/main/java/ai/spice/Version.java create mode 100644 src/test/java/ai/spice/UserAgentTest.java diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5d94cc0..286ffa6 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -8,10 +8,88 @@ on: workflow_dispatch: jobs: - build: + build_multi_os: + name: Build and test ${{matrix.os}} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + steps: + - uses: actions/checkout@v4 - runs-on: ubuntu-latest + - name: Set up JDK 17 (Oracle) + uses: actions/setup-java@v4 + with: + java-version: 17 + distribution: oracle + + - name: Display Java version + run: java -version + + - name: Display Javac version + run: javac -version + + - name: Display Maven version + run: mvn -version + + - name: Build + run: mvn install -DskipTests=true -Dgpg.skip -B -V + + - name: Install Spice (https://install.spiceai.org) (Linux) + if: matrix.os == 'ubuntu-latest' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + curl https://install.spiceai.org | /bin/bash + echo "$HOME/.spice/bin" >> $GITHUB_PATH + $HOME/.spice/bin/spice install + + - name: Install Spice (https://install.spiceai.org) (MacOS) + if: matrix.os == 'macos-latest' + run: | + brew install spiceai/spiceai/spice + brew install spiceai/spiceai/spiced + + - name: install Spice (Windows) + if: matrix.os == 'windows-latest' + run: | + curl -L "https://install.spiceai.org/Install.ps1" -o Install.ps1 && PowerShell -ExecutionPolicy Bypass -File ./Install.ps1 + + - name: add Spice bin to PATH (Windows) + if: matrix.os == 'windows-latest' + run: | + Add-Content $env:GITHUB_PATH (Join-Path $HOME ".spice\bin") + shell: pwsh + + - name: Init and start spice app (Unix) + if: matrix.os != 'windows-latest' + run: | + spice init spice_qs + cd spice_qs + spice add spiceai/quickstart + spiced &> spice.log & + # time to initialize added dataset + sleep 10 + + - name: Init and start spice app (Windows) + if: matrix.os == 'windows-latest' + run: | + spice init spice_qs + cd spice_qs + spice add spiceai/quickstart + Start-Process -FilePath spice run + # time to initialize added dataset + Start-Sleep -Seconds 10 + shell: pwsh + - name: Test + run: mvn test -B + env: + API_KEY: ${{ secrets.SPICE_CLOUD_API_KEY }} + + build: + runs-on: ubuntu-latest strategy: fail-fast: false matrix: diff --git a/src/main/java/ai/spice/Config.java b/src/main/java/ai/spice/Config.java index fc24d85..2a04a52 100644 --- a/src/main/java/ai/spice/Config.java +++ b/src/main/java/ai/spice/Config.java @@ -45,10 +45,10 @@ public class Config { LOCAL_FLIGHT_ADDRESS = System.getenv("SPICE_FLIGHT_URL") != null ? System.getenv("SPICE_FLIGHT_URL") : "http://localhost:50051"; - + CLOUD_HTTP_ADDRESS = System.getenv("SPICE_HTTP_URL") != null ? System.getenv("SPICE_HTTP_URL") : "https://data.spiceai.io"; - + LOCAL_HTTP_ADDRESS = System.getenv("SPICE_HTTP_URL") != null ? System.getenv("SPICE_HTTP_URL") : "http://localhost:8090"; } @@ -92,4 +92,24 @@ public static URI getLocalHttpAddressUri() throws URISyntaxException { public static URI getCloudHttpAddressUri() throws URISyntaxException { return new URI(CLOUD_HTTP_ADDRESS); } + + /** + * Returns the Spice SDK user agent for this system, including the package + * version, system OS, version and arch. + * + * @return the Spice SDK user agent string for this system. + */ + public static String getUserAgent() { + // change the os arch to match the pattern set in other SDKs + String osArch = System.getProperty("os.arch"); + if (osArch.equals("amd64")) { + osArch = "x86_64"; + } else if (osArch.equals("x86")) { + osArch = "i386"; + } + + return "spice-java " + Version.SPICE_JAVA_VERSION + " (" + System.getProperty("os.name") + "/" + + System.getProperty("os.version") + " " + + osArch + ")"; + } } \ No newline at end of file diff --git a/src/main/java/ai/spice/Version.java b/src/main/java/ai/spice/Version.java new file mode 100644 index 0000000..77725a1 --- /dev/null +++ b/src/main/java/ai/spice/Version.java @@ -0,0 +1,35 @@ +/* +Copyright 2024 The Spice.ai OSS Authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package ai.spice; + +public class Version { + /** + * spice-java SDK version, defined statically to support more platforms vs + * relying on .jar packaging + */ + public static final String SPICE_JAVA_VERSION; + + static { + SPICE_JAVA_VERSION = "0.3.0"; + } +} diff --git a/src/test/java/ai/spice/UserAgentTest.java b/src/test/java/ai/spice/UserAgentTest.java new file mode 100644 index 0000000..9d03fdb --- /dev/null +++ b/src/test/java/ai/spice/UserAgentTest.java @@ -0,0 +1,44 @@ +/* +Copyright 2024 The Spice.ai OSS Authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package ai.spice; + +import java.util.concurrent.ExecutionException; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import junit.framework.TestCase; + +public class UserAgentTest + extends TestCase { + public void testUserAgent() throws ExecutionException, InterruptedException { + // use a regex to match the expected user agent + String regex = "spice-java \\d+\\.\\d+\\.\\d+ \\((Linux|Windows|Darwin)/[\\d\\w\\.\\-\\_]+ (x86_64|aarch64|i386)\\)"; + Pattern pattern = Pattern.compile(regex); + + String userAgent = Config.getUserAgent(); + Matcher matcher = pattern.matcher(userAgent); + + assertTrue("User agent did not match the expected pattern: " + userAgent, matcher.matches()); + } +} From e52550fcd236b98e58f1688dbdb20133fe457739 Mon Sep 17 00:00:00 2001 From: peasee <98815791+peasee@users.noreply.github.com> Date: Fri, 4 Oct 2024 14:33:55 +1000 Subject: [PATCH 2/4] fix: MacOS and Windows test runs for useragent --- .github/workflows/build.yaml | 7 ++++++- src/main/java/ai/spice/Config.java | 8 +++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 286ffa6..38f8f7f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -33,8 +33,13 @@ jobs: - name: Display Maven version run: mvn -version - - name: Build + - name: Build (Unix) + if: matrix.os != 'windows-latest' run: mvn install -DskipTests=true -Dgpg.skip -B -V + + - name: Build (Windows) + if: matrix.os == 'windows-latest' + run: mvn --% install -DskipTests=true -Dgpg.skip -B -V # tell powershell to stop parsing with --% so it doesn't error with "Unknown lifecycle phase .skip" - name: Install Spice (https://install.spiceai.org) (Linux) if: matrix.os == 'ubuntu-latest' diff --git a/src/main/java/ai/spice/Config.java b/src/main/java/ai/spice/Config.java index 2a04a52..8ea758d 100644 --- a/src/main/java/ai/spice/Config.java +++ b/src/main/java/ai/spice/Config.java @@ -108,7 +108,13 @@ public static String getUserAgent() { osArch = "i386"; } - return "spice-java " + Version.SPICE_JAVA_VERSION + " (" + System.getProperty("os.name") + "/" + // change the os name to match the pattern set in other SDKs + String osName = System.getProperty("os.name"); + if (osName.equals("Mac OS X")) { + osName = "Darwin"; + } + + return "spice-java " + Version.SPICE_JAVA_VERSION + " (" + osName + "/" + System.getProperty("os.version") + " " + osArch + ")"; } From ac41df1152963d674beb04c83c64f70e19f0f286 Mon Sep 17 00:00:00 2001 From: peasee <98815791+peasee@users.noreply.github.com> Date: Fri, 4 Oct 2024 14:45:09 +1000 Subject: [PATCH 3/4] fix: User agent OS name and version on Windows --- src/main/java/ai/spice/Config.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/ai/spice/Config.java b/src/main/java/ai/spice/Config.java index 8ea758d..9a6a591 100644 --- a/src/main/java/ai/spice/Config.java +++ b/src/main/java/ai/spice/Config.java @@ -112,6 +112,15 @@ public static String getUserAgent() { String osName = System.getProperty("os.name"); if (osName.equals("Mac OS X")) { osName = "Darwin"; + } else if (osName.contains("Windows")) { + osName = "Windows"; // this renames Windows OS names like "Windows Server 2022" to just "Windows" + } + + // The Windows OS version strings also include the arch after the version, so we + // need to remove it + String osVersion = System.getProperty("os.version"); + if (osName.equals("Windows")) { + osVersion = osVersion.split(" ")[0]; } return "spice-java " + Version.SPICE_JAVA_VERSION + " (" + osName + "/" From eec957d5df9d338a2f48d80006eefe167f721427 Mon Sep 17 00:00:00 2001 From: peasee <98815791+peasee@users.noreply.github.com> Date: Fri, 4 Oct 2024 15:16:43 +1000 Subject: [PATCH 4/4] feat: Add x-spice-user-agent header --- .../ai/spice/HeaderAuthMiddlewareFactory.java | 42 +++++++++++++++++++ src/main/java/ai/spice/SpiceClient.java | 17 ++++++-- 2 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 src/main/java/ai/spice/HeaderAuthMiddlewareFactory.java diff --git a/src/main/java/ai/spice/HeaderAuthMiddlewareFactory.java b/src/main/java/ai/spice/HeaderAuthMiddlewareFactory.java new file mode 100644 index 0000000..e94c681 --- /dev/null +++ b/src/main/java/ai/spice/HeaderAuthMiddlewareFactory.java @@ -0,0 +1,42 @@ +package ai.spice; + +import java.util.Map; + +import org.apache.arrow.flight.CallHeaders; +import org.apache.arrow.flight.CallInfo; +import org.apache.arrow.flight.CallStatus; +import org.apache.arrow.flight.FlightClientMiddleware; +import org.apache.arrow.flight.FlightClientMiddleware.Factory; +import org.apache.arrow.flight.auth2.ClientIncomingAuthHeaderMiddleware; + +public class HeaderAuthMiddlewareFactory implements Factory { + private final ClientIncomingAuthHeaderMiddleware.Factory authFactory; + private final Map headers; + + public HeaderAuthMiddlewareFactory(ClientIncomingAuthHeaderMiddleware.Factory authFactory, + Map headers) { + this.authFactory = authFactory; + this.headers = headers; + } + + @Override + public FlightClientMiddleware onCallStarted(CallInfo callInfo) { + return new FlightClientMiddleware() { + @Override + public void onBeforeSendingHeaders(CallHeaders callHeaders) { + authFactory.onCallStarted(callInfo).onBeforeSendingHeaders(callHeaders); + headers.forEach(callHeaders::insert); + } + + @Override + public void onHeadersReceived(CallHeaders callHeaders) { + authFactory.onCallStarted(callInfo).onHeadersReceived(callHeaders); + } + + @Override + public void onCallCompleted(CallStatus callStatus) { + authFactory.onCallStarted(callInfo).onCallCompleted(callStatus); + } + }; + } +} \ No newline at end of file diff --git a/src/main/java/ai/spice/SpiceClient.java b/src/main/java/ai/spice/SpiceClient.java index 07842d0..2fba260 100644 --- a/src/main/java/ai/spice/SpiceClient.java +++ b/src/main/java/ai/spice/SpiceClient.java @@ -28,6 +28,8 @@ of this software and associated documentation files (the "Software"), to deal import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.ExecutionException; import org.apache.arrow.flight.CallStatus; @@ -112,12 +114,20 @@ public SpiceClient(String appId, String apiKey, URI flightAddress, URI httpAddre return; } - final ClientIncomingAuthHeaderMiddleware.Factory factory = new ClientIncomingAuthHeaderMiddleware.Factory( + // prepare additional headers to insert into Flight requests + Map headers = new HashMap<>(); + headers.put("X-Spice-User-Agent", Config.getUserAgent()); + + final ClientIncomingAuthHeaderMiddleware.Factory authFactory = new ClientIncomingAuthHeaderMiddleware.Factory( new ClientBearerHeaderHandler()); - final FlightClient client = builder.intercept(factory).build(); + // builder can't chain .intercept()s, so we need to chain the middleware + // factories instead + final HeaderAuthMiddlewareFactory combinedFactory = new HeaderAuthMiddlewareFactory(authFactory, headers); + + final FlightClient client = builder.intercept(combinedFactory).build(); client.handshake(new CredentialCallOption(new BasicAuthCredentialWriter(this.appId, this.apiKey))); - this.authCallOptions = factory.getCredentialCallOption(); + this.authCallOptions = authFactory.getCredentialCallOption(); this.flightClient = new FlightSqlClient(client); } @@ -151,6 +161,7 @@ public void refresh(String dataset) throws ExecutionException { HttpRequest request = HttpRequest.newBuilder() .uri(new URI(String.format("%s/v1/datasets/%s/acceleration/refresh", this.httpAddress, dataset))) .header("Content-Type", "application/json") + .header("X-Spice-User-Agent", Config.getUserAgent()) .POST(HttpRequest.BodyPublishers.noBody()) .build();