Merge pull request #5 from chenyufeifei/feat/workflows

Feat/workflows

.github/workflows/build.yml

@@ -0,0 +1,58 @@
+name: Build
+
+on:
+  push:
+    branches: [main, release-*]
+  pull_request:
+    branches: [main, release-*]
+
+jobs:
+  build:
+    name: Build libtritonrepoagent_dragonfly.so
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update && sudo apt-get install -y \
+          build-essential \
+          cmake \
+          git \
+          curl \
+          zip \
+          unzip \
+          tar \
+          pkg-config \
+          python3
+
+      - name: Install vcpkg
+        run: |
+          git clone https://github.com/Microsoft/vcpkg.git $HOME/vcpkg
+          $HOME/vcpkg/bootstrap-vcpkg.sh
+
+      - name: Install project dependencies with vcpkg
+        run: |
+          $HOME/vcpkg/vcpkg install re2
+          $HOME/vcpkg/vcpkg install aws-sdk-cpp
+          $HOME/vcpkg/vcpkg install google-cloud-cpp[storage]
+          $HOME/vcpkg/vcpkg install azure-storage-blobs-cpp
+          $HOME/vcpkg/vcpkg install rapidjson
+
+      - name: Create build directory
+        run: mkdir -p $HOME/build
+
+      - name: Build the repository agent
+        run: |
+          mkdir -p $HOME/build && cd $HOME/build
+          cmake -DCMAKE_TOOLCHAIN_FILE=$HOME/vcpkg/scripts/buildsystems/vcpkg.cmake \
+                -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install \
+                -DTRITON_ENABLE_GCS=true \
+                -DTRITON_ENABLE_AZURE_STORAGE=true \
+                -DTRITON_ENABLE_S3=true \
+                -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \
+                $GITHUB_WORKSPACE
+          make install

.github/workflows/lint.yml

@@ -0,0 +1,30 @@
+name: Lint
+
+on:
+  push:
+    branches: [main, release-*]
+  pull_request:
+    branches: [main, release-*]
+
+jobs:
+  lint:
+    name: CPP Lint
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install clang-format
+        run: sudo apt-get install -y clang-format
+
+      - name: Run clang-format
+        run: |
+          find . -iname *.h -o -iname *.cpp | xargs clang-format -i
+
+      - name: Check for changes
+        run: |
+          if ! git diff --exit-code; then
+            echo "Code style issues found"
+            git diff
+            exit 1
+          fi

CMakeLists.txt

@@ -101,16 +101,19 @@ target_link_libraries(triton-dragonfly-repoagent PRIVATE re2::re2)
 #
 if(${TRITON_ENABLE_S3})
     find_package(ZLIB REQUIRED)
-    find_package(AWSSDK REQUIRED COMPONENTS s3)
+    find_package(AWSSDK REQUIRED COMPONENTS core s3)
     message(STATUS "Using aws-sdk-cpp ${AWSSDK_VERSION}")
     target_include_directories(
             triton-dragonfly-repoagent
-            PRIVATE $<TARGET_PROPERTY:aws-cpp-sdk-s3,INTERFACE_INCLUDE_DIRECTORIES>
+            PRIVATE
+            $<TARGET_PROPERTY:aws-cpp-sdk-core,INTERFACE_INCLUDE_DIRECTORIES>
+            $<TARGET_PROPERTY:aws-cpp-sdk-s3,INTERFACE_INCLUDE_DIRECTORIES>
     )
+
     target_link_libraries(
             triton-dragonfly-repoagent
             PRIVATE
-            aws-cpp-sdk-s3
+            aws-cpp-sdk-s3 aws-cpp-sdk-core
     )
 endif() # TRITON_ENABLE_S3
 

Dockerfile

@@ -0,0 +1,56 @@
+# Use a base image that includes development essentials and git
+FROM ubuntu:latest AS builder
+
+# Install build tools, git, and other dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    cmake \
+    git \
+    curl \
+    zip \
+    unzip \
+    tar \
+    pkg-config \
+    python3
+
+# Install vcpkg
+RUN git clone https://github.com/Microsoft/vcpkg.git /vcpkg \
+    && /vcpkg/bootstrap-vcpkg.sh
+
+# Install the required dependencies using vcpkg
+RUN /vcpkg/vcpkg install re2 \
+    && /vcpkg/vcpkg install aws-sdk-cpp \
+    && /vcpkg/vcpkg install google-cloud-cpp[storage] \
+    && /vcpkg/vcpkg install azure-storage-blobs-cpp \
+    && /vcpkg/vcpkg install rapidjson
+
+RUN mkdir -p /dragonfly-repository-agent/build
+
+# Assuming the source code is located in a directory called 'source'
+# You can COPY this in or clone from a repository as needed
+COPY ./src /dragonfly-repository-agent/src
+COPY ./cmake /dragonfly-repository-agent/cmake
+COPY ./CMakeLists.txt /dragonfly-repository-agent/CMakeLists.txt
+
+# Set the working directory
+WORKDIR /dragonfly-repository-agent/build
+
+# Set the environment variable for the vcpkg toolchain file
+ENV VCPKG_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake
+
+# Build the repository agent
+RUN cmake -DCMAKE_TOOLCHAIN_FILE=${VCPKG_TOOLCHAIN_FILE} \
+          -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install \
+          -DTRITON_ENABLE_GCS=true \
+          -DTRITON_ENABLE_AZURE_STORAGE=true \
+          -DTRITON_ENABLE_S3=true \
+          -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \
+          ..
+
+RUN make install
+
+FROM nvcr.io/nvidia/tritonserver:23.08-py3
+
+RUN mkdir /opt/tritonserver/repoagents/dragonfly
+
+COPY --from=builder /dragonfly-repository-agent/build/libtritonrepoagent_dragonfly.so /opt/tritonserver/repoagents/dragonfly/libtritonrepoagent_dragonfly.so

src/common_utils.h

@@ -1,28 +1,28 @@
 /*
-*     Copyright 2023 The Dragonfly Authors
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*      http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
+ *     Copyright 2023 The Dragonfly Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 #pragma once
 
 
 #include <fstream>
-#include <sstream>
 #include <iostream>
+#include <sstream>
 
 #include "config.h"
-#include "triton/core/tritonserver.h"
 #include "curl/curl.h"
+#include "triton/core/tritonserver.h"
 
 namespace triton::repoagent::dragonfly {
 
@@ -92,14 +92,15 @@ BaseName(const std::string& path)
 }
 
 TRITONSERVER_Error*
-DownloadFile (const std::string &url, const std::string &path, DragonflyConfig& config) {
+DownloadFile(
+    const std::string& url, const std::string& path, DragonflyConfig& config)
+{
   CURL* curl;
 
   curl = curl_easy_init();
   if (!curl) {
     return TRITONSERVER_ErrorNew(
-        TRITONSERVER_ERROR_INTERNAL,
-        "Failed to initialize CURL.");
+        TRITONSERVER_ERROR_INTERNAL, "Failed to initialize CURL.");
   }
 
   FILE* fp = fopen(path.c_str(), "wb");
@@ -111,15 +112,18 @@ DownloadFile (const std::string &url, const std::string &path, DragonflyConfig&
   }
 
 
-  struct curl_slist *headers = NULL;
+  struct curl_slist* headers = NULL;
 
   auto cleanup = [&]() {
-    if (fp) fclose(fp);
-    if (headers) curl_slist_free_all(headers);
+    if (fp)
+      fclose(fp);
+    if (headers)
+      curl_slist_free_all(headers);
     curl_easy_cleanup(curl);
   };
 
-  auto write_data = [](void* ptr, size_t size, size_t nmemb, void* stream) -> size_t {
+  auto write_data = [](void* ptr, size_t size, size_t nmemb,
+                       void* stream) -> size_t {
     size_t written = fwrite(ptr, size, nmemb, static_cast<FILE*>(stream));
     return written;
   };
@@ -128,29 +132,29 @@ DownloadFile (const std::string &url, const std::string &path, DragonflyConfig&
   curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
   curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
 
-  for(const auto& header : config.headers) {
+  for (const auto& header : config.headers) {
     std::string header_str = header.first + ": " + header.second;
     headers = curl_slist_append(headers, header_str.c_str());
     if (!headers) {
       cleanup();
       return TRITONSERVER_ErrorNew(
-          TRITONSERVER_ERROR_INTERNAL,
-          "Failed to append headers.");
+          TRITONSERVER_ERROR_INTERNAL, "Failed to append headers.");
     }
   }
 
   if (!config.filter.empty()) {
     std::ostringstream oss;
     for (size_t i = 0; i < config.filter.size(); ++i) {
-      if (i != 0) oss << "&";
+      if (i != 0)
+        oss << "&";
       oss << config.filter[i];
     }
-    headers = curl_slist_append(headers, ("X-Dragonfly-Filter: " + oss.str()).c_str());
+    headers = curl_slist_append(
+        headers, ("X-Dragonfly-Filter: " + oss.str()).c_str());
     if (!headers) {
       cleanup();
       return TRITONSERVER_ErrorNew(
-          TRITONSERVER_ERROR_INTERNAL,
-          "Failed to append filters.");
+          TRITONSERVER_ERROR_INTERNAL, "Failed to append filters.");
     }
   }
 
@@ -162,23 +166,22 @@ DownloadFile (const std::string &url, const std::string &path, DragonflyConfig&
 
   CURLcode res = curl_easy_perform(curl);
 
-  if(res != CURLE_OK) {
+  if (res != CURLE_OK) {
     cleanup();
     return TRITONSERVER_ErrorNew(
-        TRITONSERVER_ERROR_INTERNAL,
-        curl_easy_strerror(res));
+        TRITONSERVER_ERROR_INTERNAL, curl_easy_strerror(res));
   }
 
   cleanup();
   return nullptr;
 }
 
 TRITONSERVER_Error*
-ReadLocalFile(const std::string &path, std::string *contents) {
+ReadLocalFile(const std::string& path, std::string* contents)
+{
   if (!contents) {
     return TRITONSERVER_ErrorNew(
-        TRITONSERVER_ERROR_INTERNAL,
-        "Output string pointer is null.");
+        TRITONSERVER_ERROR_INTERNAL, "Output string pointer is null.");
   }
 
   std::ifstream in(path, std::ios::in | std::ios::binary);
@@ -201,4 +204,4 @@ ReadLocalFile(const std::string &path, std::string *contents) {
   return nullptr;
 }
 
-}
+}  // namespace triton::repoagent::dragonfly

src/config.h

@@ -1,22 +1,22 @@
 /*
-*     Copyright 2023 The Dragonfly Authors
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*      http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
+ *     Copyright 2023 The Dragonfly Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 #pragma once
 
-#include <string>
 #include <map>
+#include <string>
 #include <vector>
 
 #define TRITONJSON_STATUSTYPE TRITONSERVER_Error*
@@ -25,7 +25,7 @@
 #define TRITONJSON_STATUSSUCCESS nullptr
 #include "triton/common/triton_json.h"
 
-namespace triton::repoagent::dragonfly{
+namespace triton::repoagent::dragonfly {
 
 struct DragonflyConfig {
   std::string proxy;
@@ -63,4 +63,4 @@ DragonflyConfig::DragonflyConfig(triton::common::TritonJson::Value& config)
     }
   }
 }
-}
+}  // namespace triton::repoagent::dragonfly