Merge branch 'chroma-core:main' into bugfix/per-thread-pool-lru-cache

.gitattributes

@@ -1 +1,2 @@
 *_pb2.py* linguist-generated
+*_pb2_grpc.py* linguist-generated

.github/workflows/check-pr-title.yml

@@ -0,0 +1,28 @@
+name: Check PR Title
+
+on:
+  pull_request:
+    branches:
+      - main
+      - '**'
+
+jobs:
+  check-title:
+    runs-on: ubuntu-latest
+    steps:
+        - name: Check PR Title
+          uses: Slashgear/[email protected]
+          with:
+            regexp: '\[(ENH|BUG|DOC|TST|BLD|PERF|TYP|CLN|CHORE|RELEASE)\].*'
+            helpMessage: "Please tag your PR title. See https://docs.trychroma.com/contributing#contributing-code-and-ideas. You must push new code to this PR for this check to run again."
+        - name: Comment explaining failure
+          if: failure()
+          uses: actions/github-script@v6
+          with:
+            script: |
+              github.rest.issues.createComment({
+                issue_number: context.issue.number,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body: 'Please tag your PR title with one of: \\[ENH | BUG | DOC | TST | BLD | PERF | TYP | CLN | CHORE\\]. See https://docs.trychroma.com/contributing#contributing-code-and-ideas'
+              })

.github/workflows/chroma-client-integration-test.yml

@@ -15,7 +15,7 @@ jobs:
     timeout-minutes: 90
     strategy:
       matrix:
-        python: ['3.8', '3.9', '3.10', '3.11']
+        python: ['3.8', '3.9', '3.10', '3.11', '3.12']
         platform: [ubuntu-latest, windows-latest]
     runs-on: ${{ matrix.platform }}
     steps:

.github/workflows/chroma-cluster-test.yml

@@ -11,14 +11,15 @@ on:
   workflow_dispatch:
 
 jobs:
-  test:
+  test-python:
     strategy:
       matrix:
         python: ['3.8']
         platform: ['16core-64gb-ubuntu-latest']
-        testfile: ["chromadb/test/ingest/test_producer_consumer.py",
-                   "chromadb/test/db/test_system.py",
-                   "chromadb/test/segment/distributed/test_memberlist_provider.py",]
+        testfile: ["chromadb/test/db/test_system.py",
+                   "chromadb/test/ingest/test_producer_consumer.py",
+                   "chromadb/test/segment/distributed/test_memberlist_provider.py",
+                   "chromadb/test/test_logservice.py"]
     runs-on: ${{ matrix.platform }}
     steps:
     - name: Checkout
@@ -29,14 +30,40 @@ jobs:
         python-version: ${{ matrix.python }}
     - name: Install test dependencies
       run: python -m pip install -r requirements.txt && python -m pip install -r requirements_dev.txt
-    - name: Start minikube
-      id: minikube
-      uses: medyagh/setup-minikube@latest
-      with:
-        minikube-version: latest
-        kubernetes-version: latest
-        driver: docker
-        addons: ingress, ingress-dns
-        start-args: '--profile chroma-test'
-    - name: Integration Test
-      run: bin/cluster-test.sh ${{ matrix.testfile }}
+    - name: Install Tilt
+      run: |
+        TILT_VERSION="0.33.3"
+        curl -fsSL https://github.com/tilt-dev/tilt/releases/download/v$TILT_VERSION/tilt.$TILT_VERSION.linux.x86_64.tar.gz | \
+          tar -xzv -C /usr/local/bin tilt
+    - name: Install ctlptlc
+      run: |
+        CTLPTL_VERSION="0.8.20"
+        curl -fsSL https://github.com/tilt-dev/ctlptl/releases/download/v$CTLPTL_VERSION/ctlptl.$CTLPTL_VERSION.linux.x86_64.tar.gz | \
+          tar -xzv -C /usr/local/bin ctlptl
+    - name: Set up kind
+      run: ctlptl create cluster kind --registry=ctlptl-registry
+    - name: Start Tilt
+      run: tilt ci
+    - name: Test
+      run: bin/cluster-test.sh bash -c 'python -m pytest "${{ matrix.testfile }}"'
+  test-go:
+    runs-on: '16core-64gb-ubuntu-latest'
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Install Tilt
+      run: |
+        TILT_VERSION="0.33.3"
+        curl -fsSL https://github.com/tilt-dev/tilt/releases/download/v$TILT_VERSION/tilt.$TILT_VERSION.linux.x86_64.tar.gz | \
+          tar -xzv -C /usr/local/bin tilt
+    - name: Install ctlptlc
+      run: |
+        CTLPTL_VERSION="0.8.20"
+        curl -fsSL https://github.com/tilt-dev/ctlptl/releases/download/v$CTLPTL_VERSION/ctlptl.$CTLPTL_VERSION.linux.x86_64.tar.gz | \
+          tar -xzv -C /usr/local/bin ctlptl
+    - name: Set up kind
+      run: ctlptl create cluster kind --registry=ctlptl-registry
+    - name: Start Tilt
+      run: tilt ci
+    - name: Test
+      run: bin/cluster-test.sh bash -c 'cd go && go test -timeout 30s -run ^TestNodeWatcher$ github.com/chroma-core/chroma/go/pkg/memberlist_manager'

.github/workflows/chroma-coordinator-test.yaml

@@ -16,8 +16,25 @@ jobs:
       matrix:
         platform: [ubuntu-latest]
     runs-on: ${{ matrix.platform }}
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_USER: chroma
+          POSTGRES_PASSWORD: chroma
+          POSTGRES_DB: chroma
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
     steps:
     - name: Checkout
       uses: actions/checkout@v3
     - name: Build and test
-      run: cd go/coordinator && make test
+      run: cd go && make test
+      env:
+        POSTGRES_HOST: localhost
+        POSTGRES_PORT: 5432

.github/workflows/chroma-release-python-client.yml

@@ -33,7 +33,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: '3.10'
+        python-version: '3.12'
     - name: Install Client Dev Dependencies
       run: python -m pip install -r ./clients/python/requirements.txt && python -m pip install -r ./clients/python/requirements_dev.txt
     - name: Build Client

.github/workflows/chroma-release.yml

@@ -141,39 +141,3 @@ jobs:
         artifacts: "dist/chroma-${{steps.version.outputs.version}}.tar.gz"
         allowUpdates: true
         prerelease: true
-    - name: Trigger Hosted Chroma FE Release
-      uses: actions/github-script@v6
-      with:
-        github-token: ${{ secrets.HOSTED_CHROMA_WORKFLOW_DISPATCH_TOKEN }}
-        script: |
-          const result = await github.rest.actions.createWorkflowDispatch({
-            owner: 'chroma-core',
-            repo: 'hosted-chroma',
-            workflow_id: 'build-and-publish-frontend.yaml',
-            ref: 'main'
-          })
-          console.log(result)
-    - name: Trigger Hosted Chroma Coordinator Release
-      uses: actions/github-script@v6
-      with:
-        github-token: ${{ secrets.HOSTED_CHROMA_WORKFLOW_DISPATCH_TOKEN }}
-        script: |
-          const result = await github.rest.actions.createWorkflowDispatch({
-            owner: 'chroma-core',
-            repo: 'hosted-chroma',
-            workflow_id: 'build-and-deploy-coordinator.yaml',
-            ref: 'main'
-          })
-          console.log(result)
-    - name: Trigger Hosted Worker Release
-      uses: actions/github-script@v6
-      with:
-        github-token: ${{ secrets.HOSTED_CHROMA_WORKFLOW_DISPATCH_TOKEN }}
-        script: |
-          const result = await github.rest.actions.createWorkflowDispatch({
-            owner: 'chroma-core',
-            repo: 'hosted-chroma',
-            workflow_id: 'build-and-deploy-worker.yaml',
-            ref: 'main'
-          })
-          console.log(result)

.github/workflows/chroma-test.yml

@@ -16,7 +16,7 @@ jobs:
     timeout-minutes: 90
     strategy:
       matrix:
-        python: ['3.8', '3.9', '3.10', '3.11']
+        python: ['3.8', '3.9', '3.10', '3.11', '3.12']
         platform: [ubuntu-latest, windows-latest]
         testfile: ["--ignore-glob 'chromadb/test/property/*' --ignore-glob 'chromadb/test/stress/*' --ignore='chromadb/test/auth/test_simple_rbac_authz.py'",
                    "chromadb/test/auth/test_simple_rbac_authz.py",

.github/workflows/release-helm-chart.yml

@@ -0,0 +1,30 @@
+# TODO Once distributed chroma is operational, update this to update the
+# OSS helm chart we'll host. For now, just kick off the job which updates
+# hosted-chroma's version.
+
+name: Release Helm Chart
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'k8s/**'
+  workflow_dispatch:
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Trigger Hosted Chroma Coordinator Release
+      uses: actions/github-script@v6
+      with:
+        github-token: ${{ secrets.HOSTED_CHROMA_WORKFLOW_DISPATCH_TOKEN }}
+        script: |
+          const result = await github.rest.actions.createWorkflowDispatch({
+            owner: 'chroma-core',
+            repo: 'hosted-chroma',
+            workflow_id: 'copy-oss-helm.yaml',
+            ref: 'main'
+          })
+          console.log(result)

.gitignore

@@ -3,8 +3,9 @@
 
 **/__pycache__
 
+go/bin/
+go/**/testdata/
 go/coordinator/bin/
-go/coordinator/**/testdata/
 
 *.log
 
@@ -32,3 +33,8 @@ dist
 terraform.tfstate
 .hypothesis/
 .idea
+
+target/
+
+# environment file generated by the Javascript tests
+.chroma_env

.pre-commit-config.yaml

@@ -34,3 +34,10 @@ repos:
       - id: mypy
         args: [--strict, --ignore-missing-imports, --follow-imports=silent, --disable-error-code=type-abstract, --config-file=./pyproject.toml]
         additional_dependencies: ["types-requests", "pydantic", "overrides", "hypothesis", "pytest", "pypika", "numpy", "types-protobuf", "kubernetes"]
+
+
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: "v3.1.0"
+    hooks:
+      - id: prettier
+        files: "^clients/js/.+"

.vscode/settings.json

@@ -128,4 +128,4 @@
     "unordered_set": "cpp",
     "algorithm": "cpp"
   },
-}
+}