Merge branch 'main' into feature/client-upgrade-notification

.gitattributes

@@ -1 +1,4 @@
 *_pb2.py* linguist-generated
+*_pb2_grpc.py* linguist-generated
+go/database/**/db/** linguist-generated=true
+go/pkg/proto/** linguist-generated=true

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -1,5 +1,5 @@
-name: Bug Report
-description: File a bug report with Chroma
+name: 🐛 Bug Report
+description: File a bug report to help us improve Chroma
 title: "[Bug]: "
 labels: ["bug", "triage"]
 # assignees:

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: true
+contact_links:
+  - name: 🤷🏻‍♀️ Questions
+    url: https://discord.com/invite/MMeYNTmh3x
+    about: Interact with the Chroma community here by asking for help, discussing and more!

.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -1,4 +1,4 @@
-name: "Feature Request"
+name: 🚀 Feature request
 description: Suggest an idea for Chroma
 title: "[Feature Request]: "
 labels: ["enhancement"]

.github/workflows/check-pr-title.yml

@@ -0,0 +1,28 @@
+name: Check PR Title
+
+on:
+  pull_request:
+    branches:
+      - main
+      - '**'
+
+jobs:
+  check-title:
+    runs-on: ubuntu-latest
+    steps:
+        - name: Check PR Title
+          uses: Slashgear/[email protected]
+          with:
+            regexp: '\[(ENH|BUG|DOC|TST|BLD|PERF|TYP|CLN|CHORE|RELEASE)\].*'
+            helpMessage: "Please tag your PR title. See https://docs.trychroma.com/contributing#contributing-code-and-ideas. You must push new code to this PR for this check to run again."
+        - name: Comment explaining failure
+          if: failure()
+          uses: actions/github-script@v6
+          with:
+            script: |
+              github.rest.issues.createComment({
+                issue_number: context.issue.number,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body: 'Please tag your PR title with one of: \\[ENH | BUG | DOC | TST | BLD | PERF | TYP | CLN | CHORE\\]. See https://docs.trychroma.com/contributing#contributing-code-and-ideas'
+              })

.github/workflows/chroma-client-integration-test.yml

@@ -15,7 +15,7 @@ jobs:
     timeout-minutes: 90
     strategy:
       matrix:
-        python: ['3.8', '3.9', '3.10', '3.11']
+        python: ['3.8', '3.9', '3.10', '3.11', '3.12']
         platform: [ubuntu-latest, windows-latest]
     runs-on: ${{ matrix.platform }}
     steps:

.github/workflows/chroma-cluster-test.yml

@@ -11,13 +11,15 @@ on:
   workflow_dispatch:
 
 jobs:
-  test:
+  test-python:
     strategy:
       matrix:
         python: ['3.8']
-        platform: [ubuntu-latest]
-        testfile: ["chromadb/test/ingest/test_producer_consumer.py",
-                   "chromadb/test/segment/distributed/test_memberlist_provider.py",]
+        platform: ['16core-64gb-ubuntu-latest']
+        testfile: ["chromadb/test/db/test_system.py",
+                   "chromadb/test/ingest/test_producer_consumer.py",
+                   "chromadb/test/segment/distributed/test_memberlist_provider.py",
+                   "chromadb/test/test_logservice.py"]
     runs-on: ${{ matrix.platform }}
     steps:
     - name: Checkout
@@ -28,14 +30,40 @@ jobs:
         python-version: ${{ matrix.python }}
     - name: Install test dependencies
       run: python -m pip install -r requirements.txt && python -m pip install -r requirements_dev.txt
-    - name: Start minikube
-      id: minikube
-      uses: medyagh/setup-minikube@latest
-      with:
-        minikube-version: latest
-        kubernetes-version: latest
-        driver: docker
-        addons: ingress, ingress-dns
-        start-args: '--profile chroma-test'
-    - name: Integration Test
-      run: bin/cluster-test.sh ${{ matrix.testfile }}
+    - name: Install Tilt
+      run: |
+        TILT_VERSION="0.33.3"
+        curl -fsSL https://github.com/tilt-dev/tilt/releases/download/v$TILT_VERSION/tilt.$TILT_VERSION.linux.x86_64.tar.gz | \
+          tar -xzv -C /usr/local/bin tilt
+    - name: Install ctlptlc
+      run: |
+        CTLPTL_VERSION="0.8.20"
+        curl -fsSL https://github.com/tilt-dev/ctlptl/releases/download/v$CTLPTL_VERSION/ctlptl.$CTLPTL_VERSION.linux.x86_64.tar.gz | \
+          tar -xzv -C /usr/local/bin ctlptl
+    - name: Set up kind
+      run: ctlptl create cluster kind --registry=ctlptl-registry
+    - name: Start Tilt
+      run: tilt ci
+    - name: Test
+      run: bin/cluster-test.sh bash -c 'python -m pytest "${{ matrix.testfile }}"'
+  test-go:
+    runs-on: '16core-64gb-ubuntu-latest'
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Install Tilt
+      run: |
+        TILT_VERSION="0.33.3"
+        curl -fsSL https://github.com/tilt-dev/tilt/releases/download/v$TILT_VERSION/tilt.$TILT_VERSION.linux.x86_64.tar.gz | \
+          tar -xzv -C /usr/local/bin tilt
+    - name: Install ctlptlc
+      run: |
+        CTLPTL_VERSION="0.8.20"
+        curl -fsSL https://github.com/tilt-dev/ctlptl/releases/download/v$CTLPTL_VERSION/ctlptl.$CTLPTL_VERSION.linux.x86_64.tar.gz | \
+          tar -xzv -C /usr/local/bin ctlptl
+    - name: Set up kind
+      run: ctlptl create cluster kind --registry=ctlptl-registry
+    - name: Start Tilt
+      run: tilt ci
+    - name: Test
+      run: bin/cluster-test.sh bash -c 'cd go && go test -timeout 30s -run ^TestNodeWatcher$ github.com/chroma-core/chroma/go/pkg/memberlist_manager'

.github/workflows/chroma-coordinator-test.yaml

@@ -0,0 +1,41 @@
+name: Chroma Coordinator Tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+      - '**'
+  workflow_dispatch:
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        platform: [ubuntu-latest]
+    runs-on: ${{ matrix.platform }}
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_USER: chroma
+          POSTGRES_PASSWORD: chroma
+          POSTGRES_DB: chroma
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - uses: ariga/setup-atlas@v0
+    - name: Build and test
+      run: cd go && make test
+      env:
+        POSTGRES_HOST: localhost
+        POSTGRES_PORT: 5432

.github/workflows/chroma-release-python-client.yml

@@ -33,7 +33,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: '3.10'
+        python-version: '3.12'
     - name: Install Client Dev Dependencies
       run: python -m pip install -r ./clients/python/requirements.txt && python -m pip install -r ./clients/python/requirements_dev.txt
     - name: Build Client

.github/workflows/chroma-release.yml

@@ -141,15 +141,3 @@ jobs:
         artifacts: "dist/chroma-${{steps.version.outputs.version}}.tar.gz"
         allowUpdates: true
         prerelease: true
-    - name: Trigger Hosted Chroma Release
-      uses: actions/github-script@v6
-      with:
-        github-token: ${{ secrets.HOSTED_CHROMA_WORKFLOW_DISPATCH_TOKEN }}
-        script: |
-          const result = await github.rest.actions.createWorkflowDispatch({
-            owner: 'chroma-core',
-            repo: 'hosted-chroma',
-            workflow_id: 'build-and-publish-image.yaml',
-            ref: 'main'
-          })
-          console.log(result)

.github/workflows/chroma-test.yml

@@ -16,7 +16,7 @@ jobs:
     timeout-minutes: 90
     strategy:
       matrix:
-        python: ['3.8', '3.9', '3.10', '3.11']
+        python: ['3.8', '3.9', '3.10', '3.11', '3.12']
         platform: [ubuntu-latest, windows-latest]
         testfile: ["--ignore-glob 'chromadb/test/property/*' --ignore-glob 'chromadb/test/stress/*' --ignore='chromadb/test/auth/test_simple_rbac_authz.py'",
                    "chromadb/test/auth/test_simple_rbac_authz.py",

.github/workflows/chroma-worker-test.yml

@@ -17,9 +17,20 @@ jobs:
         platform: [ubuntu-latest]
     runs-on: ${{ matrix.platform }}
     steps:
+    - name: Checkout chroma-hnswlib
+      uses: actions/checkout@v3
+      with:
+        repository: chroma-core/hnswlib
+        path: hnswlib
     - name: Checkout
       uses: actions/checkout@v3
+      with:
+        path: chroma
+    - name: Install Protoc
+      uses: arduino/setup-protoc@v2
     - name: Build
       run: cargo build --verbose
+      working-directory: chroma
     - name: Test
       run: cargo test --verbose
+      working-directory: chroma

.github/workflows/pr-review-checklist.yml

@@ -33,5 +33,5 @@ jobs:
             - [ ] Are there any potential impacts on other parts of the system or backward compatibility?
             - [ ] Does this change intersect with any items on our roadmap, and if so, is there a plan for fitting them together?
             ## Quality
-            - [ ] Is this code of a unexpectedly high quality (Readbility, Modularity, Intuitiveness)`
+            - [ ] Is this code of a unexpectedly high quality (Readability, Modularity, Intuitiveness)`
           })

.github/workflows/release-helm-chart.yml

@@ -0,0 +1,30 @@
+# TODO Once distributed chroma is operational, update this to update the
+# OSS helm chart we'll host. For now, just kick off the job which updates
+# hosted-chroma's version.
+
+name: Release Helm Chart
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'k8s/**'
+  workflow_dispatch:
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Trigger Hosted Chroma Coordinator Release
+      uses: actions/github-script@v6
+      with:
+        github-token: ${{ secrets.HOSTED_CHROMA_WORKFLOW_DISPATCH_TOKEN }}
+        script: |
+          const result = await github.rest.actions.createWorkflowDispatch({
+            owner: 'chroma-core',
+            repo: 'hosted-chroma',
+            workflow_id: 'copy-oss-helm.yaml',
+            ref: 'main'
+          })
+          console.log(result)

.gitignore

@@ -2,6 +2,9 @@
 **/.DS_Store
 
 **/__pycache__
+go/bin/
+go/**/testdata/
+go/coordinator/bin/
 
 *.log
 
@@ -29,3 +32,11 @@ dist
 terraform.tfstate
 .hypothesis/
 .idea
+
+target/
+
+# environment file generated by the Javascript tests
+.chroma_env
+
+# Rapid test data
+testdata

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 exclude: 'chromadb/proto/(chroma_pb2|coordinator_pb2)\.(py|pyi|py_grpc\.py)' # Generated files
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
       - id: trailing-whitespace
       - id: mixed-line-ending
@@ -21,7 +21,7 @@ repos:
       - id: black
 
   - repo: https://github.com/PyCQA/flake8
-    rev: 6.0.0
+    rev: 6.1.0
     hooks:
       - id: flake8
         args:
@@ -32,5 +32,12 @@ repos:
     rev: "v1.2.0"
     hooks:
       - id: mypy
-        args: [--strict, --ignore-missing-imports, --follow-imports=silent, --disable-error-code=type-abstract, --config-file=./mypy.ini]
+        args: [--strict, --ignore-missing-imports, --follow-imports=silent, --disable-error-code=type-abstract, --config-file=./pyproject.toml]
         additional_dependencies: ["types-requests", "pydantic", "overrides", "hypothesis", "pytest", "pypika", "numpy", "types-protobuf", "kubernetes"]
+
+
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: "v3.1.0"
+    hooks:
+      - id: prettier
+        files: "^clients/js/.+"

.vscode/settings.json

@@ -128,4 +128,4 @@
     "unordered_set": "cpp",
     "algorithm": "cpp"
   },
-}
+}