Log Service Setup (#1721)

## Description of changes
https://linear.app/trychroma/issue/CHR-241/stand-up-log-service

- Stand up Log Service in Dev
  - stand up postgres DB
  - stand up migration: atlas - depend on postgres
  - stand up logservice - depend on migration
  - stand up coordinator - depend on migration
- database migration
  - change env name
  - change database name
- add definition for reccord log (we can test perf for this later, not
hard to change)
- log service: go
  - entry point: main with Cmd
  - grpc service: with proto change
- coordinator
  - connect to docker postgres
  - reorganize packages to accommodate with logservice
  - rename bin to coordinator instead of chroma
  - tests connect to local postgres instead of sqlite
    - fix a bug from segment delete

- system_test fix will be in a separate PR

.github/workflows/chroma-coordinator-test.yaml

@@ -16,8 +16,25 @@ jobs:
       matrix:
         platform: [ubuntu-latest]
     runs-on: ${{ matrix.platform }}
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_USER: chroma
+          POSTGRES_PASSWORD: chroma
+          POSTGRES_DB: chroma
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
     steps:
     - name: Checkout
       uses: actions/checkout@v3
     - name: Build and test
       run: cd go/coordinator && make test
+      env:
+        POSTGRES_HOST: localhost
+        POSTGRES_PORT: 5432

Tiltfile

@@ -1,3 +1,8 @@
+docker_build('migration',
+             context='.',
+             dockerfile='./go/coordinator/Dockerfile.migration'
+)
+
 docker_build('coordinator',
              context='.',
              dockerfile='./go/coordinator/Dockerfile'
@@ -22,9 +27,15 @@ k8s_resource(
 )
 k8s_yaml(['k8s/dev/pulsar.yaml'])
 k8s_resource('pulsar', resource_deps=['k8s_setup'], labels=["infrastructure"])
+k8s_yaml(['k8s/dev/postgres.yaml'])
+k8s_resource('postgres', resource_deps=['k8s_setup'], labels=["infrastructure"])
+k8s_yaml(['k8s/dev/migration.yaml'])
+k8s_resource('migration', resource_deps=['postgres'], labels=["chroma"])
 k8s_yaml(['k8s/dev/server.yaml'])
 k8s_resource('server', resource_deps=['k8s_setup'],labels=["chroma"], port_forwards=8000 )
 k8s_yaml(['k8s/dev/coordinator.yaml'])
-k8s_resource('coordinator', resource_deps=['pulsar', 'server'], labels=["chroma"])
+k8s_resource('coordinator', resource_deps=['pulsar', 'server', 'migration'], labels=["chroma"])
+k8s_yaml(['k8s/dev/logservice.yaml'])
+k8s_resource('logservice', resource_deps=['migration'], labels=["chroma"])
 k8s_yaml(['k8s/dev/worker.yaml'])
 k8s_resource('worker', resource_deps=['coordinator'],labels=["chroma"])

bin/cluster-test.sh

@@ -25,6 +25,7 @@ minikube addons enable ingress-dns -p chroma-test
 # Setup docker to build inside the minikube cluster and build the image
 eval $(minikube -p chroma-test docker-env)
 docker build -t server:latest -f Dockerfile .
+docker build -t migration -f go/coordinator/Dockerfile.migration .
 docker build -t chroma-coordinator:latest -f go/coordinator/Dockerfile .
 docker build -t worker -f rust/worker/Dockerfile . --build-arg CHROMA_KUBERNETES_INTEGRATION=1
 
@@ -35,6 +36,8 @@ kubectl apply -f k8s/cr
 kubectl apply -f k8s/test
 
 # Wait for the pods in the chroma namespace to be ready
+kubectl wait --for=condition=complete --timeout=100s job/migration -n chroma
+kubectl delete job migration -n chroma
 kubectl wait --namespace chroma --for=condition=Ready pods --all --timeout=400s
 
 # Run mini kube tunnel in the background to expose the service