Firestore CI (#877)

* Add firestore CI runner * Correct hook file name * Setup emulator credentials * Swap dependency to firestore alone * Hacky setup for firestore * Fix firestore hostname * Ensure firestore connection * Fix CI issues
newrelic · Jul 21, 2023 · dde35d2 · dde35d2
1 parent b8333fa
commit dde35d2
Show file tree

Hide file tree

Showing 4 changed files with 89 additions and 20 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -924,3 +924,68 @@ jobs:
           name: coverage-${{ github.job }}-${{ strategy.job-index }}
           path: ./**/.coverage.*
           retention-days: 1
+
+  firestore:
+    env:
+      TOTAL_GROUPS: 1
+
+    strategy:
+      fail-fast: false
+      matrix:
+        group-number: [1]
+
+    runs-on: ubuntu-20.04
+    container:
+      image: ghcr.io/newrelic/newrelic-python-agent-ci:latest
+      options: >-
+        --add-host=host.docker.internal:host-gateway
+    timeout-minutes: 30
+
+    services:
+      firestore:
+        # Image set here MUST be repeated down below in options. See comment below.
+        image: gcr.io/google.com/cloudsdktool/google-cloud-cli:437.0.1-emulators
+        ports:
+          - 8080:8080
+        # Set health checks to wait 5 seconds in lieu of an actual healthcheck
+        options: >-
+          --health-cmd "echo success"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+          --health-start-period 5s
+          gcr.io/google.com/cloudsdktool/google-cloud-cli:437.0.1-emulators /bin/bash -c "gcloud emulators firestore start --host-port=0.0.0.0:8080" ||
+        # This is a very hacky solution. GitHub Actions doesn't provide APIs for setting commands on services, but allows adding arbitrary options.
+        # --entrypoint won't work as it only accepts an executable and not the [] syntax.
+        # Instead, we specify the image again the command afterwards like a call to docker create. The result is a few environment variables
+        # and the original command being appended to our hijacked docker create command. We can avoid any issues by adding || to prevent that
+        # from every being executed as bash commands.
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Fetch git tags
+        run: |
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
+          git fetch --tags origin
+
+      - name: Get Environments
+        id: get-envs
+        run: |
+          echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT
+        env:
+          GROUP_NUMBER: ${{ matrix.group-number }}
+
+      - name: Test
+        run: |
+          tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto
+        env:
+          TOX_PARALLEL_NO_SPINNER: 1
+          PY_COLORS: 0
+
+      - name: Upload Coverage Artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage-${{ github.job }}-${{ strategy.job-index }}
+          path: ./**/.coverage.*
+          retention-days: 1
diff --git a/newrelic/config.py b/newrelic/config.py
@@ -2271,31 +2271,31 @@ def _process_module_builtin_defaults():
 
     _process_module_definition(
         "google.cloud.firestore_v1.base_document",
-        "newrelic.hooks.database_firestore",
+        "newrelic.hooks.datastore_firestore",
         "instrument_google_cloud_firestore_v1_base_document",
     )
 
     _process_module_definition(
         "google.cloud.firestore_v1.base_collection",
-        "newrelic.hooks.database_firestore",
+        "newrelic.hooks.datastore_firestore",
         "instrument_google_cloud_firestore_v1_base_collection",
     )
 
     _process_module_definition(
         "google.cloud.firestore_v1.document",
-        "newrelic.hooks.database_firestore",
+        "newrelic.hooks.datastore_firestore",
         "instrument_google_cloud_firestore_v1_document",
     )
 
     _process_module_definition(
         "google.cloud.firestore_v1.collection",
-        "newrelic.hooks.database_firestore",
+        "newrelic.hooks.datastore_firestore",
         "instrument_google_cloud_firestore_v1_collection",
     )
 
     _process_module_definition(
         "google.cloud.firestore_v1.base_client",
-        "newrelic.hooks.database_firestore",
+        "newrelic.hooks.datastore_firestore",
         "instrument_google_cloud_firestore_v1_base_client",
     )
     _process_module_definition(

diff --git a/tests/datastore_firestore/conftest.py b/tests/datastore_firestore/conftest.py
@@ -11,18 +11,21 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import os
 import uuid
 
 import pytest
 
-import firebase_admin
-from firebase_admin import credentials
-from firebase_admin import firestore
+from google.cloud.firestore import Client
 
+from testing_support.db_settings import firestore_settings
 from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture  # noqa: F401; pylint: disable=W0611
 
 
+DB_SETTINGS = firestore_settings()[0]
+FIRESTORE_HOST = DB_SETTINGS["host"]
+FIRESTORE_PORT = DB_SETTINGS["port"]
+
 _default_settings = {
     'transaction_tracer.explain_threshold': 0.0,
     'transaction_tracer.transaction_threshold': 0.0,
@@ -40,18 +43,19 @@
 
 @pytest.fixture(scope="session")
 def client():
-    creds = credentials.ApplicationDefault()
-
-    firebase_admin.initialize_app(creds)
-    client = firestore.client()
+    os.environ["FIRESTORE_EMULATOR_HOST"] = "%s:%d" % (FIRESTORE_HOST, FIRESTORE_PORT)
+    client = Client()
+    client.collection("healthcheck").document("healthcheck").set({}, retry=None, timeout=5)  # Ensure connection is available
     return client
 
 
 @pytest.fixture(scope="function")
 def collection(client):
-    collection = client.collection("firestore_collection_" + str(uuid.uuid4()))
-
-    yield collection
+    yield client.collection("firestore_collection_" + str(uuid.uuid4()))
+
 
-    for doc in collection.list_documents():
-        doc.delete()
+@pytest.fixture(scope="function", autouse=True)
+def reset_firestore(client):
+    for coll in client.collections():
+        for document in coll.list_documents():
+            document.delete()
diff --git a/tox.ini b/tox.ini
@@ -83,7 +83,7 @@ envlist =
     memcached-datastore_memcache-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}-memcached01,
     mysql-datastore_mysql-mysql080023-py27,
     mysql-datastore_mysql-mysqllatest-{py37,py38,py39,py310,py311},
-    firestore-datastore_firestore-{py37,py38,py39,py310,py311,pypy38},
+    firestore-datastore_firestore-{py37,py38,py39,py310,py311},
     postgres-datastore_postgresql-{py37,py38,py39},
     postgres-datastore_psycopg2-{py27,py37,py38,py39,py310,py311}-psycopg2latest
     postgres-datastore_psycopg2cffi-{py27,pypy27,py37,py38,py39,py310,py311}-psycopg2cffilatest,
@@ -239,7 +239,7 @@ deps =
     datastore_elasticsearch: requests
     datastore_elasticsearch-elasticsearch07: elasticsearch<8.0
     datastore_elasticsearch-elasticsearch08: elasticsearch<9.0
-    datastore_firestore: firebase-admin
+    datastore_firestore: google-cloud-firestore
     datastore_memcache-memcached01: python-memcached<2
     datastore_mysql-mysqllatest: mysql-connector-python
     datastore_mysql-mysql080023: mysql-connector-python<8.0.24