[Indexer] Integrate pocketdex into Tiltfile (#885)

## Summary

- Adds the pocketdex to the tilt environment **if it has been cloned to
a sibling directory as "pocketdex"**
- Otherwise, if `localnet_config.yaml` has `indexer.enabled` and
`indexer.clone_if_not_present` set to `true` Tilt will clone it to a
sibling directory
- Otherwise a stub service titled "Indexer Disabled" under the
"Pocketdex" label group explains what changes need to be made to the
localnet_config.yaml:


![image](https://github.com/user-attachments/assets/c51f6abc-8430-4788-8442-b309cdbd01a9)

```
Running cmd: echo 'Pocketdex indexer disabled. Set `indexer.enabled` to `true` in `localnet_config.yaml` to enable it.'
Pocketdex indexer disabled. Set `indexer.enabled` to `true` in `localnet_config.yaml` to enable it.
```

```
Running cmd: echo 'Pocketdex repo not found. Set `clone_if_not_present` to `true` in `localnet_config.yaml` to clone the repo.'
Pocketdex repo not found. Set `clone_if_not_present` to `true` in `localnet_config.yaml` to clone the repo.
```

## Issue

- #873 

## Type of change

Select one or more from the following:

- [x] New feature, functionality or library
- [ ] Consensus breaking; add the `consensus-breaking` label if so. See
#791 for details
- [ ] Bug fix
- [ ] Code health or cleanup
- [ ] Documentation
- [ ] Other (specify)

## Testing

- [ ] **Documentation**: `make docusaurus_start`; only needed if you
make doc changes
- [ ] **Unit Tests**: `make go_develop_and_test`
- [ ] **LocalNet E2E Tests**: `make test_e2e`
- [ ] **DevNet E2E Tests**: Add the `devnet-test-e2e` label to the PR.

## Sanity Checklist

- [ ] I have tested my changes using the available tooling
- [ ] I have commented my code
- [ ] I have performed a self-review of my own code; both comments &
source code
- [ ] I create and reference any new tickets, if applicable
- [ ] I have left TODOs throughout the codebase, if applicable

---------

Co-authored-by: Daniel Olshansky <[email protected]>

Tiltfile

@@ -55,9 +55,20 @@ localnet_config_defaults = {
     "rest": {
         "enabled": True,
     },
+
+    # NOTE: git submodule usage was explicitly avoided to reduce environment complexity.
+
     # By default, we use the `helm_repo` function below to point to the remote repository
     # but can update it to the locally cloned repo for testing & development
-    "helm_chart_local_repo": {"enabled": False, "path": "../helm-charts"},
+    "helm_chart_local_repo": {
+        "enabled": False,
+        "path": os.path.join("..", "helm-charts")
+    },
+    "indexer": {
+        "repo_path": os.path.join("..", "pocketdex"),
+        "enabled": True,
+        "clone_if_not_present": False,
+    }
 }
 localnet_config_file = read_yaml(localnet_config_path, default=localnet_config_defaults)
 # Initial empty config
@@ -75,14 +86,15 @@ if (localnet_config_file != localnet_config) or (not os.path.exists(localnet_con
 
 # Configure helm chart reference. If using a local repo, set the path to the local repo; otherwise, use our own helm repo.
 helm_repo("pokt-network", "https://pokt-network.github.io/helm-charts/")
+# TODO_IMPROVE: Use os.path.join to make this more OS-agnostic.
 chart_prefix = "pokt-network/"
 if localnet_config["helm_chart_local_repo"]["enabled"]:
     helm_chart_local_repo = localnet_config["helm_chart_local_repo"]["path"]
     hot_reload_dirs.append(helm_chart_local_repo)
     print("Using local helm chart repo " + helm_chart_local_repo)
+    # TODO_IMPROVE: Use os.path.join to make this more OS-agnostic.
     chart_prefix = helm_chart_local_repo + "/charts/"
 
-
 # Observability
 print("Observability enabled: " + str(localnet_config["observability"]["enabled"]))
 if localnet_config["observability"]["enabled"]:
@@ -383,3 +395,14 @@ if localnet_config["rest"]["enabled"]:
     print("REST enabled: " + str(localnet_config["rest"]["enabled"]))
     deployment_create("rest", image="davarski/go-rest-api-demo")
     k8s_resource("rest", labels=["data_nodes"], port_forwards=["10000"])
+
+### Pocketdex Shannon Indexer
+load("./tiltfiles/pocketdex.tilt", "check_and_load_pocketdex")
+
+# Check if sibling pocketdex repo exists.
+# If it does, load the pocketdex.tilt file from the sibling repo.
+# Otherwise, check the `indexer.clone_if_not_present` flag in `localnet_config.yaml` and EITHER:
+#   1. clone pocketdex to ../pocketdex
+#   -- OR --
+#   2. Prints a message if true or false
+check_and_load_pocketdex(localnet_config["indexer"])

docusaurus/docs/develop/contributing/_category_.json

@@ -1,6 +1,6 @@
 {
   "label": "Contributing",
-  "position": 9,
+  "position": 10,
   "link": {
     "type": "generated-index",
     "description": "Guidelines on how to contribute to the poktroll repo."

docusaurus/docs/develop/localnet/_category_.json

@@ -0,0 +1,8 @@
+{
+  "label": "LocalNet",
+  "position": 9,
+  "link": {
+    "type": "generated-index",
+    "description": "Tips on how to leverage the most out of your LocalNet Environment."
+  }
+}

docusaurus/docs/develop/localnet/observability.md

@@ -0,0 +1,72 @@
+---
+sidebar_position: 1
+title: Pocketdex Indexer
+---
+
+- [Usage](#usage)
+  - [GraphQL](#graphql)
+  - [Postgres - CLI](#postgres---cli)
+- [Debugging](#debugging)
+  - [Port already in use](#port-already-in-use)
+
+:::warning
+
+This document is a living WIP and assumes you are familiar with the LocalNet environment.
+
+:::
+
+## Pocketdex <!-- omit in toc -->
+
+[Pocketdex](https://github.com/pokt-network/pocketdex/), the poktroll indexer starts up as part of the default LocalNet.
+
+### Usage
+
+#### GraphQL
+
+The localnet graphiql playground is available at [http://localhost:3000](http://localhost:3000), by default.
+
+![GraphiQL Playground](../../../static/img/pocketdex_graphiql_screenshot.png)
+
+A link is accessible from the ["GraphQL API" tab in tilt](http://localhost:10350/r/GraphQL%20API/overview):
+
+![LocalNet Dashboard](../../../static/img/pocketdex_graphiql_link.png)
+
+See the [pocketdex docs](https://github.com/pokt-network/pocketdex?tab=readme-ov-file#usage--query-docs) for more details.
+
+#### Postgres - CLI
+
+You can connect using a tool of your choice or with the `psql` CLI via:
+
+```bash
+psql -h localhost -p 5432 -U postgres -d postgres
+```
+
+After you've connected, you MUST update your schema to `localnet` and start exploring the data:
+
+```sql
+set schema 'localnet';
+\dt
+select * from accounts limit 10; # Example query
+```
+
+### Debugging
+
+#### Port already in use
+
+If you go to [http://localhost:10350/r/Postgres/overview](http://localhost:10350/r/Postgres/overview) and see the following error:
+
+```bash
+Reconnecting... Error port-forwarding Postgres (5432 -> 5432): Unable to listen on port 5432: Listeners failed to create with the following errors: [unable to create listener: Error listen tcp4 127.0.0.1:5432: bind: address already in use unable to create listener: Error listen tcp6 [::1]:5432: bind: address already in use]
+```
+
+You likely have another local Postgres instance running. You can identify it by running
+
+```bash
+lsof -i:5432
+```
+
+On macOS, if installed via `brew`, it can be stopped with:
+
+```bash
+brew services stop postgresql
+```

localnet/kubernetes/values-pocketdex-gql-engine.yaml

@@ -0,0 +1,24 @@
+enable_graphiql_playground: true
+indexer_endpoint: "http://indexer-service:3000"
+port: 3000
+env:
+  NODE_ENV: development
+  # LocalNet
+  ENDPOINT: "http://validator-poktroll-validator:26657"
+  CHAIN_ID: "poktroll"
+  # db schema name
+  DB_SCHEMA: "localnet"
+
+  START_BLOCK: 1
+  # Check docs to see what else you can set here:
+  # Ref: https://academy.subquery.network/indexer/run_publish/references.html#subql-cli
+  SUBQUERY_NODE_EXTRA_PARAMS: "--unfinalized-blocks=true"
+  # Check docs to see what else you can set here:
+  # Ref: https://academy.subquery.network/indexer/run_publish/references.html#subql-query
+  SUBQUERY_GRAPHQL_EXTRA_PARAMS: ""
+
+  DB_HOST: postgres-service
+  DB_PORT: 5432
+  DB_USER: postgres
+  DB_PASS: postgres
+  DB_DATABASE: postgres

localnet/kubernetes/values-pocketdex-indexer.yaml

@@ -0,0 +1,24 @@
+port: 3000
+env:
+  NODE_ENV: development
+  # LocalNet
+  ENDPOINT: "http://validator-poktroll-validator:26657"
+  CHAIN_ID: "poktroll"
+  # db schema name
+  DB_SCHEMA: "localnet"
+
+  START_BLOCK: 1
+  # Check docs to see what else you can set here:
+  # Ref: https://academy.subquery.network/indexer/run_publish/references.html#subql-cli
+  SUBQUERY_NODE_EXTRA_PARAMS: "--unfinalized-blocks=true"
+  # Check docs to see what else you can set here:
+  # Ref: https://academy.subquery.network/indexer/run_publish/references.html#subql-query
+  SUBQUERY_GRAPHQL_EXTRA_PARAMS: ""
+
+  SUBQUERY_GRAPHQL_ENGINE_VERSION: "latest"
+
+  DB_HOST: postgres-service
+  DB_PORT: 5432
+  DB_USER: postgres
+  DB_PASS: postgres
+  DB_DATABASE: postgres

localnet/kubernetes/values-pocketdex-pgadmin.yaml

@@ -0,0 +1,14 @@
+env:
+  # Pgadmin4
+  # DEV_NOTE: if you modify POSTGRES_USER/PASSWORD/DB then auto-imported server will not match and you need to setup
+  # your own with the modified values on the pgadmin website after first login.
+  PGADMIN_VERSION: 8.11
+  PGADMIN_DEFAULT_EMAIL: [email protected]
+  PGADMIN_DEFAULT_PASSWORD: admin
+  PGADMIN_LISTEN_PORT: 5050
+
+  DB_HOST: postgres-service
+  DB_PORT: 5432
+  DB_USER: postgres
+  DB_PASS: postgres
+  DB_DATABASE: postgres

localnet/kubernetes/values-pocketdex-postgres.yaml

@@ -0,0 +1,4 @@
+env:
+  POSTGRES_USER: postgres
+  POSTGRES_PASSWORD: postgres
+  POSTGRES_DB: postgres

tiltfiles/git.tilt

@@ -0,0 +1,46 @@
+# TODO_INVESTIGATE: https://github.com/tilt-dev/tilt-extensions/tree/master/git_resource
+
+# repo_remote_name returns the name of the remote from which the git repo was cloned.
+def repo_remote_name(repo_root):
+    return str(
+        local("git rev-parse --abbrev-ref @{u} | cut -d '/' -f1",
+              dir=repo_root,
+              echo_off=True),
+    ).strip()
+
+
+# fetch_repo_main fetches the main branch from the remote from which the git repo was cloned.
+def fetch_repo_main(repo_root):
+    local("git fetch {} main".format(repo_remote_name(repo_root)),
+          dir=repo_root,
+          echo_off=True)
+
+
+# repo_changes returns a tuple of integers representing the number of local
+#  and remote changes for the git repos main branch, respectively.
+def repo_changes(repo_root):
+    all_changes = str(
+        local("""
+              git rev-list --left-right --count HEAD...{}/main
+              """.format(repo_remote_name(repo_root)),
+              dir=repo_root,
+              echo_off=True)
+    )
+
+    # Split the output into local and remote changes and convert to integers.
+    num_local_changes, num_remote_changes = [int(x) for x in all_changes.split()]
+
+    return (num_local_changes, num_remote_changes)
+
+
+# repo_is_outdated returns true if there's a diff between the local and remote main branches.
+def repo_is_outdated(repo_root):
+    fetch_repo_main(repo_root)
+    _, num_remote_changes = repo_changes(repo_root)
+    return num_remote_changes != 0
+
+
+# clone_repo clones the main branch from the repo at repo_url to local_path.
+def clone_repo(repo_url, local_path):
+    print("Cloning pocketdex repo")
+    local("git clone {} --branch main {}".format(repo_url, local_path))

tiltfiles/pocketdex.tilt

@@ -0,0 +1,68 @@
+load("./git.tilt",
+     "clone_repo",
+     "fetch_repo_main",
+     "repo_remote_name",
+     "repo_changes",
+     "repo_is_outdated")
+
+
+# pocketdex_disabled_resource creates a tilt resource that prints a message indicating
+# that the indexer is disabled and how to enable it.
+def pocketdex_disabled_resource(reason):
+    local_resource("⚠️ Indexer Disabled",
+                   "echo '{}'".format(reason),
+                   labels=["Pocketdex"])
+
+
+# pocketdex_outdated_resource creates a tilt resource that prints a message indicating
+# that the indexer is outdated and how many commits behind it is.
+def pocketdex_outdated_resource(pocketdex_root_path):
+    _, num_remote_changes = repo_changes(pocketdex_root_path)
+    local_resource("🔄 Updates Available",
+                   """
+                   echo 'Pocketdex main branch is outdated; {} commits behind. Please `git pull --ff-only` to update pocketdex.'
+                   """.format(num_remote_changes),
+                   labels=["Pocketdex"])
+
+
+# load_pocketdex loads the pocketdex.tilt file from the pocketdex repo at pocketdex_root_path.
+# It also checks if the pocketdex repo has updates and, if so, creates a resource which prints instructions to update.
+def load_pocketdex(pocketdex_root_path):
+    if repo_is_outdated(pocketdex_root_path):
+        pocketdex_outdated_resource(pocketdex_root_path)
+
+    pocketdex_tilt_path = os.path.join(pocketdex_root_path, "tiltfiles", "pocketdex.tilt")
+    pocketdex_tilt = load_dynamic(pocketdex_tilt_path)
+
+    postgres_values_path = os.path.join(".", "localnet", "kubernetes", "values-pocketdex-postgres.yaml")
+    pgadmin_values_path = os.path.join(".", "localnet", "kubernetes", "values-pocketdex-pgadmin.yaml")
+    indexer_values_path = os.path.join(".", "localnet", "kubernetes", "values-pocketdex-indexer.yaml")
+    gql_engine_values_path = os.path.join(".", "localnet", "kubernetes", "values-pocketdex-gql-engine.yaml")
+    pocketdex_tilt["pocketdex"](pocketdex_root_path,
+                                genesis_file_name="localnet.json",
+                                postgres_values_path=postgres_values_path,
+                                pgadmin_values_path=pgadmin_values_path,
+                                indexer_values_path=indexer_values_path,
+                                gql_engine_values_path=gql_engine_values_path)
+
+
+# check_and_load_pocketdex checks if sibling pocketdex repo exists.
+# If it does, load the pocketdex.tilt file from the sibling repo.
+# Otherwise, check the `indexer.clone_if_not_present` flag in `localnet_config.yaml` and EITHER:
+#   1. clone pocketdex to ../pocketdex
+#   -- OR --
+#   2. Prints a message if true or false
+def check_and_load_pocketdex(indexer_config):
+    if indexer_config["enabled"]:
+        pocketdex_root_path = indexer_config["repo_path"]
+        if not os.path.exists(pocketdex_root_path):
+            if indexer_config["clone_if_not_present"]:
+                clone_repo("https://github.com/pokt-network/pocketdex", pocketdex_root_path)
+                load_pocketdex(pocketdex_root_path)
+            else:
+                pocketdex_disabled_resource("Pocketdex repo not found at {}. Set `clone_if_not_present` to `true` in `localnet_config.yaml`.".format(pocketdex_root_path))
+        else:
+            print("Using existing pocketdex repo")
+            load_pocketdex(pocketdex_root_path)
+    else:
+        pocketdex_disabled_resource("Pocketdex indexer disabled. Set `indexer.enabled` to `true` in `localnet_config.yaml` to enable it.")