Merge branch 'main' into support_postgres_ingestion_for_analytics

supabase · Aug 7, 2023 · e179faf · e179faf
2 parents 8671a33 + 6f8b258
commit e179faf
Show file tree

Hide file tree

Showing 36 changed files with 462 additions and 170 deletions.
diff --git a/cmd/db.go b/cmd/db.go
@@ -121,6 +121,7 @@ var (
 	}
 
 	dryRun       bool
+	includeAll   bool
 	includeRoles bool
 	includeSeed  bool
 
@@ -133,7 +134,7 @@ var (
 				return err
 			}
 			ctx, _ := signal.NotifyContext(cmd.Context(), os.Interrupt)
-			return push.Run(ctx, dryRun, includeRoles, includeSeed, dbConfig, fsys)
+			return push.Run(ctx, dryRun, includeAll, includeRoles, includeSeed, dbConfig, fsys)
 		},
 	}
 
@@ -263,6 +264,7 @@ func init() {
 	dbCmd.AddCommand(dbDumpCmd)
 	// Build push command
 	pushFlags := dbPushCmd.Flags()
+	pushFlags.BoolVar(&includeAll, "include-all", false, "Include all migrations not found on remote history table.")
 	pushFlags.BoolVar(&includeRoles, "include-roles", false, "Include custom roles from "+utils.CustomRolesPath+".")
 	pushFlags.BoolVar(&includeSeed, "include-seed", false, "Include seed data from "+utils.SeedDataPath+".")
 	pushFlags.BoolVar(&dryRun, "dry-run", false, "Print the migrations that would be applied, but don't actually apply them.")

diff --git a/cmd/migration.go b/cmd/migration.go
@@ -95,7 +95,7 @@ var (
 		Short: "Apply pending migrations to local database",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx, _ := signal.NotifyContext(cmd.Context(), os.Interrupt)
-			return up.Run(ctx, afero.NewOsFs())
+			return up.Run(ctx, includeAll, afero.NewOsFs())
 		},
 		PostRun: func(cmd *cobra.Command, args []string) {
 			fmt.Println("Local database is up to date.")
@@ -131,6 +131,8 @@ func init() {
 	migrationSquashCmd.MarkFlagsMutuallyExclusive("db-url", "linked")
 	migrationCmd.AddCommand(migrationSquashCmd)
 	// Build up command
+	upFlags := migrationUpCmd.Flags()
+	upFlags.BoolVar(&includeAll, "include-all", false, "Include all migrations not found on remote history table.")
 	migrationCmd.AddCommand(migrationUpCmd)
 	// Build new command
 	migrationCmd.AddCommand(migrationNewCmd)

diff --git a/docs/supabase/db/dump.md b/docs/supabase/db/dump.md
@@ -4,6 +4,6 @@ Dumps contents from a remote database.
 
 Requires your local project to be linked to a remote database by running `supabase link`. For self-hosted databases, you can pass in the connection parameters using `--db-url` flag.
 
-Runs `pg_dump` in a container with additional flags to exclude Supabase managed schemas. The ignored schemas include auth, stroage, and those created by extensions.
+Runs `pg_dump` in a container with additional flags to exclude Supabase managed schemas. The ignored schemas include auth, storage, and those created by extensions.
 
 The default dump does not contain any data or custom roles. To dump those contents explicitly, specify either the `--data-only` and `--role-only` flag.
diff --git a/docs/supabase/inspect/db-bloat.md b/docs/supabase/inspect/db-bloat.md
@@ -0,0 +1,14 @@
+## db-bloat
+
+This command displays an estimation of table "bloat" - Due to Postgres' [MVCC](https://www.postgresql.org/docs/current/mvcc.html) when data is updated or deleted new rows are created and old rows are made invisible and marked as "dead tuples". Usually the [autovaccum](https://supabase.com/docs/guides/platform/database-size#vacuum-operations) process will asynchronously clean the dead tuples. Sometimes the autovaccum is unable to work fast enough to reduce or prevent tables from becoming bloated. High bloat can slow down queries, cause excessive IOPS and waste space in your database.
+
+Tables with a high bloat ratio should be investigated to see if there are vacuuming is not quick enough or there are other issues.
+
+```
+    TYPE  │ SCHEMA NAME │        OBJECT NAME         │ BLOAT │ WASTE
+  ────────┼─────────────┼────────────────────────────┼───────┼─────────────
+    table │ public      │ very_bloated_table         │  41.0 │ 700 MB
+    table │ public      │ my_table                   │   4.0 │ 76 MB
+    table │ public      │ happy_table                │   1.0 │ 1472 kB
+    index │ public      │ happy_table::my_nice_index │   0.7 │ 880 kB
+```
diff --git a/docs/supabase/inspect/db-blocking.md b/docs/supabase/inspect/db-blocking.md
@@ -0,0 +1,9 @@
+## db-blocking
+
+This command shows you statements that are currently holding locks and blocking, as well as the statement that is being blocked. This can be used in conjunction with `inspect db locks` to determine which statements need to be terminated in order to resolve lock contention.
+
+```
+    BLOCKED PID │ BLOCKING STATEMENT           │ BLOCKING DURATION │ BLOCKING PID │ BLOCKED STATEMENT                                                                      │ BLOCKED DURATION
+  ──────────────┼──────────────────────────────┼───────────────────┼──────────────┼────────────────────────────────────────────────────────────────────────────────────────┼───────────────────
+    253         │ select count(*) from mytable │ 00:00:03.838314   │        13495 │ UPDATE "mytable" SET "updated_at" = '2023─08─03 14:07:04.746688' WHERE "id" = 83719341 │ 00:00:03.821826
+```
diff --git a/docs/supabase/inspect/db-cache-hit.md b/docs/supabase/inspect/db-cache-hit.md
@@ -0,0 +1,14 @@
+# db-cache-hit
+
+This command provides information on the efficiency of the buffer cache and how often your queries have to go hit the disk rather than reading from memory. Information on both index reads (`index hit rate`) as well as table reads (`table hit rate`) are shown. In general, databases with low cache hit rates perform worse as it is slower to go to disk than retrieve data from memory. If your table hit rate is low, this can indicate that you do not have enough RAM and you may benefit from upgrading to a larger compute addon with more memory. If your index hit rate is low, this may indicate that there is scope to add more appropriate indexes.
+
+The hit rates are calculated as a ratio of number of table or index blocks fetched from the postgres buffer cache against the sum of cached blocks and uncached blocks read from disk.
+
+On smaller compute plans (free, small, medium), a ratio of below 99% can indicate a problem. On larger plans the hit rates may be lower but performance will remain constant as the data may use the OS cache rather than Postgres buffer cache.
+
+```
+         NAME      │  RATIO
+  ─────────────────┼───────────
+    index hit rate │ 0.996621
+    table hit rate │ 0.999341
+ ```
diff --git a/docs/supabase/inspect/db-calls.md b/docs/supabase/inspect/db-calls.md
@@ -0,0 +1,15 @@
+# db-calls
+
+This command is much like the `supabase inspect db outliers` command, but ordered by the number of times a statement has been called.
+
+You can use this information to see which queries are called most often, which can potentially be good candidates for optimisation.
+
+```
+
+                        QUERY                      │ TOTAL EXECUTION TIME │ PROPORTION OF TOTAL EXEC TIME │ NUMBER CALLS │  SYNC IO TIME
+  ─────────────────────────────────────────────────┼──────────────────────┼───────────────────────────────┼──────────────┼──────────────────
+    SELECT * FROM users WHERE id = $1              │ 14:50:11.828939      │ 89.8%                         │  183,389,757 │ 00:00:00.002018
+    SELECT * FROM user_events                      │ 01:20:23.466633      │ 1.4%                          │       78,325 │ 00:00:00
+    INSERT INTO users (email, name) VALUES ($1, $2)│ 00:40:11.616882      │ 0.8%                          │       54,003 │ 00:00:00.000322
+
+```
diff --git a/docs/supabase/inspect/db-index-sizes.md b/docs/supabase/inspect/db-index-sizes.md
@@ -0,0 +1,14 @@
+# db-index-sizes
+
+This command displays the size of each each index in the database. It is calculated by taking the number of pages (reported in `relpages`) and multiplying it by the page size (8192 bytes).
+
+```
+              NAME              │    SIZE
+  ──────────────────────────────┼─────────────
+    user_events_index           │ 2082 MB
+    job_run_details_pkey        │ 3856 kB
+    schema_migrations_pkey      │ 16 kB
+    refresh_tokens_token_unique │ 8192 bytes
+    users_instance_id_idx       │ 0 bytes
+    buckets_pkey                │ 0 bytes
+```
diff --git a/docs/supabase/inspect/db-index-usage.md b/docs/supabase/inspect/db-index-usage.md
@@ -0,0 +1,14 @@
+# db-index-usage
+
+This command provides information on the efficiency of indexes, represented as what percentage of total scans were index scans. A low percentage can indicate under indexing, or wrong data being indexed.
+
+```
+       TABLE NAME     │ PERCENTAGE OF TIMES INDEX USED │ ROWS IN TABLE
+  ────────────────────┼────────────────────────────────┼────────────────
+    user_events       │                             99 │       4225318 
+    user_feed         │                             99 │       3581573
+    unindexed_table   │                              0 │        322911
+    job               │                            100 │         33242
+    schema_migrations │                             97 │             0
+    migrations        │ Insufficient data              │             0
+```
diff --git a/docs/supabase/inspect/db-locks.md b/docs/supabase/inspect/db-locks.md
@@ -0,0 +1,11 @@
+# db-locks
+
+This command displays queries that have taken out an exclusive lock on a relation. Exclusive locks typically prevent other operations on that relation from taking place, and can be a cause of "hung" queries that are waiting for a lock to be granted.
+
+If you see a query that is hanging for a very long time or causing blocking issues you may consider killing the query by connecting to the database and running `SELECT pg_cancel_backend(PID);` to cancel the query. If the query still does not stop you can force a hard stop by running `SELECT pg_terminate_backend(PID);`
+
+```
+     PID   │ RELNAME │ TRANSACTION ID │ GRANTED │                  QUERY                  │   AGE
+  ─────────┼─────────┼────────────────┼─────────┼─────────────────────────────────────────┼───────────
+    328112 │ null    │              0 │ t       │ SELECT * FROM logs;                     │ 00:04:20
+```
diff --git a/docs/supabase/inspect/db-long-running-queries.md b/docs/supabase/inspect/db-long-running-queries.md
@@ -0,0 +1,11 @@
+# db-long-running-queries
+
+This command displays currently running queries, that have been running for longer than 5 minutes, descending by duration. Very long running queries can be a source of multiple issues, such as preventing DDL statements completing or vacuum being unable to update `relfrozenxid`.
+
+```
+  PID  │     DURATION    │                                         QUERY
+───────┼─────────────────┼───────────────────────────────────────────────────────────────────────────────────────
+ 19578 | 02:29:11.200129 | EXPLAIN SELECT  "students".* FROM "students"  WHERE "students"."id" = 1450645 LIMIT 1
+ 19465 | 02:26:05.542653 | EXPLAIN SELECT  "students".* FROM "students"  WHERE "students"."id" = 1889881 LIMIT 1
+ 19632 | 02:24:46.962818 | EXPLAIN SELECT  "students".* FROM "students"  WHERE "students"."id" = 1581884 LIMIT 1
+```
diff --git a/docs/supabase/inspect/db-outliers.md b/docs/supabase/inspect/db-outliers.md
@@ -0,0 +1,16 @@
+# db-outliers
+
+This command displays statements, obtained from `pg_stat_statements`, ordered by the amount of time to execute in aggregate. This includes the statement itself, the total execution time for that statement, the proportion of total execution time for all statements that statement has taken up, the number of times that statement has been called, and the amount of time that statement spent on synchronous I/O (reading/writing from the file system).
+
+Typically, an efficient query will have an appropriate ratio of calls to total execution time, with as little time spent on I/O as possible. Queries that have a high total execution time but low call count should be investigated to improve their performance. Queries that have a high proportion of execution time being spent on synchronous I/O should also be investigated.
+
+```
+
+                 QUERY                   │ EXECUTION TIME   │ PROPORTION OF EXEC TIME │ NUMBER CALLS │ SYNC IO TIME
+─────────────────────────────────────────┼──────────────────┼─────────────────────────┼──────────────┼───────────────
+ SELECT * FROM archivable_usage_events.. │ 154:39:26.431466 │ 72.2%                   │ 34,211,877   │ 00:00:00
+ COPY public.archivable_usage_events (.. │ 50:38:33.198418  │ 23.6%                   │ 13           │ 13:34:21.00108
+ COPY public.usage_events (id, reporte.. │ 02:32:16.335233  │ 1.2%                    │ 13           │ 00:34:19.784318
+ INSERT INTO usage_events (id, retaine.. │ 01:42:59.436532  │ 0.8%                    │ 12,328,187   │ 00:00:00
+ SELECT * FROM usage_events WHERE (alp.. │ 01:18:10.754354  │ 0.6%                    │ 102,114,301  │ 00:00:00
+```
diff --git a/docs/supabase/inspect/db-replication-slots.md b/docs/supabase/inspect/db-replication-slots.md
@@ -0,0 +1,13 @@
+# db-replication-slots
+
+This command shows information about [logical replication slots](https://www.postgresql.org/docs/current/logical-replication.html) that are setup on the database. It shows if the slot is active, the state of the WAL sender process ('startup', 'catchup', 'streaming', 'backup', 'stopping') the replication client address and the replication lag in GB.
+
+This command is useful to check that the amount of replication lag is as low as possible, replication lag can occur due to network latency issues, slow disk I/O, long running transactions or lack of ability for the subscriber to consume WAL fast enough.
+
+
+```
+                       NAME                    │ ACTIVE │ STATE   │ REPLICATION CLIENT ADDRESS │ REPLICATION LAG GB
+  ─────────────────────────────────────────────┼────────┼─────────┼────────────────────────────┼─────────────────────
+    supabase_realtime_replication_slot         │ t      │ N/A     │ N/A                        │                  0
+    datastream                                 │ t      │ catchup │ 24.201.24.106              │                 45
+```
diff --git a/docs/supabase/inspect/db-role-connections.md b/docs/supabase/inspect/db-role-connections.md
@@ -0,0 +1,33 @@
+# db-role-connections
+
+This command shows the number of active connections for each database roles to see which specific role might be consuming more connections than expected.
+
+This is a Supabase specific command. You can see this breakdown on the dashboard as well:
+https://app.supabase.com/project/_/database/roles
+
+The maximum number of active connections depends [on your instance size](https://supabase.com/docs/guides/platform/compute-add-ons). You can [manually overwrite](https://supabase.com/docs/guides/platform/performance#allowing-higher-number-of-connections) the allowed number of connection but it is not advised.
+
+```
+
+
+            ROLE NAME         │ ACTIVE CONNCTION
+  ────────────────────────────┼───────────────────
+    authenticator             │                5
+    postgres                  │                5
+    supabase_admin            │                1
+    pgbouncer                 │                1
+    anon                      │                0
+    authenticated             │                0
+    service_role              │                0
+    dashboard_user            │                0
+    supabase_auth_admin       │                0
+    supabase_storage_admin    │                0
+    supabase_functions_admin  │                0
+    pgsodium_keyholder        │                0
+    pg_read_all_data          │                0
+    pg_write_all_data         │                0
+    pg_monitor                │                0
+
+Active connections 12/90
+
+```
diff --git a/docs/supabase/inspect/db-seq-scans.md b/docs/supabase/inspect/db-seq-scans.md
@@ -0,0 +1,15 @@
+# db-seq-scans
+
+This command displays the number of sequential scans recorded against all tables, descending by count of sequential scans. Tables that have very high numbers of sequential scans may be underindexed, and it may be worth investigating queries that read from these tables.
+
+
+```
+                  NAME               │ COUNT
+  ───────────────────────────────────┼─────────
+    emails                           │ 182435
+    users                            │  25063
+    job_run_details                  │     60
+    schema_migrations                │      0
+    migrations                       │      0
+```
+
diff --git a/docs/supabase/inspect/db-table-index-sizes.md b/docs/supabase/inspect/db-table-index-sizes.md
@@ -0,0 +1,13 @@
+# db-table-index-sizes
+
+This command displays the total size of indexes for each table. It is calculated by using the system administration function `pg_indexes_size()`.
+
+```
+                 TABLE               │ INDEX SIZE
+  ───────────────────────────────────┼─────────────
+    job_run_details                  │ 10104 kB
+    users                            │ 128 kB
+    job                              │ 32 kB
+    instances                        │ 8192 bytes
+    http_request_queue               │ 0 bytes
+```
diff --git a/docs/supabase/inspect/db-table-record-counts.md b/docs/supabase/inspect/db-table-record-counts.md
@@ -0,0 +1,13 @@
+# db-table-record-counts
+
+This command displays an estimated count of rows per table, descending by estimated count. The estimated count is derived from `n_live_tup`, which is updated by vacuum operations. Due to the way `n_live_tup` is populated, sparse vs. dense pages can result in estimations that are significantly out from the real count of rows.
+
+
+```
+       NAME    │ ESTIMATED COUNT
+  ─────────────┼──────────────────
+    logs       │          322943
+    emails     │            1103
+    job        │               1
+    migrations │               0
+```
diff --git a/docs/supabase/inspect/db-table-sizes.md b/docs/supabase/inspect/db-table-sizes.md
@@ -0,0 +1,14 @@
+# db-table-sizes
+
+This command displays the size of each table in the database. It is calculated by using the system administration function `pg_table_size()`, which includes the size of the main data fork, free space map, visibility map and TOAST data. It does not include the size of the table's indexes.
+
+
+```
+                  NAME               │    SIZE
+  ───────────────────────────────────┼─────────────
+    job_run_details                  │ 385 MB
+    emails                           │ 584 kB
+    job                              │ 40 kB
+    sessions                         │ 0 bytes
+    prod_resource_notifications_meta │ 0 bytes
+```
diff --git a/docs/supabase/inspect/db-total-index-size.md b/docs/supabase/inspect/db-total-index-size.md
@@ -0,0 +1,9 @@
+# db-total-index-size
+
+This command displays the total size of all indexes on the database. It is calculated by taking the number of pages (reported in `relpages`) and multiplying it by the page size (8192 bytes).
+
+```
+    SIZE
+  ─────────
+    12 MB
+```