From abb60b4ef7d280603b0e84fb2855ed0f6dfcc114 Mon Sep 17 00:00:00 2001 From: Franco Reyes <79299724+francojreyes@users.noreply.github.com> Date: Wed, 26 Jun 2024 01:27:35 +1000 Subject: [PATCH] docs: documentation of batch_insert route (#11) * docs: documentation for batch_insert * fix: typo --- app/helpers/postgres.py | 4 ++-- scrapers.md | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/app/helpers/postgres.py b/app/helpers/postgres.py index b7771a4..bce081a 100644 --- a/app/helpers/postgres.py +++ b/app/helpers/postgres.py @@ -49,11 +49,11 @@ def execute_up_down(metadata: Metadata): ) try: - cur.execute(metadata.sql_down) + cur.execute(metadata.sql_up) except Error as e: raise HTTPException( status_code=400, - detail=f"sql_down of '{metadata.table_name}' does not fully undo sql_up" + detail=f"sql_down of '{metadata.table_name}' does not fully undo sql_up:\n{e}" ) diff --git a/scrapers.md b/scrapers.md index cbf76e4..41b6741 100644 --- a/scrapers.md +++ b/scrapers.md @@ -76,6 +76,30 @@ X-API-Key: my_key ``` +### POST `/batch_insert` Route +If you have a scraper which scrapes data for and inserts into multiple tables at once, you can use the `/batch_insert` route. This route ensures that if one of the inserts fails, none of the inserts will be committed. + +The `/batch_insert` route accepts a list of objects containing `metadata` and `payload` (note that this list is not in an object, the top level JSON entity in the body of the request is the list). + +Example: +```http request +POST /batch_insert HTTP/1.1 +Content-Type: application/json +X-API-Key: my_key + +[ + { + "metadata": { ... }, + "payload": [ ... ] + }, + { + "metadata": { ... }, + "payload": [ ... ] + }, + ... +] +``` + ## Multiple Scrapers for One Table If you want to connect multiple scrapers to the same table, for example if you have multiple data sources, then Hasuragres is able to support this. Follow the guidelines below to set this up.