Add Polymarket (#6575)

* polymarket init * add outcomes * add conditions * add ctf tokens * fix alias * change to view, remove dupes * market conditions enriched + clean up * fill in markets data * add trades * fix uniqueness * add capital actions * update schema * update fpmm source * add safe proxies, update user captial actions * deposits, withdrawals AND transfers.. * fix where clause * fix trades price, add shares * add wallet proxies and filter internal addresses * fix source def * fix and add metadata * fix incremental condition * also exclude fpmm trades * actually remove fpmm * split trades to activity and ordersmatched * Revert "split trades to activity and ordersmatched" This reverts commit 6996223. * add documentation * rerun models * push market metadata * see if balances run this way * remove * fix schema * change name of file * fix naming conventions * fix * maybe like this * remove date filter * fix test * push * fix tests * push * fix compile * add prices? * comm * ggwp? * revert position change * send it * fix * fix * fix * change name and convert to view * ss * fix market details * o god * fix schema * fix schema for real * fix incremental build of positions * fix fpmm recognition * fix address mention * test adding usdc * fix model name * add distinct * change to new balances table * fix * fix * fix safe * fix positions and trades failing tests * fix timeframe for testing * fix duplicates? * fix trades * fix incremental predicate * take out date filter * gg? * exclude system address * rename tests to data_tests * pls run * fix compile * fix * fix users group by * fix schema issues * fix failing test * fix incremental merge condition * fix DBT_INTERNAL_DEST column * fix exclusion conditiions * add in swap category to capital actions * fix capital actions again * fix conversion logic * fix pricefeeds * no from_hex * fix hex and switch to min_by * one min_by too many * fix pricefeed join condition * fix parentheses * move to _projects directory * update config blocks * add partitions on large raw tables * test using balances macro * fix columns, update schema file * add token_id back to unique --------- Co-authored-by: Boxer <[email protected]> Co-authored-by: 0xBoxer <[email protected]> Co-authored-by: jeff-dude <[email protected]> Co-authored-by: jeff-dude <[email protected]>
duneanalytics · Oct 28, 2024 · 84871cb · 84871cb
1 parent 1a2a1ad
commit 84871cb
Show file tree

Hide file tree

Showing 18 changed files with 1,562 additions and 0 deletions.
diff --git a/dbt_subprojects/daily_spellbook/models/_projects/polymarket/polygon/_schema.yml b/dbt_subprojects/daily_spellbook/models/_projects/polymarket/polygon/_schema.yml
diff --git a/...aily_spellbook/models/_projects/polymarket/polygon/polymarket_polygon_base_ctf_tokens.sql b/...aily_spellbook/models/_projects/polymarket/polygon/polymarket_polygon_base_ctf_tokens.sql
@@ -0,0 +1,45 @@
+{{
+  config(
+    schema = 'polymarket_polygon',
+    alias = 'base_ctf_tokens',
+    materialized = 'view'
+  )
+}}
+
+with ctf_tokens as (
+  select
+    *,
+    row_number() over (partition by condition_id, token0, token1 order by block_time) as rn
+  from (
+    select
+      evt_block_time as block_time,
+      evt_block_number as block_number,
+      conditionId as condition_id,
+      token0,
+      token1,
+      evt_index,
+      evt_tx_hash as tx_hash
+    from {{ source('polymarket_polygon', 'CTFExchange_evt_TokenRegistered') }}
+    union all
+    select
+      evt_block_time as block_time,
+      evt_block_number as block_number,
+      conditionId as condition_id,
+      token0,
+      token1,
+      evt_index,
+      evt_tx_hash as tx_hash
+    from {{ source('polymarket_polygon', 'NegRiskCtfExchange_evt_TokenRegistered') }}
+  ) t
+)
+
+select
+  block_time,
+  block_number,
+  condition_id,
+  token0,
+  token1,
+  evt_index,
+  tx_hash
+from ctf_tokens
+where rn = 1
diff --git a/...ellbook/models/_projects/polymarket/polygon/polymarket_polygon_base_market_conditions.sql b/...ellbook/models/_projects/polymarket/polygon/polymarket_polygon_base_market_conditions.sql
@@ -0,0 +1,39 @@
+{{
+  config(
+    schema = 'polymarket_polygon',
+    alias = 'base_market_conditions',
+    materialized = 'incremental',
+    file_format = 'delta',
+    incremental_strategy = 'merge',
+    unique_key = ['condition_id'],
+    incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')]
+  )
+}}
+
+with conditions as (
+  select
+    evt_block_time as block_time,
+    evt_block_number as block_number,
+    conditionId as condition_id,
+    questionId as question_id,
+    outcomeSlotCount as outcome_slot_count,
+    oracle,
+    evt_index,
+    evt_tx_hash as tx_hash
+  from {{ source('polymarket_polygon', 'ctf_evt_ConditionPreparation') }}
+  {% if is_incremental() %}
+  where {{ incremental_predicate('evt_block_time') }}
+  {% endif %}
+)
+
+select
+  c.block_time,
+  c.block_number,
+  c.condition_id,
+  c.question_id,
+  c.outcome_slot_count,
+  c.oracle,
+  c.evt_index,
+  c.tx_hash
+from conditions c
+  inner join {{ ref('polymarket_polygon_markets') }} m on c.question_id = m.question_id
diff --git a/...daily_spellbook/models/_projects/polymarket/polygon/polymarket_polygon_market_details.sql b/...daily_spellbook/models/_projects/polymarket/polygon/polymarket_polygon_market_details.sql
@@ -0,0 +1,117 @@
+{{
+  config(
+    schema = 'polymarket_polygon',
+    alias = 'market_details',
+    materialized = 'table',
+    post_hook='{{ expose_spells(blockchains = \'["polygon"]\',
+                                  spell_type = "project",
+                                  spell_name = "polymarket",
+                                  contributors = \'["tomfutago", "0xboxer"]\') }}'
+  )
+}}
+
+WITH onchain_metadata AS (
+  SELECT
+    evt_block_time AS block_time,
+    evt_block_number AS block_number,
+    marketId AS market_id,
+    CASE
+      WHEN json_value(from_utf8(data), 'lax $.title' NULL ON ERROR) IS NOT NULL
+      THEN json_extract_scalar(from_utf8(data), '$.title')
+      ELSE regexp_extract(from_utf8(data), 'title:\s*(.*?),\s*description:', 1)
+    END AS neg_risk_market_name,
+    CASE
+      WHEN json_value(from_utf8(data), 'lax $.description' NULL ON ERROR) IS NOT NULL
+      THEN json_extract_scalar(from_utf8(data), '$.description')
+      ELSE substr(from_utf8(data), strpos(from_utf8(data), 'description: ') + length('description: '))
+    END AS neg_risk_market_description,
+    oracle,
+    feeBips AS fee_bips,
+    evt_index,
+    evt_tx_hash AS tx_hash
+  FROM {{ source('polymarket_polygon', 'NegRiskAdapter_evt_MarketPrepared') }}
+)
+
+,polymarket_api_upload as
+(
+    SELECT
+      CASE 
+        WHEN neg_risk = true THEN neg_risk_market_id
+        WHEN neg_risk = false THEN cast(condition_id AS varchar)
+      END AS unique_key,
+      try_cast(substring(token_1_id, 5) AS UINT256) AS token_id,
+      token_1_outcome AS token_outcome,
+      *
+    FROM {{ source('dune', 'dataset_polymarket_markets', database="dune") }}
+    UNION ALL 
+    SELECT
+      CASE 
+        WHEN neg_risk = true THEN neg_risk_market_id
+        WHEN neg_risk = false THEN cast(condition_id AS varchar)
+      END AS unique_key,
+      try_cast(substring(token_2_id, 5) AS UINT256) AS token_id,
+      token_2_outcome AS token_outcome,
+      *
+    FROM {{ source('dune', 'dataset_polymarket_markets', database="dune") }}
+),
+
+combine as 
+(
+Select 
+    api.*, 
+    oc.* 
+from polymarket_api_upload api
+LEFT JOIN onchain_metadata oc ON cast(oc.market_id AS varchar) = api.unique_key
+)
+
+, naming_things as (
+SELECT 
+  unique_key AS unique_key,
+  condition_id AS condition_id,
+  CASE 
+    WHEN neg_risk_market_id IS NULL THEN NULL
+    ELSE neg_risk_market_id
+  END AS event_market_id,
+  CASE 
+    WHEN neg_risk_market_name IS NULL THEN 'single market'
+    ELSE neg_risk_market_name
+  END AS event_market_name,
+  neg_risk_market_description as event_market_description,
+  c.question_id AS question_id,
+  question,
+  description as market_description,
+  try_cast(token_id AS UINT256) AS token_id,
+  token_outcome,
+  token_outcome || '-' || question as token_outcome_name,
+  active,
+  archived,
+  closed,
+  accepting_orders,
+  enable_order_book,
+  neg_risk AS neg_risk,
+  CASE 
+    WHEN neg_risk = false THEN get_href('https://polymarket.com/event/' || market_slug, market_slug)
+    WHEN neg_risk = true  THEN  get_href('https://polymarket.com/event/' || replace(replace(replace(lower(neg_risk_market_name), ' ', '-'), '$', ''), '''',''), neg_risk_market_name)
+  END AS polymarket_link,
+  accepting_order_timestamp as market_start_time,
+  end_date_iso as market_end_time,
+  game_start_time,
+  seconds_delay,
+  fpmm,
+  icon,
+  image,
+  tags, 
+  oracle AS oracle,
+  fee_bips,
+  case when pm.outcome is null then 'unresolved' else pm.outcome end as outcome,
+  pm.block_time as resolved_on_timestamp,
+  last_updated_at as last_uploaded_at
+FROM combine c
+left join {{ ref('polymarket_polygon_market_outcomes') }} pm on pm.question_id = c.question_id
+)
+
+--these nulls get introduced by the polymarket api responses 
+
+SELECT * FROM naming_things
+where token_id is not null
+and condition_id is not null
diff --git a/...aily_spellbook/models/_projects/polymarket/polygon/polymarket_polygon_market_outcomes.sql b/...aily_spellbook/models/_projects/polymarket/polygon/polymarket_polygon_market_outcomes.sql
@@ -0,0 +1,67 @@
+{{
+  config(
+    schema = 'polymarket_polygon',
+    alias = 'market_outcomes',
+    materialized = 'incremental',
+    file_format = 'delta',
+    incremental_strategy = 'merge',
+    unique_key = ['question_id'],
+    incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')],
+    post_hook = '{{ expose_spells(blockchains = \'["polygon"]\',
+                                  spell_type = "project",
+                                  spell_name = "polymarket",
+                                  contributors = \'["tomfutago, 0xboxer"]\') }}'
+  )
+}}
+
+select
+  evt_block_time as block_time,
+  evt_block_number as block_number,
+  'uma-v1' as source,
+  questionID as question_id,
+  case settledPrice / 1e18
+    when 1 then 'yes'
+    when 0.5 then '50/50'
+    else 'no'
+  end as outcome,
+  evt_index,
+  evt_tx_hash as tx_hash
+from {{ source('minereum_polygon', 'UmaConditionalTokensBinaryAdapter_evt_QuestionSettled') }}
+{% if is_incremental() %}
+where {{ incremental_predicate('evt_block_time') }}
+{% endif %}
+
+union all
+
+select
+  evt_block_time as block_time,
+  evt_block_number as block_number,
+  'uma-v2' as source,
+  questionID as question_id,
+  case settledPrice / 1e18
+    when 1 then 'yes'
+    when 0.5 then '50/50'
+    else 'no'
+  end as outcome,
+  evt_index,
+  evt_tx_hash as tx_hash
+from {{ source('polymarket_polygon', 'UmaCtfAdapter_evt_QuestionResolved') }}
+{% if is_incremental() %}
+where {{ incremental_predicate('evt_block_time') }}
+{% endif %}
+
+union all
+
+select
+  evt_block_time as block_time,
+  evt_block_number as block_number,
+  'polymarket' as source,
+  --marketId as market_id,
+  questionId as question_id,
+  if(outcome, 'yes', 'no') as outcome,
+  evt_index,
+  evt_tx_hash as tx_hash
+from {{ source('polymarket_polygon', 'NegRiskAdapter_evt_OutcomeReported') }}
+{% if is_incremental() %}
+where {{ incremental_predicate('evt_block_time') }}
+{% endif %}
diff --git a/..._spellbook/models/_projects/polymarket/polygon/polymarket_polygon_market_prices_daily.sql b/..._spellbook/models/_projects/polymarket/polygon/polymarket_polygon_market_prices_daily.sql
@@ -0,0 +1,49 @@
+{{ config(
+    schema = 'polymarket_polygon',
+    alias = 'market_prices_daily',
+    materialized = 'view',
+    post_hook = '{{ expose_spells(blockchains = \'["polygon"]\',
+                                  spell_type = "project",
+                                  spell_name = "polymarket",
+                                  contributors = \'["0xboxer, tomfutago"]\') }}'
+  )
+}}
+
+WITH changed_prices AS (
+    SELECT
+        date_trunc('day', block_time) AS day,
+        block_time,
+        condition_id,
+        asset_id AS token_id,
+        price,
+        LEAD(CAST(date_trunc('day', block_time) AS timestamp)) OVER (PARTITION BY asset_id ORDER BY block_time ASC) AS next_update_day
+    FROM (
+        SELECT *,
+               ROW_NUMBER() OVER (PARTITION BY DATE_TRUNC('day', block_time), asset_id ORDER BY block_time DESC) as rn
+        FROM {{ ref('polymarket_polygon_market_trades_raw') }}
+    ) ranked
+    WHERE rn = 1
+),
+
+--sequences are limited to 10k so just pulling this in from the transactions table, no other relationship
+days AS (
+    SELECT *
+    FROM UNNEST(
+        SEQUENCE(CAST('2015-01-01' AS date), DATE(DATE_TRUNC('day', NOW())), INTERVAL '1' day)
+    ) AS foo(day)
+),
+
+forward_fill AS (
+    SELECT
+        CAST(d.day AS timestamp) AS day,
+        lp.condition_id,
+        lp.token_id,
+        lp.price
+    FROM days d
+    LEFT JOIN changed_prices lp
+        ON d.day >= lp.day
+        AND (lp.next_update_day IS NULL OR d.day < lp.next_update_day)
+)
+
+SELECT * FROM forward_fill
+WHERE price > 0
diff --git a/...spellbook/models/_projects/polymarket/polygon/polymarket_polygon_market_prices_hourly.sql b/...spellbook/models/_projects/polymarket/polygon/polymarket_polygon_market_prices_hourly.sql
@@ -0,0 +1,46 @@
+{{ config(
+    schema = 'polymarket_polygon',
+    alias = 'market_prices_hourly',
+    materialized = 'view',
+    post_hook = '{{ expose_spells(blockchains = \'["polygon"]\',
+                                  spell_type = "project",
+                                  spell_name = "polymarket",
+                                  contributors = \'["0xboxer, tomfutago"]\') }}'
+  )
+}}
+
+WITH changed_prices AS (
+    SELECT
+        date_trunc('hour', block_time) AS hour,
+        block_time,
+        condition_id,
+        asset_id AS token_id,
+        price,
+        LEAD(CAST(date_trunc('hour', block_time) AS timestamp)) OVER (PARTITION BY asset_id ORDER BY block_time ASC) AS next_update_hour
+    FROM (
+        SELECT *,
+               ROW_NUMBER() OVER (PARTITION BY DATE_TRUNC('hour', block_time), asset_id ORDER BY block_time DESC) as rn
+        FROM {{ ref('polymarket_polygon_market_trades_raw') }}
+    ) ranked
+    WHERE rn = 1
+),
+
+hours AS (
+   Select distinct date_trunc('hour', block_time) as hour
+   from {{ source('polygon', 'transactions') }}
+),
+
+forward_fill AS (
+    SELECT
+        CAST(h.hour AS timestamp) AS hour,
+        lp.condition_id,
+        lp.token_id,
+        lp.price
+    FROM hours h
+    LEFT JOIN changed_prices lp
+        ON h.hour >= lp.hour
+        AND (lp.next_update_hour IS NULL OR h.hour < lp.next_update_hour)
+)
+
+SELECT * FROM forward_fill
+WHERE price > 0