Merge pull request #5 from NEAR-DevHub/test/local-nearblocks

Test local nearblocks reponses

.gitignore

@@ -1,4 +1,7 @@
 /target
 .envrc
 .bin
-.env
+.env
+devhub.near*
+events-committee.near*
+infrastructure-committee.near*

bring-cache-up-to-date.py

@@ -0,0 +1,50 @@
+import time
+import requests
+
+local = False 
+reset_from_zero = False # False to continue from where it left off  
+fly_app_name = "devhub-cache-api-rs"
+# ~120 calls for devhub
+# ~20 calls for infra
+# ~40 calls for events
+max_calls = 120 # This is for devhub to catch up to the latest block
+
+base_url = f"http://localhost:8080/" if local else f"https://{fly_app_name}.fly.dev/"
+
+def call_api(count):
+    url = f"{base_url}proposals"  # Replace with your API URL
+    try:
+        response = requests.get(url)
+        if response.status_code == 200:
+            print(f"{count} API call successful: - response length {response.json().get('total_records')}")
+        else:
+            print("API call failed with status code:", response.status_code)
+    except requests.exceptions.RequestException as e:
+        print("An error occurred:", e)
+    except Exception as e:
+        print("An error2 occurred:", e)
+        print(response.json())
+
+def reset_cache():
+    url = f"{base_url}proposals/info/reset"  # Replace with your API URL
+    try:
+        response = requests.get(url)
+        if response.status_code == 200:
+            print("Cache reset successful")
+        else:
+            print("Cache reset failed with status code:", response.status_code)
+    except requests.exceptions.RequestException as e:
+        print("An error occurred:", e)
+
+def main():
+    if reset_from_zero:
+        reset_cache()
+    count = 0
+    while count < max_calls: 
+        call_api(count)
+        count += 1
+        # 6 calls/minute limit of near blocks https://nearblocks.io/apis#
+        time.sleep(0.5)  # Wait for 11 seconds before the next call
+
+if __name__ == "__main__":
+    main()

migrations/20241205065201_cursor.sql

@@ -0,0 +1,7 @@
+-- Add migration script here
+ALTER TABLE last_updated_info ADD COLUMN cursor varchar;
+
+UPDATE last_updated_info SET cursor = '';
+
+-- Add NOT NULL constraint
+ALTER TABLE last_updated_info ALTER COLUMN cursor SET NOT NULL;

scripts/check_max_snapshots_response.js

@@ -0,0 +1,143 @@
+let response = [
+  {
+    proposal_id: 253,
+    block_height: 133226797,
+    ts: 1732211893808166100,
+    editor_id: "theori.near",
+    social_db_post_block_height: 132779971,
+    labels: [],
+    proposal_version: "V0",
+    proposal_body_version: "V2",
+    name: "DevHub Fin Ops 09/23/2024 - 09/27/2024",
+    category: "DevDAO Operations",
+    summary:
+      "Oversee that proposals flow thru the payment process efficiently. Continuously improve upon the process.",
+    description:
+      "I facilitate the payment processing of proposal submissions. I operate as the hub between DevHub admin, contributors and NF in communictions and problem solving as it relates to payments of completed work by the community.\nThe time I spend in this capacity involves weekly update meetings with DevHub admin, email and TG communications with NF legal, NF finance, DevHub admins and DevHub community contributors. We utilize database tools to manage the payment process.\nThe DevHub community members have benefited by receiving their proposal payments much more timely, recently narrowing in on 48 hr turn around and reducing the administrative steps each contributor had to complete to receive payment. The larger NEAR ecosystem has benefited by reducing the amount of time spent by the NF Legal and Finance team members, as well as DevHub admins, on the proposal payment process. These process improvements brought additional controls ensuring overall accuracy as DevHub has a fiduciary responsibility to NF that grant funds are spent responsibly, payments are correct and appropriate KYC/KYB measures have been taken.\nWeek 09/23/2024 - 09/27/2024\nExtra time was spent setting up testing invoicing for the Treasury system and communicating with teammembers on various hurdles and outcomes.\nTotal hours 15 @$80/hr\nRate determination is below the low end of the average hourly contract rate ($87 - $162) for an accountant with the same years of experience. That average rate does not account for my 5 years of blockchain specific experience. I'm not fulfilling all the duties that typically fall under this type of role, that is why I proposed a rate lower than the low end of the average. https://www.salary.com/research/salary/benchmark/controller-hourly-wages",
+    linked_proposals: [],
+    linked_rfp: null,
+    requested_sponsorship_usd_amount: 1200,
+    requested_sponsorship_paid_in_currency: "USDC",
+    requested_sponsor: "neardevdao.near",
+    receiver_account:
+      "9bd73bc9ba6ff289e2deb597dd6fe83dc0056af7a2937d6dec119660ba0146a4",
+    supervisor: "theori.near",
+    timeline:
+      '{"timeline_version":"V1","status":"APPROVED","sponsor_requested_review":true,"reviewer_completed_attestation":false,"kyc_verified":true}',
+    views: null,
+  },
+  {
+    proposal_id: 253,
+    block_height: 133169442,
+    ts: 1732148248149526000,
+    editor_id: "theori.near",
+    social_db_post_block_height: 132779971,
+    labels: [],
+    proposal_version: "V0",
+    proposal_body_version: "V2",
+    name: "DevHub Fin Ops 09/23/2024 - 09/27/2024",
+    category: "DevDAO Operations",
+    summary:
+      "Oversee that proposals flow thru the payment process efficiently. Continuously improve upon the process.",
+    description:
+      "I facilitate the payment processing of proposal submissions. I operate as the hub between DevHub admin, contributors and NF in communictions and problem solving as it relates to payments of completed work by the community.\nThe time I spend in this capacity involves weekly update meetings with DevHub admin, email and TG communications with NF legal, NF finance, DevHub admins and DevHub community contributors. We utilize database tools to manage the payment process.\nThe DevHub community members have benefited by receiving their proposal payments much more timely, recently narrowing in on 48 hr turn around and reducing the administrative steps each contributor had to complete to receive payment. The larger NEAR ecosystem has benefited by reducing the amount of time spent by the NF Legal and Finance team members, as well as DevHub admins, on the proposal payment process. These process improvements brought additional controls ensuring overall accuracy as DevHub has a fiduciary responsibility to NF that grant funds are spent responsibly, payments are correct and appropriate KYC/KYB measures have been taken.\nWeek 09/23/2024 - 09/27/2024\nExtra time was spent setting up testing invoicing for the Treasury system and communicating with teammembers on various hurdles and outcomes.\nTotal hours 15 @$80/hr\nRate determination is below the low end of the average hourly contract rate ($87 - $162) for an accountant with the same years of experience. That average rate does not account for my 5 years of blockchain specific experience. I'm not fulfilling all the duties that typically fall under this type of role, that is why I proposed a rate lower than the low end of the average. https://www.salary.com/research/salary/benchmark/controller-hourly-wages",
+    linked_proposals: [],
+    linked_rfp: null,
+    requested_sponsorship_usd_amount: 1200,
+    requested_sponsorship_paid_in_currency: "USDC",
+    requested_sponsor: "neardevdao.near",
+    receiver_account:
+      "9bd73bc9ba6ff289e2deb597dd6fe83dc0056af7a2937d6dec119660ba0146a4",
+    supervisor: "theori.near",
+    timeline:
+      '{"status":"REVIEW","sponsor_requested_review":true,"reviewer_completed_attestation":false}',
+    views: null,
+  },
+  {
+    proposal_id: 253,
+    block_height: 133169475,
+    ts: 1732148284402408700,
+    editor_id: "theori.near",
+    social_db_post_block_height: 132779971,
+    labels: [],
+    proposal_version: "V0",
+    proposal_body_version: "V2",
+    name: "DevHub Fin Ops 09/23/2024 - 09/27/2024",
+    category: "DevDAO Operations",
+    summary:
+      "Oversee that proposals flow thru the payment process efficiently. Continuously improve upon the process.",
+    description:
+      "I facilitate the payment processing of proposal submissions. I operate as the hub between DevHub admin, contributors and NF in communictions and problem solving as it relates to payments of completed work by the community.\nThe time I spend in this capacity involves weekly update meetings with DevHub admin, email and TG communications with NF legal, NF finance, DevHub admins and DevHub community contributors. We utilize database tools to manage the payment process.\nThe DevHub community members have benefited by receiving their proposal payments much more timely, recently narrowing in on 48 hr turn around and reducing the administrative steps each contributor had to complete to receive payment. The larger NEAR ecosystem has benefited by reducing the amount of time spent by the NF Legal and Finance team members, as well as DevHub admins, on the proposal payment process. These process improvements brought additional controls ensuring overall accuracy as DevHub has a fiduciary responsibility to NF that grant funds are spent responsibly, payments are correct and appropriate KYC/KYB measures have been taken.\nWeek 09/23/2024 - 09/27/2024\nExtra time was spent setting up testing invoicing for the Treasury system and communicating with teammembers on various hurdles and outcomes.\nTotal hours 15 @$80/hr\nRate determination is below the low end of the average hourly contract rate ($87 - $162) for an accountant with the same years of experience. That average rate does not account for my 5 years of blockchain specific experience. I'm not fulfilling all the duties that typically fall under this type of role, that is why I proposed a rate lower than the low end of the average. https://www.salary.com/research/salary/benchmark/controller-hourly-wages",
+    linked_proposals: [],
+    linked_rfp: null,
+    requested_sponsorship_usd_amount: 1200,
+    requested_sponsorship_paid_in_currency: "USDC",
+    requested_sponsor: "neardevdao.near",
+    receiver_account:
+      "9bd73bc9ba6ff289e2deb597dd6fe83dc0056af7a2937d6dec119660ba0146a4",
+    supervisor: "theori.near",
+    timeline:
+      '{"timeline_version":"V1","status":"APPROVED","sponsor_requested_review":true,"reviewer_completed_attestation":false,"kyc_verified":true}',
+    views: null,
+  },
+  {
+    proposal_id: 253,
+    block_height: 132779971,
+    ts: 1731707398226201000,
+    editor_id: "tfdevhub.near",
+    social_db_post_block_height: 132779971,
+    labels: [],
+    proposal_version: "V0",
+    proposal_body_version: "V2",
+    name: "DevHub Fin Ops 09/23/2024 - 09/27/2024",
+    category: "DevDAO Operations",
+    summary:
+      "Oversee that proposals flow thru the payment process efficiently. Continuously improve upon the process.",
+    description:
+      "I facilitate the payment processing of proposal submissions. I operate as the hub between DevHub admin, contributors and NF in communictions and problem solving as it relates to payments of completed work by the community.\nThe time I spend in this capacity involves weekly update meetings with DevHub admin, email and TG communications with NF legal, NF finance, DevHub admins and DevHub community contributors. We utilize database tools to manage the payment process.\nThe DevHub community members have benefited by receiving their proposal payments much more timely, recently narrowing in on 48 hr turn around and reducing the administrative steps each contributor had to complete to receive payment. The larger NEAR ecosystem has benefited by reducing the amount of time spent by the NF Legal and Finance team members, as well as DevHub admins, on the proposal payment process. These process improvements brought additional controls ensuring overall accuracy as DevHub has a fiduciary responsibility to NF that grant funds are spent responsibly, payments are correct and appropriate KYC/KYB measures have been taken.\nWeek 09/23/2024 - 09/27/2024\nExtra time was spent setting up testing invoicing for the Treasury system and communicating with teammembers on various hurdles and outcomes.\nTotal hours 15 @$80/hr\nRate determination is below the low end of the average hourly contract rate ($87 - $162) for an accountant with the same years of experience. That average rate does not account for my 5 years of blockchain specific experience. I'm not fulfilling all the duties that typically fall under this type of role, that is why I proposed a rate lower than the low end of the average. https://www.salary.com/research/salary/benchmark/controller-hourly-wages",
+    linked_proposals: [],
+    linked_rfp: null,
+    requested_sponsorship_usd_amount: 1200,
+    requested_sponsorship_paid_in_currency: "USDC",
+    requested_sponsor: "neardevdao.near",
+    receiver_account:
+      "9bd73bc9ba6ff289e2deb597dd6fe83dc0056af7a2937d6dec119660ba0146a4",
+    supervisor: "theori.near",
+    timeline:
+      '{"timeline_version":"V1","status":"PAYMENT_PROCESSING","sponsor_requested_review":true,"reviewer_completed_attestation":false,"kyc_verified":true,"kyc_verified_deprecated":false,"test_transaction_sent":true,"request_for_trustees_created":false}',
+    views: null,
+  },
+];
+
+// log the biggest timestamp ts
+console.log(
+  " Max ts: ",
+  response.reduce((max, r) => Math.max(max, r.ts), 0)
+);
+// log the biggest block height
+console.log(
+  " Max block height: ",
+  response.reduce((max, r) => Math.max(max, r.block_height), 0)
+);
+
+let smallest_ts = response.reduce((min, r) => Math.min(min, r.ts), Infinity);
+
+// log the ts sorted as dates
+console.log(
+  response
+    .sort((a, b) => a.ts - b.ts)
+    .map((r) => ({
+      ts: r.ts,
+      diff: r.ts - smallest_ts,
+      date: new Date(r.ts / 1000000).toISOString().slice(5, 19),
+    }))
+);
+
+// log the biggest timestamp ts
+console.log(
+  response.map((r) => ({
+    ts: r.ts,
+    timeline: r.timeline,
+    block_height: r.block_height,
+  }))
+);