diff --git a/block-server/handler.js b/block-server/handler.js index 79734640e..8bd8e476e 100644 --- a/block-server/handler.js +++ b/block-server/handler.js @@ -5,20 +5,27 @@ const S3= new AWS.S3(); const NETWORK = process.env.NETWORK || 'mainnet'; module.exports.block = async (event) => { - try { - // parse request params - const { block_height } = event.pathParameters; - const block = await fetchStreamerMessage(block_height); - return { - statusCode: 200, - body: JSON.stringify(block) - } - } catch (err) { - return { - statusCode: err.statusCode || 400, - body: err.message || JSON.stringify(err.message) - } + let headers = { + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Credentials": true, + }; + + try { + // parse request params + const { block_height } = event.pathParameters; + const block = await fetchStreamerMessage(block_height); + return { + statusCode: 200, + headers, + body: JSON.stringify(block) + } + } catch (err) { + return { + statusCode: err.statusCode || 400, + headers, + body: err.message || JSON.stringify(err.message) } + } }; const normalizeBlockHeight = function(block_height) { diff --git a/block-server/serverless.yml b/block-server/serverless.yml index 6cdb0b227..2d4040d98 100644 --- a/block-server/serverless.yml +++ b/block-server/serverless.yml @@ -46,6 +46,7 @@ functions: - httpApi: path: /block/{block_height} method: get + cors: true # Define function environment variables here # environment: diff --git a/frontend/Dockerfile b/frontend/Dockerfile index dab0b61ee..886283a68 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -3,13 +3,14 @@ WORKDIR ./ COPY package.json yarn.lock ./ RUN npm install + +FROM node:16-alpine as builder # Set build arguments and environment variables ARG NEXT_PUBLIC_REGISTRY_CONTRACT_ID ARG NEXT_PUBLIC_HASURA_ENDPOINT ENV NEXT_PUBLIC_HASURA_ENDPOINT=$NEXT_PUBLIC_HASURA_ENDPOINT ENV NEXT_PUBLIC_REGISTRY_CONTRACT_ID=$NEXT_PUBLIC_REGISTRY_CONTRACT_ID -FROM node:16-alpine as builder WORKDIR ./ COPY . . COPY --from=dependencies ./node_modules ./node_modules diff --git a/frontend/src/components/Editor/Editor.js b/frontend/src/components/Editor/Editor.js index dcd3a87af..c55c1d371 100644 --- a/frontend/src/components/Editor/Editor.js +++ b/frontend/src/components/Editor/Editor.js @@ -9,7 +9,7 @@ import { queryIndexerFunctionDetails } from "../../utils/queryIndexerFunction"; import { Alert } from "react-bootstrap"; import primitives from "!!raw-loader!../../../primitives.d.ts"; import { request, useInitialPayload } from "near-social-bridge"; -import Indexer from "../../utils/indexerRunner"; +import IndexerRunner from "../../utils/indexerRunner"; import { block_details } from "./block_details"; import ResizableLayoutEditor from "./ResizableLayoutEditor"; import { ResetChangesModal } from "../Modals/resetChanges"; @@ -17,8 +17,6 @@ import { FileSwitcher } from "./FileSwitcher"; import EditorButtons from "./EditorButtons"; import { PublishModal } from "../Modals/PublishModal"; const BLOCKHEIGHT_LIMIT = 3600; -const BLOCK_FETCHER_API = - "https://70jshyr5cb.execute-api.eu-central-1.amazonaws.com/block/"; const contractRegex = RegExp( "^(([a-zd]+[-_])*[a-zd]+.)*([a-zd]+[-_])*[a-zd]+$" @@ -38,16 +36,17 @@ const Editor = ({ const [originalSQLCode, setOriginalSQLCode] = useState(defaultSchema); const [originalIndexingCode, setOriginalIndexingCode] = useState(defaultCode); const [debugMode, setDebugMode] = useState(false); - const [logs, setLogs] = useState([]); const [heights, setHeights] = useState([]); const [showPublishModal, setShowPublishModal] = useState(false); const [debugModeInfoDisabled, setDebugModeInfoDisabled] = useState(false); - const handleLog = (blockHeight, log) => { - console.log(`Block #${blockHeight}: ${log}`); - setLogs((prevLogs) => [...prevLogs, log]); + const handleLog = (blockHeight, log, callback) => { + if(log) console.log(log); + if (callback) { + callback(); + } }; - const indexerRunner = new Indexer(handleLog); + const indexerRunner = new IndexerRunner(handleLog); const [indexingCode, setIndexingCode] = useState(defaultCode); const [schema, setSchema] = useState(defaultSchema); @@ -184,6 +183,9 @@ const Editor = ({ setSelectedOption("specificBlockHeight"); setBlockHeight(data.start_block_height); } + if(data.filter) { + setContractFilter(data.filter.matching_rule.affected_account_id) + } } catch (error) { console.log(error); setError(() => "An Error occured while trying to format the code."); @@ -269,7 +271,6 @@ const Editor = ({ const modifiedEditor = editor.getModifiedEditor(); modifiedEditor.onDidChangeModelContent((_) => { if (fileName == "indexingLogic.js") { - console.log("mountin"); setIndexingCode(modifiedEditor.getValue()); } if (fileName == "schema.sql") { @@ -300,28 +301,8 @@ const Editor = ({ } } - async function fetchBlockDetails(blockHeight) { - try { - const response = await fetch( - `${BLOCK_FETCHER_API}${String(blockHeight)}` - ); - const block_details = await response.json(); - return block_details; - } catch { - console.log(`Error Fetching Block Height details at ${blockHeight}`); - } - } - async function executeIndexerFunction() { - setLogs(() => []); - let innerCode = indexingCode.match(/getBlock\s*\([^)]*\)\s*{([\s\S]*)}/)[1]; - // for loop with await - for await (const height of heights) { - const block_details = await fetchBlockDetails(height); - if (block_details) { - await indexerRunner.runFunction(block_details, height, innerCode); - } - } + await indexerRunner.executeIndexerFunction(heights,indexingCode) } return ( @@ -371,12 +352,12 @@ const Editor = ({ handleOptionChange={handleOptionChange} blockHeight={blockHeight} setBlockHeight={setBlockHeight} - contractFilter={contractFilter} - handleSetContractFilter={handleSetContractFilter} - isContractFilterValid={isContractFilterValid} actionButtonText={getActionButtonText()} submit={submit} blockHeightError={blockHeightError} + contractFilter={contractFilter} + handleSetContractFilter={handleSetContractFilter} + isContractFilterValid={isContractFilterValid} />
diff --git a/frontend/src/components/Editor/EditorButtons.jsx b/frontend/src/components/Editor/EditorButtons.jsx index f4530b5ca..11066d3fc 100644 --- a/frontend/src/components/Editor/EditorButtons.jsx +++ b/frontend/src/components/Editor/EditorButtons.jsx @@ -43,6 +43,9 @@ const EditorButtons = ({ latestHeight, isUserIndexer, handleDeleteIndexer, + contractFilter, + handleSetContractFilter, + isContractFilterValid, }) => { const removeHeight = (index) => { setHeights(heights.filter((_, i) => i !== index)); @@ -59,7 +62,8 @@ const EditorButtons = ({ }} > - + + {accountId} @@ -78,7 +82,21 @@ const EditorButtons = ({ )} - + + Contract Filter + + + Please provide a valid contract name. + + + {debugMode && ( Please provide a valid contract name. diff --git a/frontend/src/components/Playground/graphiql.jsx b/frontend/src/components/Playground/graphiql.jsx index 3afa68d17..482edd7d3 100644 --- a/frontend/src/components/Playground/graphiql.jsx +++ b/frontend/src/components/Playground/graphiql.jsx @@ -7,7 +7,6 @@ const HASURA_ENDPOINT = "https://queryapi-hasura-graphql-24ktefolwq-ew.a.run.app/v1/graphql"; const graphQLFetcher = async (graphQLParams, accountId) => { - console.log(HASURA_ENDPOINT, "Hashura Endpoint"); const response = await fetch(HASURA_ENDPOINT, { method: "post", credentials: "omit", diff --git a/frontend/src/utils/fetchBlock.js b/frontend/src/utils/fetchBlock.js new file mode 100644 index 000000000..f773c109a --- /dev/null +++ b/frontend/src/utils/fetchBlock.js @@ -0,0 +1,14 @@ +const BLOCK_FETCHER_API = + "https://70jshyr5cb.execute-api.eu-central-1.amazonaws.com/block/"; + +export async function fetchBlockDetails(blockHeight) { + try { + const response = await fetch( + `${BLOCK_FETCHER_API}${String(blockHeight)}` + ); + const block_details = await response.json(); + return block_details; + } catch { + console.log(`Error Fetching Block Height details at ${blockHeight}`); + } + } diff --git a/frontend/src/utils/indexerRunner.js b/frontend/src/utils/indexerRunner.js index 05afb3e59..4007d56f5 100644 --- a/frontend/src/utils/indexerRunner.js +++ b/frontend/src/utils/indexerRunner.js @@ -1,10 +1,35 @@ import { Block } from "@near-lake/primitives"; import { Buffer } from "buffer"; +import {fetchBlockDetails} from "./fetchBlock"; + global.Buffer = Buffer; -export default class Indexer { +export default class IndexerRunner { constructor(handleLog) { this.handleLog = handleLog; } + + async executeIndexerFunction(heights, indexingCode) { + console.clear() + console.group('%c Welcome! Lets test your indexing logic on some Near Blocks!', 'color: white; background-color: navy; padding: 5px;'); + if(heights.length === 0) { + console.warn("No Block Heights Selected") + } + console.log("Note: GraphQL Mutations & Queries will not be executed on your database. They will simply return an empty object. Please keep this in mind as this may cause unintended behavior of your indexer function.") + let innerCode = indexingCode.match(/getBlock\s*\([^)]*\)\s*{([\s\S]*)}/)[1]; + // for loop with await + for await (const height of heights) { + console.group(`Block Height #${height}`) + const block_details = await fetchBlockDetails(height); + console.time('Indexing Execution Complete') + if (block_details) { + await this.runFunction(block_details, height, innerCode); + } + console.timeEnd('Indexing Execution Complete') + console.groupEnd() + } + console.groupEnd() + } + async runFunction(streamerMessage, blockHeight, indexerCode) { const innerCodeWithBlockHelper = ` @@ -25,21 +50,40 @@ export default class Indexer { // Define the custom context object const context = { - set: async () => { + set: async (key, value) => { this.handleLog( blockHeight, - "Context.set() is not supported in debug mode." + "", + () => { + console.group(`Setting Key/Value`); + console.log({key: value}); + console.groupEnd(); + } ); return {}; }, graphql: async (query, mutationData) => { this.handleLog( blockHeight, - "Context.graphql() is not supported in debug mode." - ); - this.handleLog( - blockHeight, - `mutationData: ${JSON.stringify(mutationData)}` + "", + () => { + let operationType, operationName + const match = query.match(/(query|mutation)\s+(\w+)\s*(\(.*?\))?\s*\{([\s\S]*)\}/); + if (match) { + operationType = match[1]; + operationName = match[2]; + } + + console.group(`Executing GraphQL ${operationType}: (${operationName})`); + if (operationType === 'mutation') console.log('%c Mutations in debug mode do not alter the database', 'color: black; background-color: yellow; padding: 5px;'); + console.group(`Data passed to ${operationType}`); + console.dir(mutationData); + console.groupEnd(); + console.group(`Data returned by ${operationType}`); + console.log({}) + console.groupEnd(); + console.groupEnd(); + } ); return {}; }, diff --git a/indexer-js-queue-handler/handler.js b/indexer-js-queue-handler/handler.js index 13e50179e..e508a3098 100644 --- a/indexer-js-queue-handler/handler.js +++ b/indexer-js-queue-handler/handler.js @@ -8,7 +8,6 @@ AWSXRay.captureAWS(AWS); export const consumer = async (event) => { const indexer = new Indexer('mainnet', 'eu-central-1'); - const results = []; for (const record of event.Records) { const jsonBody = JSON.parse(record.body); const block_height = jsonBody.block_height; @@ -19,7 +18,5 @@ export const consumer = async (event) => { functions[function_name] = function_config; const mutations = await indexer.runFunctions(block_height, functions, {imperative: true, provision: true}); - results.push(...mutations); } - return results; }; diff --git a/indexer-js-queue-handler/indexer.js b/indexer-js-queue-handler/indexer.js index feec98f8e..e2b84d5d8 100644 --- a/indexer-js-queue-handler/indexer.js +++ b/indexer-js-queue-handler/indexer.js @@ -43,7 +43,7 @@ export default class Indexer { simultaneousPromises.push(this.writeLog(function_name, block_height, 'Running function', function_name)); const hasuraRoleName = function_name.split('/')[0].replace(/[.-]/g, '_'); - const functionNameWithoutAccount = function_name.split('/')[1]; + const functionNameWithoutAccount = function_name.split('/')[1].replace(/[.-]/g, '_'); if (options.provision && !indexerFunction["provisioned"]) { const schemaName = `${function_name.replace(/[.\/-]/g, '_')}` diff --git a/indexer-js-queue-handler/serverless.yml b/indexer-js-queue-handler/serverless.yml index 6c87086a3..216583ffa 100644 --- a/indexer-js-queue-handler/serverless.yml +++ b/indexer-js-queue-handler/serverless.yml @@ -7,7 +7,7 @@ provider: name: aws runtime: nodejs16.x region: eu-central-1 - timeout: 60 + timeout: 15 environment: HASURA_ENDPOINT: ${env:HASURA_ENDPOINT} HASURA_ADMIN_SECRET: ${env:HASURA_ADMIN_SECRET} @@ -21,13 +21,13 @@ constructs: fifo: true worker: handler: handler.consumer - timeout: 60 # 6 minutes as lift multiplies this value by 6 (https://github.com/getlift/lift/blob/master/docs/queue.md#retry-delay) + timeout: 15 # 1.5 minutes as lift multiplies this value by 6 (https://github.com/getlift/lift/blob/master/docs/queue.md#retry-delay) startFromBlock-runner: type: queue fifo: true worker: handler: handler.consumer - timeout: 60 # 6 minutes as lift multiplies this value by 6 (https://github.com/getlift/lift/blob/master/docs/queue.md#retry-delay) + timeout: 15 # 1.5 minutes as lift multiplies this value by 6 (https://github.com/getlift/lift/blob/master/docs/queue.md#retry-delay) functions: diff --git a/indexer/Cargo.lock b/indexer/Cargo.lock index d894ebc0c..b0e37ad68 100644 --- a/indexer/Cargo.lock +++ b/indexer/Cargo.lock @@ -3370,6 +3370,7 @@ dependencies = [ "tokio-stream", "tracing", "tracing-subscriber 0.2.25", + "unescape", ] [[package]] @@ -4540,6 +4541,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "unescape" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccb97dac3243214f8d8507998906ca3e2e0b900bf9bf4870477f125b82e68f6e" + [[package]] name = "unicode-bidi" version = "0.3.13" diff --git a/indexer/queryapi_coordinator/Cargo.toml b/indexer/queryapi_coordinator/Cargo.toml index 55b407cb3..c3b526627 100644 --- a/indexer/queryapi_coordinator/Cargo.toml +++ b/indexer/queryapi_coordinator/Cargo.toml @@ -36,4 +36,5 @@ aws-types = "0.53.0" aws-credential-types = "0.53.0" aws-sdk-s3 = "0.23.0" aws-sdk-sqs = "0.23.0" -tracing-subscriber = "0.2.4" \ No newline at end of file +tracing-subscriber = "0.2.4" +unescape = "0.1.0" diff --git a/indexer/queryapi_coordinator/src/indexer_registry.rs b/indexer/queryapi_coordinator/src/indexer_registry.rs index 291291da1..868613917 100644 --- a/indexer/queryapi_coordinator/src/indexer_registry.rs +++ b/indexer/queryapi_coordinator/src/indexer_registry.rs @@ -10,6 +10,7 @@ use std::collections::hash_map::Entry; use std::collections::HashMap; use tokio::sync::MutexGuard; use tokio::task::JoinHandle; +use unescape::unescape; use crate::indexer_reducer; use crate::indexer_reducer::FunctionCallInfo; @@ -130,7 +131,6 @@ fn index_and_process_register_calls( let new_indexer_function = build_indexer_function_from_args( parse_indexer_function_args(&update), update.signer_id, - ®istry_calls_rule, ); match new_indexer_function { @@ -260,7 +260,6 @@ fn build_function_invocation_from_args( fn build_indexer_function_from_args( args: Option, signer_id: String, - indexer_rule: &IndexerRule, ) -> Option { match args { None => None, @@ -272,18 +271,44 @@ fn build_indexer_function_from_args( let function_name = args["function_name"].as_str(); match function_name { None => { - tracing::error!( + tracing::warn!( "Unable to parse function_name from indexer function: {:?}", &args ); return None; } - Some(function_name) => build_indexer_function( - &args, - function_name.to_string(), - account_id, - indexer_rule, - ), + Some(function_name) => { + match unescape(&args["filter_json"].to_string()) { + Some(filter_string) => { + let filter_json_strip_quotes = &filter_string[1..filter_string.len() - 1]; + match serde_json::from_str(&filter_json_strip_quotes) { + Ok(filter_json) => match serde_json::from_value(filter_json) { + Ok(indexer_rule) => build_indexer_function( + &args, + function_name.to_string(), + account_id, + &indexer_rule, + ), + Err(e) => { + tracing::warn!("Error parsing filter into indexer_rule for account {} function {}: {}, {}", account_id, function_name, e, filter_string); + None + } + }, + Err(e) => { + tracing::warn!("Error parsing indexer_rule filter for account {} function {}: {}, {}", account_id, function_name, e, filter_string); + None + } + } + }, + None => { + tracing::warn!( + "Unable to unescape filter_json from registration args: {:?}", + &args + ); + None + } + } + } } } } @@ -291,7 +316,6 @@ fn build_indexer_function_from_args( fn parse_indexer_function_args(update: &FunctionCallInfo) -> Option { if let Ok(mut args_json) = serde_json::from_str(&update.args) { - escape_json(&mut args_json); return Some(args_json); } else { tracing::error!( diff --git a/indexer/queryapi_coordinator/src/main.rs b/indexer/queryapi_coordinator/src/main.rs index d4b2e3da1..328598ba9 100644 --- a/indexer/queryapi_coordinator/src/main.rs +++ b/indexer/queryapi_coordinator/src/main.rs @@ -169,6 +169,14 @@ async fn handle_streamer_message( let mut indexer_function_messages: Vec = Vec::new(); for indexer_rule_match in indexer_rule_matches.iter() { + tracing::debug!( + target: INDEXER, + "Matched filter {:?} for function {} {}", + indexer_function.indexer_rule.matching_rule, + indexer_function.account_id, + indexer_function.function_name, + ); + let msg = IndexerQueueMessage { chain_id: indexer_rule_match.chain_id.clone(), indexer_rule_id: indexer_rule_match.indexer_rule_id.unwrap_or(0), @@ -189,11 +197,11 @@ async fn handle_streamer_message( stream::iter(indexer_function_messages.into_iter()) .chunks(10) - .for_each(|alert_queue_messages_batch| async { + .for_each(|indexer_queue_messages_batch| async { match opts::send_to_indexer_queue( context.queue_client, context.queue_url.to_string(), - alert_queue_messages_batch, + indexer_queue_messages_batch, ) .await { diff --git a/indexer/queryapi_coordinator/src/opts.rs b/indexer/queryapi_coordinator/src/opts.rs index f61ff941a..84c59c630 100644 --- a/indexer/queryapi_coordinator/src/opts.rs +++ b/indexer/queryapi_coordinator/src/opts.rs @@ -227,10 +227,6 @@ pub async fn send_to_indexer_queue( queue_url: String, indexer_queue_messages: Vec, ) -> anyhow::Result<()> { - tracing::info!( - target: "queryapi_coordinator", - "Sending indexer tasks to the queue: {queue_url}", - ); let message_bodies: Vec = indexer_queue_messages .into_iter()