Skip to content

Commit

Permalink
feat: Stream blocks while executing (#87)
Browse files Browse the repository at this point in the history
  • Loading branch information
@roshaans @gabehamilton
roshaans authored and gabehamilton committed Jun 26, 2023
1 parent 3268dfa commit 4dd1935
Show file tree
Hide file tree
Showing 5 changed files with 181 additions and 77 deletions.
79 changes: 65 additions & 14 deletions frontend/src/components/Editor/BlockPicker.jsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,19 @@ import {
Badge,
InputGroup,
FormControl,
Dropdown,
ButtonGroup,
} from "react-bootstrap";

import { Play, Plus } from "react-bootstrap-icons";
import { Play, Plus, Stop } from "react-bootstrap-icons";

export const BlockPicker = ({
heights = [],
setHeights,
executeIndexerFunction,
latestHeight,
isExecuting,
stopExecution,
}) => {
const [inputValue, setInputValue] = useState(String(latestHeight));

Expand All @@ -37,24 +41,71 @@ export const BlockPicker = ({
value={inputValue}
onChange={(e) => setInputValue(e.target.value)}
/>
<Button variant="outline-secondary" onClick={addHeight}>
<Plus size={24} style={{ cursor: "pointer" }} />
</Button>
<OverlayTrigger
{isExecuting === true &&
<OverlayTrigger
placement="bottom"
overlay={<Tooltip>Stop Indexer Execution</Tooltip>}
>
<Button variant="outline-secondary" onClick={() => stopExecution()}>
<Stop size={24} style={{ cursor: "pointer" }} />
</Button>
</OverlayTrigger>
}
{!isExecuting && (<> <OverlayTrigger
placement="bottom"
overlay={<Tooltip>Test Indexer Function In Browser</Tooltip>}
overlay={<Tooltip>Add Block To Debug List</Tooltip>}
>
<Button
className="mx-2"
size="sm"
variant="primary"
onClick={() => executeIndexerFunction()}
>
<Play size={24} />
<Button variant="outline-secondary" onClick={addHeight}>
<Plus size={24} style={{ cursor: "pointer" }} />
</Button>
</OverlayTrigger>
<Dropdown as={ButtonGroup}>
<OverlayTrigger
placement="bottom"
overlay={<Tooltip>
{
(() => {
if (heights.length > 0) {
return "Test Indexer Function With Debug List"
} else if (inputValue) {
return "Test Indexer Function With Specific Block"
} else {
return "Follow the Tip of the Network"
}
})()
}
</Tooltip>}
>

<Button
size="sm"
variant="primary"
onClick={() => {
if (heights.length > 0) {
executeIndexerFunction("debugList")
} else if (inputValue) {
executeIndexerFunction("specific", inputValue)
} else {
executeIndexerFunction("latest")
}
}
}
>
<Play size={24} />
</Button>
</OverlayTrigger>
<Dropdown.Toggle split variant="primary" id="dropdown-split-basic" />
<Dropdown.Menu>
<Dropdown.Item onClick={() => executeIndexerFunction("latest")}>Follow The Network</Dropdown.Item>
<Dropdown.Item disabled={heights.length === 0} onClick={() => executeIndexerFunction("debugList")}>Execute From Debug List</Dropdown.Item>
</Dropdown.Menu>
</Dropdown>
</>)
}


</InputGroup>
</div>
</div>
</div >
);
};
35 changes: 31 additions & 4 deletions frontend/src/components/Editor/Editor.js
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import React, { useEffect, useState, useCallback } from "react";
import React, { useEffect, useState, useCallback, useMemo } from "react";
import {
formatSQL,
formatIndexingCode,
Expand All @@ -16,6 +16,7 @@ import { ResetChangesModal } from "../Modals/resetChanges";
import { FileSwitcher } from "./FileSwitcher";
import EditorButtons from "./EditorButtons";
import { PublishModal } from "../Modals/PublishModal";
import {getLatestBlockHeight} from "../../utils/getLatestBlockHeight";
const BLOCKHEIGHT_LIMIT = 3600;

const contractRegex = RegExp(
Expand Down Expand Up @@ -46,7 +47,7 @@ const Editor = ({
}
};

const indexerRunner = new IndexerRunner(handleLog);
const indexerRunner = useMemo(() => new IndexerRunner(handleLog), []);

const [indexingCode, setIndexingCode] = useState(defaultCode);
const [schema, setSchema] = useState(defaultSchema);
Expand All @@ -59,6 +60,12 @@ const Editor = ({
const [isContractFilterValid, setIsContractFilterValid] = useState(true);
const [contractFilter, setContractFilter] = useState("social.near");
const { height, selectedTab, currentUserAccountId } = useInitialPayload();
const [isExecutingIndexerFunction, setIsExecutingIndexerFunction] = useState(false)

const requestLatestBlockHeight = async () => {
const blockHeight = getLatestBlockHeight()
return blockHeight
}

const handleOptionChange = (event) => {
setSelectedOption(event.target.value);
Expand Down Expand Up @@ -301,8 +308,26 @@ const Editor = ({
}
}

async function executeIndexerFunction() {
await indexerRunner.executeIndexerFunction(heights,indexingCode)
async function executeIndexerFunction(option = "latest", startingBlockHeight = null) {
setIsExecutingIndexerFunction(() => true)

switch (option) {
case "debugList":
await indexerRunner.executeIndexerFunctionOnHeights(heights, indexingCode, option)
break
case "specific":
if (startingBlockHeight === null && Number(startingBlockHeight) === 0) {
console.log("Invalid Starting Block Height: starting block height is null or 0")
break
}

await indexerRunner.start(startingBlockHeight, indexingCode, option)
break
case "latest":
const latestHeight = await requestLatestBlockHeight()
if (latestHeight) await indexerRunner.start(latestHeight - 10, indexingCode, option)
}
setIsExecutingIndexerFunction(() => false)
}

return (
Expand Down Expand Up @@ -331,6 +356,8 @@ const Editor = ({
debugMode={debugMode}
heights={heights}
setHeights={setHeights}
isExecuting={isExecutingIndexerFunction}
stopExecution={() => indexerRunner.stop()}
contractFilter={contractFilter}
handleSetContractFilter={handleSetContractFilter}
isContractFilterValid={isContractFilterValid}
Expand Down
4 changes: 4 additions & 0 deletions frontend/src/components/Editor/EditorButtons.jsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ const EditorButtons = ({
getActionButtonText,
submit,
debugMode,
isExecuting,
stopExecution,
heights,
setHeights,
setShowPublishModal,
Expand Down Expand Up @@ -104,6 +106,8 @@ const EditorButtons = ({
setHeights={setHeights}
executeIndexerFunction={executeIndexerFunction}
latestHeight={latestHeight}
isExecuting={isExecuting}
stopExecution={stopExecution}
/>
)}
</Col>
Expand Down
23 changes: 14 additions & 9 deletions frontend/src/utils/fetchBlock.js
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,19 @@
const BLOCK_FETCHER_API =
"https://70jshyr5cb.execute-api.eu-central-1.amazonaws.com/block/";

const GENESIS_BLOCK_HEIGHT = 52945886;
export async function fetchBlockDetails(blockHeight) {
try {
const response = await fetch(
`${BLOCK_FETCHER_API}${String(blockHeight)}`
);
const block_details = await response.json();
return block_details;
} catch {
console.log(`Error Fetching Block Height details at ${blockHeight}`);
}
if (blockHeight <= GENESIS_BLOCK_HEIGHT) {
throw new Error(`Block Height must be greater than genesis block height #${GENESIS_BLOCK_HEIGHT}`);
}
try {
const response = await fetch(
`${BLOCK_FETCHER_API}${String(blockHeight)}`
);
const block_details = await response.json();
return block_details;
} catch {
// console.log(`Error Fetching Block Height details at ${blockHeight}`);
throw new Error(`Error Fetching Block Height details at BlockHeight #${blockHeight}`);
}
}
117 changes: 67 additions & 50 deletions frontend/src/utils/indexerRunner.js
View file
Original file line number Diff line number Diff line change
@@ -1,29 +1,86 @@
import { Block } from "@near-lake/primitives";
import { Buffer } from "buffer";
import {fetchBlockDetails} from "./fetchBlock";
import { fetchBlockDetails } from "./fetchBlock";

global.Buffer = Buffer;
export default class IndexerRunner {
constructor(handleLog) {
this.handleLog = handleLog;
this.currentHeight = 0;
this.shouldStop = false;
}

async executeIndexerFunction(heights, indexingCode) {
async start(startingHeight, indexingCode, option) {
this.currentHeight = startingHeight;
this.shouldStop = false;
console.clear()
console.group('%c Welcome! Lets test your indexing logic on some Near Blocks!', 'color: white; background-color: navy; padding: 5px;');
if(heights.length === 0) {
if (option == "specific" && !Number(startingHeight)) {
console.log("No Start Block Height Provided to Stream Blocks From")
this.stop()
console.groupEnd()
return
}
console.log(`Streaming Blocks Starting from ${option} Block #${this.currentHeight}`)
while (!this.shouldStop) {
console.group(`Block Height #${this.currentHeight}`)
let blockDetails;
try {
blockDetails = await fetchBlockDetails(this.currentHeight);
} catch (error) {
console.log(error)
this.stop()
}
if (blockDetails) {
await this.executeIndexerFunction(this.currentHeight, blockDetails, indexingCode);
this.currentHeight++;
await this.delay(1000);
}
console.groupEnd()

}
}

stop() {
this.shouldStop = true;
console.log("%c Stopping Block Processing", 'color: white; background-color: red; padding: 5px;')
}

delay(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}

async executeIndexerFunction(height, blockDetails, indexingCode) {
let innerCode = indexingCode.match(/getBlock\s*\([^)]*\)\s*{([\s\S]*)}/)[1];
if (blockDetails) {
const block = Block.fromStreamerMessage(blockDetails);
block.actions()
block.receipts()
block.events()

console.log(block)
await this.runFunction(blockDetails, height, innerCode);
}
}

async executeIndexerFunctionOnHeights(heights, indexingCode) {
console.clear()
console.group('%c Welcome! Lets test your indexing logic on some Near Blocks!', 'color: white; background-color: navy; padding: 5px;');
if (heights.length === 0) {
console.warn("No Block Heights Selected")
return
}
console.log("Note: GraphQL Mutations & Queries will not be executed on your database. They will simply return an empty object. Please keep this in mind as this may cause unintended behavior of your indexer function.")
let innerCode = indexingCode.match(/getBlock\s*\([^)]*\)\s*{([\s\S]*)}/)[1];
// for loop with await
for await (const height of heights) {
console.group(`Block Height #${height}`)
const block_details = await fetchBlockDetails(height);
console.time('Indexing Execution Complete')
if (block_details) {
await this.runFunction(block_details, height, innerCode);
let blockDetails;
try {
blockDetails = await fetchBlockDetails(height);
} catch (error) {
console.log(error)
}
console.time('Indexing Execution Complete')
this.executeIndexerFunction(height, blockDetails, indexingCode)
console.timeEnd('Indexing Execution Complete')
console.groupEnd()
}
Expand Down Expand Up @@ -56,7 +113,7 @@ export default class IndexerRunner {
"",
() => {
console.group(`Setting Key/Value`);
console.log({key: value});
console.log({[key]: value});
console.groupEnd();
}
);
Expand Down Expand Up @@ -92,7 +149,6 @@ export default class IndexerRunner {
},
};

// Call the wrapped function, passing the imported Block and streamerMessage
wrappedFunction(Block, streamerMessage, context);
}

Expand All @@ -117,45 +173,6 @@ export default class IndexerRunner {
);
}

// async runGraphQLQuery(
// operation,
// variables,
// function_name,
// block_height,
// hasuraRoleName,
// logError = true
// ) {
// const response = await this.deps.fetch(
// `${process.env.HASURA_ENDPOINT}/v1/graphql`,
// {
// method: "POST",
// headers: {
// "Content-Type": "application/json",
// ...(hasuraRoleName && { "X-Hasura-Role": hasuraRoleName }),
// },
// body: JSON.stringify({
// query: operation,
// ...(variables && { variables }),
// }),
// }
// );
//
// const { data, errors } = await response.json();
//
// if (response.status !== 200 || errors) {
// if (logError) {
// }
// throw new Error(
// `Failed to write graphql, http status: ${
// response.status
// }, errors: ${JSON.stringify(errors, null, 2)}`
// );
// }
//
// return data;
// }
//

renameUnderscoreFieldsToCamelCase(value) {
if (value && typeof value === "object" && !Array.isArray(value)) {
// It's a non-null, non-array object, create a replacement with the keys initially-capped
Expand Down

0 comments on commit 4dd1935

Please sign in to comment.