From f81fe904e71f4fe44048fa8f9d42dc73eab04f7d Mon Sep 17 00:00:00 2001 From: evantahler Date: Mon, 16 May 2022 16:42:20 -0700 Subject: [PATCH 01/20] Display addtional failure information when sync is expanded --- airbyte-api/src/main/openapi/config.yaml | 2 + .../server/converters/JobConverter.java | 1 + .../src/components/JobItem/JobItem.tsx | 8 +++- .../JobItem/components/AttemptDetails.tsx | 4 +- .../JobItem/components/ErrorDetails.tsx | 47 +++++++++++++++++++ airbyte-webapp/src/locales/en.json | 1 + .../api/generated-api-html/index.html | 29 ++++++++++++ 7 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 airbyte-webapp/src/components/JobItem/components/ErrorDetails.tsx diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index 26adc3dfd97c..1b760d05648e 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -3844,6 +3844,8 @@ components: $ref: "#/components/schemas/AttemptFailureType" externalMessage: type: string + internalMessage: + type: string stacktrace: type: string retryable: diff --git a/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java b/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java index f42b95ca66e9..51c329881ea7 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java +++ b/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java @@ -166,6 +166,7 @@ private static AttemptFailureSummary getAttemptFailureSummary(final Attempt atte .failureOrigin(Enums.convertTo(failure.getFailureOrigin(), AttemptFailureOrigin.class)) .failureType(Enums.convertTo(failure.getFailureType(), AttemptFailureType.class)) .externalMessage(failure.getExternalMessage()) + .internalMessage(failure.getInternalMessage()) .stacktrace(failure.getStacktrace()) .timestamp(failure.getTimestamp()) .retryable(failure.getRetryable())) diff --git a/airbyte-webapp/src/components/JobItem/JobItem.tsx b/airbyte-webapp/src/components/JobItem/JobItem.tsx index 2b494a516f7d..35223b2757a4 100644 --- a/airbyte-webapp/src/components/JobItem/JobItem.tsx +++ b/airbyte-webapp/src/components/JobItem/JobItem.tsx @@ -10,6 +10,7 @@ import { JobsWithJobs } from "pages/ConnectionPage/pages/ConnectionItemPage/comp import { AttemptRead, JobStatus } from "../../core/request/AirbyteClient"; import { useAttemptLink } from "./attemptLinkUtils"; import ContentWrapper from "./components/ContentWrapper"; +import ErrorDetails from "./components/ErrorDetails"; import JobLogs from "./components/JobLogs"; import MainInfo from "./components/MainInfo"; @@ -86,7 +87,12 @@ export const JobItem: React.FC = ({ shortInfo, job }) => { } > - {isOpen && } + {isOpen ? ( + <> + + + + ) : null} diff --git a/airbyte-webapp/src/components/JobItem/components/AttemptDetails.tsx b/airbyte-webapp/src/components/JobItem/components/AttemptDetails.tsx index 036bf2c8b506..e77adf1a8669 100644 --- a/airbyte-webapp/src/components/JobItem/components/AttemptDetails.tsx +++ b/airbyte-webapp/src/components/JobItem/components/AttemptDetails.tsx @@ -60,7 +60,7 @@ const AttemptDetails: React.FC = ({ attempt, className, configType }) => })}: ${failureOrigin}`; }; - const getFailureMessage = (attempt: AttemptRead) => { + const getExternalFailureMessage = (attempt: AttemptRead) => { const failure = getFailureFromAttempt(attempt); const failureMessage = failure?.externalMessage ?? formatMessage({ id: "errorView.unknown" }); @@ -114,7 +114,7 @@ const AttemptDetails: React.FC = ({ attempt, className, configType }) => }, { key: getFailureOrigin(attempt), - value: getFailureMessage(attempt), + value: getExternalFailureMessage(attempt), } )} diff --git a/airbyte-webapp/src/components/JobItem/components/ErrorDetails.tsx b/airbyte-webapp/src/components/JobItem/components/ErrorDetails.tsx new file mode 100644 index 000000000000..a521b2da70d5 --- /dev/null +++ b/airbyte-webapp/src/components/JobItem/components/ErrorDetails.tsx @@ -0,0 +1,47 @@ +import { useIntl } from "react-intl"; +import styled from "styled-components"; + +import { Attempt, Failure } from "core/domain/job/Job"; + +type IProps = { + attempts?: Attempt[]; +}; + +const ExpandedErrorContainer = styled.div` + font-size: 12px; + line-height: 15px; + padding: 10px; + padding-left: 40px; + color: ${({ theme }) => theme.greyColor40}; +`; + +const getFailureFromAttempt = (attempt: Attempt) => { + return attempt.failureSummary && attempt.failureSummary.failures[0]; +}; + +const ErrorDetails: React.FC = ({ attempts }) => { + const { formatMessage } = useIntl(); + + if (!attempts || attempts.length === 0) { + return null; + } + + const getInternalFailureMessage = (failure: Failure) => { + const failureMessage = failure?.internalMessage ?? formatMessage({ id: "errorView.unknown" }); + return `${formatMessage({ + id: "sources.additionalFailureInfo", + })}: ${failureMessage}`; + }; + + const attempt = attempts[attempts.length - 1]; + const failure = getFailureFromAttempt(attempt); + + if (!failure) { + return null; + } + + const internalMessage = getInternalFailureMessage(failure); + return {internalMessage}; +}; + +export default ErrorDetails; diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 1d9fa12aadda..94b9f882ae52 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -291,6 +291,7 @@ "sources.copied": "Copied", "sources.failureOrigin": "Failure Origin", "sources.message": "Message", + "sources.additionalFailureInfo": "Additional Failure Information", "sources.lastAttempt": "Last attempt:", "destination.destinationSettings": "Destination Settings", diff --git a/docs/reference/api/generated-api-html/index.html b/docs/reference/api/generated-api-html/index.html index f6166403d80f..25fcbf390883 100644 --- a/docs/reference/api/generated-api-html/index.html +++ b/docs/reference/api/generated-api-html/index.html @@ -1079,11 +1079,13 @@

Example data

"failures" : [ { "retryable" : true, "stacktrace" : "stacktrace", + "internalMessage" : "internalMessage", "externalMessage" : "externalMessage", "timestamp" : 1 }, { "retryable" : true, "stacktrace" : "stacktrace", + "internalMessage" : "internalMessage", "externalMessage" : "externalMessage", "timestamp" : 1 } ], @@ -1128,11 +1130,13 @@

Example data

"failures" : [ { "retryable" : true, "stacktrace" : "stacktrace", + "internalMessage" : "internalMessage", "externalMessage" : "externalMessage", "timestamp" : 1 }, { "retryable" : true, "stacktrace" : "stacktrace", + "internalMessage" : "internalMessage", "externalMessage" : "externalMessage", "timestamp" : 1 } ], @@ -1400,11 +1404,13 @@

Example data

"failures" : [ { "retryable" : true, "stacktrace" : "stacktrace", + "internalMessage" : "internalMessage", "externalMessage" : "externalMessage", "timestamp" : 1 }, { "retryable" : true, "stacktrace" : "stacktrace", + "internalMessage" : "internalMessage", "externalMessage" : "externalMessage", "timestamp" : 1 } ], @@ -1449,11 +1455,13 @@

Example data

"failures" : [ { "retryable" : true, "stacktrace" : "stacktrace", + "internalMessage" : "internalMessage", "externalMessage" : "externalMessage", "timestamp" : 1 }, { "retryable" : true, "stacktrace" : "stacktrace", + "internalMessage" : "internalMessage", "externalMessage" : "externalMessage", "timestamp" : 1 } ], @@ -3972,11 +3980,13 @@

Example data

"failures" : [ { "retryable" : true, "stacktrace" : "stacktrace", + "internalMessage" : "internalMessage", "externalMessage" : "externalMessage", "timestamp" : 1 }, { "retryable" : true, "stacktrace" : "stacktrace", + "internalMessage" : "internalMessage", "externalMessage" : "externalMessage", "timestamp" : 1 } ], @@ -4021,11 +4031,13 @@

Example data

"failures" : [ { "retryable" : true, "stacktrace" : "stacktrace", + "internalMessage" : "internalMessage", "externalMessage" : "externalMessage", "timestamp" : 1 }, { "retryable" : true, "stacktrace" : "stacktrace", + "internalMessage" : "internalMessage", "externalMessage" : "externalMessage", "timestamp" : 1 } ], @@ -4197,11 +4209,13 @@

Example data

"failures" : [ { "retryable" : true, "stacktrace" : "stacktrace", + "internalMessage" : "internalMessage", "externalMessage" : "externalMessage", "timestamp" : 1 }, { "retryable" : true, "stacktrace" : "stacktrace", + "internalMessage" : "internalMessage", "externalMessage" : "externalMessage", "timestamp" : 1 } ], @@ -4246,11 +4260,13 @@

Example data

"failures" : [ { "retryable" : true, "stacktrace" : "stacktrace", + "internalMessage" : "internalMessage", "externalMessage" : "externalMessage", "timestamp" : 1 }, { "retryable" : true, "stacktrace" : "stacktrace", + "internalMessage" : "internalMessage", "externalMessage" : "externalMessage", "timestamp" : 1 } ], @@ -4359,11 +4375,13 @@

Example data

"failures" : [ { "retryable" : true, "stacktrace" : "stacktrace", + "internalMessage" : "internalMessage", "externalMessage" : "externalMessage", "timestamp" : 1 }, { "retryable" : true, "stacktrace" : "stacktrace", + "internalMessage" : "internalMessage", "externalMessage" : "externalMessage", "timestamp" : 1 } ], @@ -4408,11 +4426,13 @@

Example data

"failures" : [ { "retryable" : true, "stacktrace" : "stacktrace", + "internalMessage" : "internalMessage", "externalMessage" : "externalMessage", "timestamp" : 1 }, { "retryable" : true, "stacktrace" : "stacktrace", + "internalMessage" : "internalMessage", "externalMessage" : "externalMessage", "timestamp" : 1 } ], @@ -4521,11 +4541,13 @@

Example data

"failures" : [ { "retryable" : true, "stacktrace" : "stacktrace", + "internalMessage" : "internalMessage", "externalMessage" : "externalMessage", "timestamp" : 1 }, { "retryable" : true, "stacktrace" : "stacktrace", + "internalMessage" : "internalMessage", "externalMessage" : "externalMessage", "timestamp" : 1 } ], @@ -4565,11 +4587,13 @@

Example data

"failures" : [ { "retryable" : true, "stacktrace" : "stacktrace", + "internalMessage" : "internalMessage", "externalMessage" : "externalMessage", "timestamp" : 1 }, { "retryable" : true, "stacktrace" : "stacktrace", + "internalMessage" : "internalMessage", "externalMessage" : "externalMessage", "timestamp" : 1 } ], @@ -4617,11 +4641,13 @@

Example data

"failures" : [ { "retryable" : true, "stacktrace" : "stacktrace", + "internalMessage" : "internalMessage", "externalMessage" : "externalMessage", "timestamp" : 1 }, { "retryable" : true, "stacktrace" : "stacktrace", + "internalMessage" : "internalMessage", "externalMessage" : "externalMessage", "timestamp" : 1 } ], @@ -4661,11 +4687,13 @@

Example data

"failures" : [ { "retryable" : true, "stacktrace" : "stacktrace", + "internalMessage" : "internalMessage", "externalMessage" : "externalMessage", "timestamp" : 1 }, { "retryable" : true, "stacktrace" : "stacktrace", + "internalMessage" : "internalMessage", "externalMessage" : "externalMessage", "timestamp" : 1 } ], @@ -9957,6 +9985,7 @@

AttemptFailureReason - failureOrigin (optional)
failureType (optional)
externalMessage (optional)
+
internalMessage (optional)
stacktrace (optional)
retryable (optional)
Boolean True if it is known that retrying may succeed, e.g. for a transient failure. False if it is known that a retry will not succeed, e.g. for a configuration issue. If not set, retryable status is not well known.
timestamp
Long format: int64
From 9b818bcc7c572e2fe4c13a39804126d48c73b673 Mon Sep 17 00:00:00 2001 From: evantahler Date: Mon, 16 May 2022 16:44:08 -0700 Subject: [PATCH 02/20] rename --- airbyte-webapp/src/components/JobItem/JobItem.tsx | 2 +- .../components/{ErrorDetails.tsx => FailureDetails.tsx} | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename airbyte-webapp/src/components/JobItem/components/{ErrorDetails.tsx => FailureDetails.tsx} (89%) diff --git a/airbyte-webapp/src/components/JobItem/JobItem.tsx b/airbyte-webapp/src/components/JobItem/JobItem.tsx index 35223b2757a4..1eb0d9c8cfb2 100644 --- a/airbyte-webapp/src/components/JobItem/JobItem.tsx +++ b/airbyte-webapp/src/components/JobItem/JobItem.tsx @@ -10,7 +10,7 @@ import { JobsWithJobs } from "pages/ConnectionPage/pages/ConnectionItemPage/comp import { AttemptRead, JobStatus } from "../../core/request/AirbyteClient"; import { useAttemptLink } from "./attemptLinkUtils"; import ContentWrapper from "./components/ContentWrapper"; -import ErrorDetails from "./components/ErrorDetails"; +import ErrorDetails from "./components/FailureDetails"; import JobLogs from "./components/JobLogs"; import MainInfo from "./components/MainInfo"; diff --git a/airbyte-webapp/src/components/JobItem/components/ErrorDetails.tsx b/airbyte-webapp/src/components/JobItem/components/FailureDetails.tsx similarity index 89% rename from airbyte-webapp/src/components/JobItem/components/ErrorDetails.tsx rename to airbyte-webapp/src/components/JobItem/components/FailureDetails.tsx index a521b2da70d5..f3f0c6dcd45f 100644 --- a/airbyte-webapp/src/components/JobItem/components/ErrorDetails.tsx +++ b/airbyte-webapp/src/components/JobItem/components/FailureDetails.tsx @@ -7,7 +7,7 @@ type IProps = { attempts?: Attempt[]; }; -const ExpandedErrorContainer = styled.div` +const ExpandedFailureContainer = styled.div` font-size: 12px; line-height: 15px; padding: 10px; @@ -41,7 +41,7 @@ const ErrorDetails: React.FC = ({ attempts }) => { } const internalMessage = getInternalFailureMessage(failure); - return {internalMessage}; + return {internalMessage}; }; export default ErrorDetails; From e13cedf6164211964ae556bbb322c2fcd0541fa2 Mon Sep 17 00:00:00 2001 From: Evan Tahler Date: Tue, 17 May 2022 08:58:34 -0700 Subject: [PATCH 03/20] Update airbyte-webapp/src/components/JobItem/components/FailureDetails.tsx Co-authored-by: Tim Roes --- .../src/components/JobItem/components/FailureDetails.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-webapp/src/components/JobItem/components/FailureDetails.tsx b/airbyte-webapp/src/components/JobItem/components/FailureDetails.tsx index f3f0c6dcd45f..1bc2a71762eb 100644 --- a/airbyte-webapp/src/components/JobItem/components/FailureDetails.tsx +++ b/airbyte-webapp/src/components/JobItem/components/FailureDetails.tsx @@ -16,7 +16,7 @@ const ExpandedFailureContainer = styled.div` `; const getFailureFromAttempt = (attempt: Attempt) => { - return attempt.failureSummary && attempt.failureSummary.failures[0]; + return attempt.failureSummary?.failureSummary.failures[0]; }; const ErrorDetails: React.FC = ({ attempts }) => { From 72604f69d8f5ffdb29d9645ade8e8c0b121c1a37 Mon Sep 17 00:00:00 2001 From: evantahler Date: Tue, 17 May 2022 15:35:32 -0700 Subject: [PATCH 04/20] fix bad merge --- .../src/components/JobItem/components/FailureDetails.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-webapp/src/components/JobItem/components/FailureDetails.tsx b/airbyte-webapp/src/components/JobItem/components/FailureDetails.tsx index 1bc2a71762eb..061fd28405d1 100644 --- a/airbyte-webapp/src/components/JobItem/components/FailureDetails.tsx +++ b/airbyte-webapp/src/components/JobItem/components/FailureDetails.tsx @@ -16,7 +16,7 @@ const ExpandedFailureContainer = styled.div` `; const getFailureFromAttempt = (attempt: Attempt) => { - return attempt.failureSummary?.failureSummary.failures[0]; + return attempt.failureSummary?.failures[0]; }; const ErrorDetails: React.FC = ({ attempts }) => { From 873a6b803f7f0667e44795655bff519a87b33afc Mon Sep 17 00:00:00 2001 From: evantahler Date: Wed, 18 May 2022 14:28:34 -0700 Subject: [PATCH 05/20] jump to timestamp --- .../src/components/JobItem/JobItem.tsx | 8 +++- .../JobItem/components/FailureDetails.tsx | 24 ++++++++++- .../components/JobItem/components/JobLogs.tsx | 4 +- .../components/JobItem/components/Logs.tsx | 40 ++++++++++++++++++- .../JobItem/components/LogsDetails.tsx | 5 ++- 5 files changed, 72 insertions(+), 9 deletions(-) diff --git a/airbyte-webapp/src/components/JobItem/JobItem.tsx b/airbyte-webapp/src/components/JobItem/JobItem.tsx index 1eb0d9c8cfb2..af4eea0292cf 100644 --- a/airbyte-webapp/src/components/JobItem/JobItem.tsx +++ b/airbyte-webapp/src/components/JobItem/JobItem.tsx @@ -56,6 +56,7 @@ export const JobItem: React.FC = ({ shortInfo, job }) => { const [isOpen, setIsOpen] = useState(linkedJobId === getJobId(job)); const onExpand = () => setIsOpen(!isOpen); const scrollAnchor = useRef(null); + const [logTimestamp, setLogTimestamp] = useState(); const didSucceed = didJobSucceed(job); @@ -89,8 +90,11 @@ export const JobItem: React.FC = ({ shortInfo, job }) => { > {isOpen ? ( <> - - + + ) : null} diff --git a/airbyte-webapp/src/components/JobItem/components/FailureDetails.tsx b/airbyte-webapp/src/components/JobItem/components/FailureDetails.tsx index 061fd28405d1..0f06173381b7 100644 --- a/airbyte-webapp/src/components/JobItem/components/FailureDetails.tsx +++ b/airbyte-webapp/src/components/JobItem/components/FailureDetails.tsx @@ -1,10 +1,13 @@ import { useIntl } from "react-intl"; import styled from "styled-components"; +import { Button } from "components/base"; + import { Attempt, Failure } from "core/domain/job/Job"; type IProps = { attempts?: Attempt[]; + setLogTimestamp: (t: number) => void; }; const ExpandedFailureContainer = styled.div` @@ -19,7 +22,7 @@ const getFailureFromAttempt = (attempt: Attempt) => { return attempt.failureSummary?.failures[0]; }; -const ErrorDetails: React.FC = ({ attempts }) => { +const ErrorDetails: React.FC = ({ attempts, setLogTimestamp }) => { const { formatMessage } = useIntl(); if (!attempts || attempts.length === 0) { @@ -40,8 +43,25 @@ const ErrorDetails: React.FC = ({ attempts }) => { return null; } + const jumpToLogTimestamp = () => { + if (failure.timestamp) { + setLogTimestamp(failure.timestamp); + } + }; + const internalMessage = getInternalFailureMessage(failure); - return {internalMessage}; + return ( + + {failure.timestamp ? ( + <> + {" "} + + ) : null} + {internalMessage} + + ); }; export default ErrorDetails; diff --git a/airbyte-webapp/src/components/JobItem/components/JobLogs.tsx b/airbyte-webapp/src/components/JobItem/components/JobLogs.tsx index 8df8ecbdc5d1..d6a29f33784c 100644 --- a/airbyte-webapp/src/components/JobItem/components/JobLogs.tsx +++ b/airbyte-webapp/src/components/JobItem/components/JobLogs.tsx @@ -16,6 +16,7 @@ import Tabs, { TabsData } from "./Tabs"; type JobLogsProps = { jobIsFailed?: boolean; job: SynchronousJobReadWithStatus | JobsWithJobs; + logTimestamp?: number; }; const isPartialSuccess = (attempt: AttemptRead) => { @@ -28,7 +29,7 @@ const jobIsSynchronousJobRead = ( return !!(job as SynchronousJobReadWithStatus)?.logs?.logLines; }; -const JobLogs: React.FC = ({ jobIsFailed, job }) => { +const JobLogs: React.FC = ({ jobIsFailed, job, logTimestamp }) => { const isSynchronousJobRead = jobIsSynchronousJobRead(job); const id: number | string = (job as JobsWithJobs).job?.id ?? (job as SynchronousJobReadWithStatus).id; @@ -85,6 +86,7 @@ const JobLogs: React.FC = ({ jobIsFailed, job }) => { jobDebugInfo={debugInfo} showAttemptStats={attempts > 1} logs={debugInfo?.attempts[attemptNumber].logs.logLines} + logTimestamp={logTimestamp} /> ); diff --git a/airbyte-webapp/src/components/JobItem/components/Logs.tsx b/airbyte-webapp/src/components/JobItem/components/Logs.tsx index a602ebadd26c..0d50ca51a21a 100644 --- a/airbyte-webapp/src/components/JobItem/components/Logs.tsx +++ b/airbyte-webapp/src/components/JobItem/components/Logs.tsx @@ -34,10 +34,12 @@ const LogsView = styled.div<{ isEmpty?: boolean }>` type LogsProps = { logsArray?: string[]; + logTimestamp?: number; }; -const Logs: React.FC = ({ logsArray }) => { +const Logs: React.FC = ({ logsArray, logTimestamp }) => { const logsJoin = logsArray?.length ? logsArray.join("\n") : "No logs available"; + const matchingLineNumbers = getMatchingLineNumbers(logTimestamp, logsArray); return ( @@ -47,8 +49,10 @@ const Logs: React.FC = ({ logsArray }) => { lineClassName="logLine" highlightLineClassName="highlightLogLine" selectableLines - follow + follow={matchingLineNumbers.length > 0 ? false : true} style={{ background: "transparent" }} + scrollToLine={matchingLineNumbers.length > 0 ? matchingLineNumbers[0] - 1 : undefined} + highlight={matchingLineNumbers} /> ) : ( @@ -57,4 +61,36 @@ const Logs: React.FC = ({ logsArray }) => { ); }; +/** + * Matching the log's line number by time makes the following assumptions: + * 1. The log's lines are already ordered by time + * 2. The timestamps used are in the same timezone + */ +const getMatchingLineNumbers = (matchTimestamp: number | undefined, lines: string[] | undefined) => { + const matchingLineNumbers: number[] = []; + if (!matchTimestamp || !lines) { + return matchingLineNumbers; + } + + let lineCounter = 0; + if (matchTimestamp && lines && lines.length > 0) { + for (const line of lines) { + // matches the the start of a line like "2022-05-17 23:00:19 DEBUG I am a log message" + const timeString = line.match( + /^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1]) (2[0-3]|[01][0-9]):[0-5][0-9]:[0-5][0-9]/ + ); + if (timeString) { + const datetime = Date.parse(`${timeString[0].replace(" ", "T")}Z`); + if (datetime - 1000 <= matchTimestamp && datetime + 1000 >= matchTimestamp) { + // TODO: it appears that LazyLog only highlights the first 2 lines in the array + matchingLineNumbers.push(lineCounter + 1); + } + } + lineCounter++; + } + } + + return matchingLineNumbers; +}; + export default Logs; diff --git a/airbyte-webapp/src/components/JobItem/components/LogsDetails.tsx b/airbyte-webapp/src/components/JobItem/components/LogsDetails.tsx index 07bd5682c928..069ceedb4e08 100644 --- a/airbyte-webapp/src/components/JobItem/components/LogsDetails.tsx +++ b/airbyte-webapp/src/components/JobItem/components/LogsDetails.tsx @@ -34,7 +34,8 @@ export const LogsDetails: React.FC<{ jobDebugInfo?: JobDebugInfoRead; showAttemptStats: boolean; logs?: string[]; -}> = ({ path, id, currentAttempt, jobDebugInfo, showAttemptStats, logs }) => ( + logTimestamp?: number; +}> = ({ path, id, currentAttempt, jobDebugInfo, showAttemptStats, logs, logTimestamp }) => ( <> {currentAttempt && showAttemptStats && ( @@ -51,6 +52,6 @@ export const LogsDetails: React.FC<{ )} - + ); From 99e2302359cf9387a851e12034bf6d95a476acc5 Mon Sep 17 00:00:00 2001 From: evantahler Date: Wed, 18 May 2022 14:39:41 -0700 Subject: [PATCH 06/20] cleat timestamp when logs colapsed --- airbyte-webapp/src/components/JobItem/JobItem.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/airbyte-webapp/src/components/JobItem/JobItem.tsx b/airbyte-webapp/src/components/JobItem/JobItem.tsx index af4eea0292cf..83de7ec1c2fe 100644 --- a/airbyte-webapp/src/components/JobItem/JobItem.tsx +++ b/airbyte-webapp/src/components/JobItem/JobItem.tsx @@ -54,9 +54,12 @@ export const getJobId = (job: SynchronousJobReadWithStatus | JobsWithJobs) => export const JobItem: React.FC = ({ shortInfo, job }) => { const { jobId: linkedJobId } = useAttemptLink(); const [isOpen, setIsOpen] = useState(linkedJobId === getJobId(job)); - const onExpand = () => setIsOpen(!isOpen); const scrollAnchor = useRef(null); const [logTimestamp, setLogTimestamp] = useState(); + const onExpand = () => { + setIsOpen(!isOpen); + setLogTimestamp(undefined); + }; const didSucceed = didJobSucceed(job); From 47f6383bd4ee75c42e3fb0e9e1e709c3d62c48b4 Mon Sep 17 00:00:00 2001 From: evantahler Date: Thu, 19 May 2022 11:37:30 -0700 Subject: [PATCH 07/20] speed up search --- .../components/JobItem/components/Logs.tsx | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/airbyte-webapp/src/components/JobItem/components/Logs.tsx b/airbyte-webapp/src/components/JobItem/components/Logs.tsx index 0d50ca51a21a..27e993d9730a 100644 --- a/airbyte-webapp/src/components/JobItem/components/Logs.tsx +++ b/airbyte-webapp/src/components/JobItem/components/Logs.tsx @@ -65,32 +65,34 @@ const Logs: React.FC = ({ logsArray, logTimestamp }) => { * Matching the log's line number by time makes the following assumptions: * 1. The log's lines are already ordered by time * 2. The timestamps used are in the same timezone + * 3. The error is closer to the end of the log file than the beginning + * 4. Lines are matched by the start of a line with "YYYY-MM-DD HH:mm:ss" format, like "2022-05-17 23:00:19 DEBUG I am a log message" + * + * TODO: it appears that LazyLog only highlights the first 2 lines in the array. Can this be fixed? */ const getMatchingLineNumbers = (matchTimestamp: number | undefined, lines: string[] | undefined) => { - const matchingLineNumbers: number[] = []; - if (!matchTimestamp || !lines) { - return matchingLineNumbers; + if (!matchTimestamp || !lines || lines.length === 0) { + return []; } - let lineCounter = 0; - if (matchTimestamp && lines && lines.length > 0) { - for (const line of lines) { - // matches the the start of a line like "2022-05-17 23:00:19 DEBUG I am a log message" - const timeString = line.match( - /^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1]) (2[0-3]|[01][0-9]):[0-5][0-9]:[0-5][0-9]/ - ); - if (timeString) { - const datetime = Date.parse(`${timeString[0].replace(" ", "T")}Z`); - if (datetime - 1000 <= matchTimestamp && datetime + 1000 >= matchTimestamp) { - // TODO: it appears that LazyLog only highlights the first 2 lines in the array - matchingLineNumbers.push(lineCounter + 1); - } + const matchingLineNumbers: number[] = []; + const matcher = /^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1]) (2[0-3]|[01][0-9]):[0-5][0-9]:[0-5][0-9]/; + + let lineCounter = lines.length - 1; + while (lineCounter >= 0) { + const timeString = lines[lineCounter].match(matcher); + if (timeString) { + const datetime = Date.parse(`${timeString[0].replace(" ", "T")}Z`); + if (datetime - 1000 <= matchTimestamp && datetime + 1000 >= matchTimestamp) { + matchingLineNumbers.push(lineCounter + 1); + } else if (datetime - 2001 <= matchTimestamp) { + break; // Once we've reached a timestamp earlier than our search, we can stop seeking } - lineCounter++; } + lineCounter--; } - return matchingLineNumbers; + return matchingLineNumbers.sort(); }; export default Logs; From a4d29d723f99aa25b302b9dde997a903a13fbc74 Mon Sep 17 00:00:00 2001 From: evantahler Date: Thu, 19 May 2022 17:13:17 -0700 Subject: [PATCH 08/20] rename --- airbyte-webapp/src/components/JobItem/JobItem.tsx | 2 +- .../JobItem/components/{FailureDetails.tsx => ErrorDetails.tsx} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename airbyte-webapp/src/components/JobItem/components/{FailureDetails.tsx => ErrorDetails.tsx} (100%) diff --git a/airbyte-webapp/src/components/JobItem/JobItem.tsx b/airbyte-webapp/src/components/JobItem/JobItem.tsx index 83de7ec1c2fe..be84881a2633 100644 --- a/airbyte-webapp/src/components/JobItem/JobItem.tsx +++ b/airbyte-webapp/src/components/JobItem/JobItem.tsx @@ -10,7 +10,7 @@ import { JobsWithJobs } from "pages/ConnectionPage/pages/ConnectionItemPage/comp import { AttemptRead, JobStatus } from "../../core/request/AirbyteClient"; import { useAttemptLink } from "./attemptLinkUtils"; import ContentWrapper from "./components/ContentWrapper"; -import ErrorDetails from "./components/FailureDetails"; +import ErrorDetails from "./components/ErrorDetails"; import JobLogs from "./components/JobLogs"; import MainInfo from "./components/MainInfo"; diff --git a/airbyte-webapp/src/components/JobItem/components/FailureDetails.tsx b/airbyte-webapp/src/components/JobItem/components/ErrorDetails.tsx similarity index 100% rename from airbyte-webapp/src/components/JobItem/components/FailureDetails.tsx rename to airbyte-webapp/src/components/JobItem/components/ErrorDetails.tsx From 5555e40eda7fcd42c971baae24a44509b40c3aa2 Mon Sep 17 00:00:00 2001 From: evantahler Date: Thu, 19 May 2022 17:24:34 -0700 Subject: [PATCH 09/20] Do what @pedroslopez says --- airbyte-webapp/src/components/JobItem/JobItem.tsx | 7 ++----- .../src/components/JobItem/components/Logs.tsx | 13 ++++++------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/airbyte-webapp/src/components/JobItem/JobItem.tsx b/airbyte-webapp/src/components/JobItem/JobItem.tsx index be84881a2633..299879c02dfd 100644 --- a/airbyte-webapp/src/components/JobItem/JobItem.tsx +++ b/airbyte-webapp/src/components/JobItem/JobItem.tsx @@ -93,11 +93,8 @@ export const JobItem: React.FC = ({ shortInfo, job }) => { > {isOpen ? ( <> - - + + ) : null} diff --git a/airbyte-webapp/src/components/JobItem/components/Logs.tsx b/airbyte-webapp/src/components/JobItem/components/Logs.tsx index 27e993d9730a..b4ae8a3ad95b 100644 --- a/airbyte-webapp/src/components/JobItem/components/Logs.tsx +++ b/airbyte-webapp/src/components/JobItem/components/Logs.tsx @@ -1,4 +1,4 @@ -import React from "react"; +import { useMemo } from "react"; import { FormattedMessage } from "react-intl"; import { LazyLog } from "react-lazylog"; import styled from "styled-components"; @@ -39,7 +39,7 @@ type LogsProps = { const Logs: React.FC = ({ logsArray, logTimestamp }) => { const logsJoin = logsArray?.length ? logsArray.join("\n") : "No logs available"; - const matchingLineNumbers = getMatchingLineNumbers(logTimestamp, logsArray); + const matchingLineNumbers = useMemo(() => getMatchingLineNumbers(logTimestamp, logsArray), [logsArray, logTimestamp]); return ( @@ -67,14 +67,13 @@ const Logs: React.FC = ({ logsArray, logTimestamp }) => { * 2. The timestamps used are in the same timezone * 3. The error is closer to the end of the log file than the beginning * 4. Lines are matched by the start of a line with "YYYY-MM-DD HH:mm:ss" format, like "2022-05-17 23:00:19 DEBUG I am a log message" - * - * TODO: it appears that LazyLog only highlights the first 2 lines in the array. Can this be fixed? */ const getMatchingLineNumbers = (matchTimestamp: number | undefined, lines: string[] | undefined) => { if (!matchTimestamp || !lines || lines.length === 0) { return []; } + const resolutionOffset = 1000; // the resolution of the timestamps is in seconds const matchingLineNumbers: number[] = []; const matcher = /^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1]) (2[0-3]|[01][0-9]):[0-5][0-9]:[0-5][0-9]/; @@ -83,16 +82,16 @@ const getMatchingLineNumbers = (matchTimestamp: number | undefined, lines: strin const timeString = lines[lineCounter].match(matcher); if (timeString) { const datetime = Date.parse(`${timeString[0].replace(" ", "T")}Z`); - if (datetime - 1000 <= matchTimestamp && datetime + 1000 >= matchTimestamp) { + if (datetime - resolutionOffset <= matchTimestamp && datetime + resolutionOffset >= matchTimestamp) { matchingLineNumbers.push(lineCounter + 1); - } else if (datetime - 2001 <= matchTimestamp) { + } else if (datetime - (resolutionOffset * 2 + 1) <= matchTimestamp) { break; // Once we've reached a timestamp earlier than our search, we can stop seeking } } lineCounter--; } - return matchingLineNumbers.sort(); + return [Math.min(...matchingLineNumbers), Math.max(...matchingLineNumbers)]; }; export default Logs; From c98a677b2878e02591e145b2a81b8ea434c2dde8 Mon Sep 17 00:00:00 2001 From: evantahler Date: Mon, 23 May 2022 10:21:25 -0700 Subject: [PATCH 10/20] Rebase from master after big API update --- airbyte-webapp/src/components/JobItem/JobItem.tsx | 8 ++++---- .../components/JobItem/components/ErrorDetails.tsx | 12 +++++++----- airbyte-webapp/src/core/request/AirbyteClient.ts | 1 + 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/airbyte-webapp/src/components/JobItem/JobItem.tsx b/airbyte-webapp/src/components/JobItem/JobItem.tsx index 299879c02dfd..5ecf4be0c7e9 100644 --- a/airbyte-webapp/src/components/JobItem/JobItem.tsx +++ b/airbyte-webapp/src/components/JobItem/JobItem.tsx @@ -92,10 +92,10 @@ export const JobItem: React.FC = ({ shortInfo, job }) => { } > {isOpen ? ( - <> - - - + <> + + + ) : null} diff --git a/airbyte-webapp/src/components/JobItem/components/ErrorDetails.tsx b/airbyte-webapp/src/components/JobItem/components/ErrorDetails.tsx index 0f06173381b7..cf4ed28892df 100644 --- a/airbyte-webapp/src/components/JobItem/components/ErrorDetails.tsx +++ b/airbyte-webapp/src/components/JobItem/components/ErrorDetails.tsx @@ -3,10 +3,10 @@ import styled from "styled-components"; import { Button } from "components/base"; -import { Attempt, Failure } from "core/domain/job/Job"; +import { AttemptRead } from "core/request/AirbyteClient"; type IProps = { - attempts?: Attempt[]; + attempts?: AttemptRead[]; setLogTimestamp: (t: number) => void; }; @@ -18,7 +18,7 @@ const ExpandedFailureContainer = styled.div` color: ${({ theme }) => theme.greyColor40}; `; -const getFailureFromAttempt = (attempt: Attempt) => { +const getFailureFromAttempt = (attempt: AttemptRead) => { return attempt.failureSummary?.failures[0]; }; @@ -29,8 +29,10 @@ const ErrorDetails: React.FC = ({ attempts, setLogTimestamp }) => { return null; } - const getInternalFailureMessage = (failure: Failure) => { + const getInternalFailureMessage = (attempt: AttemptRead) => { + const failure = getFailureFromAttempt(attempt); const failureMessage = failure?.internalMessage ?? formatMessage({ id: "errorView.unknown" }); + return `${formatMessage({ id: "sources.additionalFailureInfo", })}: ${failureMessage}`; @@ -49,7 +51,7 @@ const ErrorDetails: React.FC = ({ attempts, setLogTimestamp }) => { } }; - const internalMessage = getInternalFailureMessage(failure); + const internalMessage = getInternalFailureMessage(attempt); return ( {failure.timestamp ? ( diff --git a/airbyte-webapp/src/core/request/AirbyteClient.ts b/airbyte-webapp/src/core/request/AirbyteClient.ts index 0e16d87b5359..b066780f316f 100644 --- a/airbyte-webapp/src/core/request/AirbyteClient.ts +++ b/airbyte-webapp/src/core/request/AirbyteClient.ts @@ -520,6 +520,7 @@ export interface AttemptFailureReason { failureOrigin?: AttemptFailureOrigin; failureType?: AttemptFailureType; externalMessage?: string; + internalMessage?: string; stacktrace?: string; /** True if it is known that retrying may succeed, e.g. for a transient failure. False if it is known that a retry will not succeed, e.g. for a configuration issue. If not set, retryable status is not well known. */ retryable?: boolean; From 49bc3b0ec451c2e1b79994358d9d13257f6d82aa Mon Sep 17 00:00:00 2001 From: evantahler Date: Mon, 23 May 2022 11:05:33 -0700 Subject: [PATCH 11/20] `floor` matchtimes for greater range matching --- airbyte-webapp/src/components/JobItem/components/Logs.tsx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/airbyte-webapp/src/components/JobItem/components/Logs.tsx b/airbyte-webapp/src/components/JobItem/components/Logs.tsx index b4ae8a3ad95b..3e2e67c68b22 100644 --- a/airbyte-webapp/src/components/JobItem/components/Logs.tsx +++ b/airbyte-webapp/src/components/JobItem/components/Logs.tsx @@ -72,6 +72,7 @@ const getMatchingLineNumbers = (matchTimestamp: number | undefined, lines: strin if (!matchTimestamp || !lines || lines.length === 0) { return []; } + const flooredMatchTimestamp = Math.floor(matchTimestamp / 1000) * 1000; const resolutionOffset = 1000; // the resolution of the timestamps is in seconds const matchingLineNumbers: number[] = []; @@ -82,9 +83,12 @@ const getMatchingLineNumbers = (matchTimestamp: number | undefined, lines: strin const timeString = lines[lineCounter].match(matcher); if (timeString) { const datetime = Date.parse(`${timeString[0].replace(" ", "T")}Z`); - if (datetime - resolutionOffset <= matchTimestamp && datetime + resolutionOffset >= matchTimestamp) { + if ( + datetime - resolutionOffset <= flooredMatchTimestamp && + datetime + resolutionOffset >= flooredMatchTimestamp + ) { matchingLineNumbers.push(lineCounter + 1); - } else if (datetime - (resolutionOffset * 2 + 1) <= matchTimestamp) { + } else if (datetime - (resolutionOffset * 2 + 1) <= flooredMatchTimestamp) { break; // Once we've reached a timestamp earlier than our search, we can stop seeking } } From 5a25024a511469a4d4debbfb818fde2ec7b1ba0c Mon Sep 17 00:00:00 2001 From: Evan Tahler Date: Tue, 24 May 2022 11:40:32 -0700 Subject: [PATCH 12/20] Update airbyte-webapp/src/components/JobItem/components/ErrorDetails.tsx Co-authored-by: Tim Roes --- .../src/components/JobItem/components/ErrorDetails.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-webapp/src/components/JobItem/components/ErrorDetails.tsx b/airbyte-webapp/src/components/JobItem/components/ErrorDetails.tsx index cf4ed28892df..392b94796bcc 100644 --- a/airbyte-webapp/src/components/JobItem/components/ErrorDetails.tsx +++ b/airbyte-webapp/src/components/JobItem/components/ErrorDetails.tsx @@ -25,7 +25,7 @@ const getFailureFromAttempt = (attempt: AttemptRead) => { const ErrorDetails: React.FC = ({ attempts, setLogTimestamp }) => { const { formatMessage } = useIntl(); - if (!attempts || attempts.length === 0) { + if (!attempts?.length) { return null; } From db57953ad3db0524b446b8a3ba24ec6d576d6ccb Mon Sep 17 00:00:00 2001 From: Evan Tahler Date: Tue, 24 May 2022 11:40:43 -0700 Subject: [PATCH 13/20] Update airbyte-webapp/src/components/JobItem/components/ErrorDetails.tsx Co-authored-by: Tim Roes --- .../src/components/JobItem/components/ErrorDetails.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-webapp/src/components/JobItem/components/ErrorDetails.tsx b/airbyte-webapp/src/components/JobItem/components/ErrorDetails.tsx index 392b94796bcc..e6a36f242435 100644 --- a/airbyte-webapp/src/components/JobItem/components/ErrorDetails.tsx +++ b/airbyte-webapp/src/components/JobItem/components/ErrorDetails.tsx @@ -54,13 +54,13 @@ const ErrorDetails: React.FC = ({ attempts, setLogTimestamp }) => { const internalMessage = getInternalFailureMessage(attempt); return ( - {failure.timestamp ? ( + {!!failure.timestamp && ( <> {" "} - ) : null} + )} {internalMessage} ); From 212ba4479218ba0d738494427efd8571fbbca5d5 Mon Sep 17 00:00:00 2001 From: Evan Tahler Date: Tue, 24 May 2022 11:41:09 -0700 Subject: [PATCH 14/20] Update airbyte-webapp/src/components/JobItem/components/Logs.tsx Co-authored-by: Tim Roes --- airbyte-webapp/src/components/JobItem/components/Logs.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-webapp/src/components/JobItem/components/Logs.tsx b/airbyte-webapp/src/components/JobItem/components/Logs.tsx index 3e2e67c68b22..24ed6352a7c7 100644 --- a/airbyte-webapp/src/components/JobItem/components/Logs.tsx +++ b/airbyte-webapp/src/components/JobItem/components/Logs.tsx @@ -49,7 +49,7 @@ const Logs: React.FC = ({ logsArray, logTimestamp }) => { lineClassName="logLine" highlightLineClassName="highlightLogLine" selectableLines - follow={matchingLineNumbers.length > 0 ? false : true} + follow={matchingLineNumbers.length === 0} style={{ background: "transparent" }} scrollToLine={matchingLineNumbers.length > 0 ? matchingLineNumbers[0] - 1 : undefined} highlight={matchingLineNumbers} From f1760bfbc8a394155953ce67dd3922ee23eaabec Mon Sep 17 00:00:00 2001 From: Evan Tahler Date: Tue, 24 May 2022 11:41:19 -0700 Subject: [PATCH 15/20] Update airbyte-webapp/src/components/JobItem/JobItem.tsx Co-authored-by: Tim Roes --- airbyte-webapp/src/components/JobItem/JobItem.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-webapp/src/components/JobItem/JobItem.tsx b/airbyte-webapp/src/components/JobItem/JobItem.tsx index 5ecf4be0c7e9..4d1403e56436 100644 --- a/airbyte-webapp/src/components/JobItem/JobItem.tsx +++ b/airbyte-webapp/src/components/JobItem/JobItem.tsx @@ -91,12 +91,12 @@ export const JobItem: React.FC = ({ shortInfo, job }) => { } > - {isOpen ? ( + {isOpen && ( <> - ) : null} + )} From 8647e806fcfc2453dfeb456051b4b1496f847487 Mon Sep 17 00:00:00 2001 From: evantahler Date: Tue, 24 May 2022 16:44:17 -0700 Subject: [PATCH 16/20] replace regexp with `dayJs` --- .../src/components/JobItem/components/Logs.tsx | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/airbyte-webapp/src/components/JobItem/components/Logs.tsx b/airbyte-webapp/src/components/JobItem/components/Logs.tsx index 24ed6352a7c7..7a8850df2db7 100644 --- a/airbyte-webapp/src/components/JobItem/components/Logs.tsx +++ b/airbyte-webapp/src/components/JobItem/components/Logs.tsx @@ -1,8 +1,15 @@ +import dayjs from "dayjs"; +import customParseFormat from "dayjs/plugin/customParseFormat"; +import utc from "dayjs/plugin/utc"; import { useMemo } from "react"; import { FormattedMessage } from "react-intl"; import { LazyLog } from "react-lazylog"; import styled from "styled-components"; +dayjs.extend(customParseFormat); +dayjs.extend(utc); +const dateTimeFormat = "YYYY-MM-DD HH:mm:ss"; + const LogsView = styled.div<{ isEmpty?: boolean }>` padding: 11px ${({ isEmpty }) => (isEmpty ? 42 : 12)}px 20px; font-size: 12px; @@ -76,13 +83,11 @@ const getMatchingLineNumbers = (matchTimestamp: number | undefined, lines: strin const resolutionOffset = 1000; // the resolution of the timestamps is in seconds const matchingLineNumbers: number[] = []; - const matcher = /^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1]) (2[0-3]|[01][0-9]):[0-5][0-9]:[0-5][0-9]/; let lineCounter = lines.length - 1; while (lineCounter >= 0) { - const timeString = lines[lineCounter].match(matcher); - if (timeString) { - const datetime = Date.parse(`${timeString[0].replace(" ", "T")}Z`); + const datetime = dayjs.utc(lines[lineCounter], dateTimeFormat, false)?.toDate()?.getTime(); + if (datetime) { if ( datetime - resolutionOffset <= flooredMatchTimestamp && datetime + resolutionOffset >= flooredMatchTimestamp From 9355d5c65940c92e6e2dac1df4497e171023a6b8 Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Wed, 25 May 2022 13:30:16 +0200 Subject: [PATCH 17/20] Extract dayjs into globals module --- .../src/components/JobItem/components/Logs.tsx | 8 ++------ airbyte-webapp/src/globals.ts | 12 ++++++++++++ airbyte-webapp/src/index.tsx | 2 ++ 3 files changed, 16 insertions(+), 6 deletions(-) create mode 100644 airbyte-webapp/src/globals.ts diff --git a/airbyte-webapp/src/components/JobItem/components/Logs.tsx b/airbyte-webapp/src/components/JobItem/components/Logs.tsx index 7a8850df2db7..9319a2e2327c 100644 --- a/airbyte-webapp/src/components/JobItem/components/Logs.tsx +++ b/airbyte-webapp/src/components/JobItem/components/Logs.tsx @@ -1,14 +1,10 @@ import dayjs from "dayjs"; -import customParseFormat from "dayjs/plugin/customParseFormat"; -import utc from "dayjs/plugin/utc"; import { useMemo } from "react"; import { FormattedMessage } from "react-intl"; import { LazyLog } from "react-lazylog"; import styled from "styled-components"; -dayjs.extend(customParseFormat); -dayjs.extend(utc); -const dateTimeFormat = "YYYY-MM-DD HH:mm:ss"; +const DATE_TIME_FORMAT = "YYYY-MM-DD HH:mm:ss"; const LogsView = styled.div<{ isEmpty?: boolean }>` padding: 11px ${({ isEmpty }) => (isEmpty ? 42 : 12)}px 20px; @@ -86,7 +82,7 @@ const getMatchingLineNumbers = (matchTimestamp: number | undefined, lines: strin let lineCounter = lines.length - 1; while (lineCounter >= 0) { - const datetime = dayjs.utc(lines[lineCounter], dateTimeFormat, false)?.toDate()?.getTime(); + const datetime = dayjs.utc(lines[lineCounter], DATE_TIME_FORMAT, false)?.toDate()?.getTime(); if (datetime) { if ( datetime - resolutionOffset <= flooredMatchTimestamp && diff --git a/airbyte-webapp/src/globals.ts b/airbyte-webapp/src/globals.ts new file mode 100644 index 000000000000..a47c4100c47b --- /dev/null +++ b/airbyte-webapp/src/globals.ts @@ -0,0 +1,12 @@ +// This file should contain all stateful modification that need to be made to libraries. +// In general this is a bad pattern that should try to be avoided, but some libraries will +// require plugins to be registered to their global instance. This file encapsulates all those +// stateful modifications. + +import dayjs from "dayjs"; +import customParseFormat from "dayjs/plugin/customParseFormat"; +import utc from "dayjs/plugin/utc"; + +// Configure dayjs instance +dayjs.extend(customParseFormat); +dayjs.extend(utc); diff --git a/airbyte-webapp/src/index.tsx b/airbyte-webapp/src/index.tsx index c168556485de..c28ca7e21d12 100644 --- a/airbyte-webapp/src/index.tsx +++ b/airbyte-webapp/src/index.tsx @@ -4,6 +4,8 @@ import { lazy, Suspense } from "react"; import ReactDOM from "react-dom"; import "react-reflex/styles.css"; +import "./globals"; + // We do not follow default config approach since we want to init sentry asap Sentry.init({ dsn: process.env.REACT_APP_SENTRY_DSN || window.REACT_APP_SENTRY_DSN, From a52b43b48dc1fe4cfc7d1c931e8abd0ec97ee195 Mon Sep 17 00:00:00 2001 From: evantahler Date: Wed, 25 May 2022 09:40:49 -0700 Subject: [PATCH 18/20] mach time in full-second resolution --- .../src/components/JobItem/components/Logs.tsx | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/airbyte-webapp/src/components/JobItem/components/Logs.tsx b/airbyte-webapp/src/components/JobItem/components/Logs.tsx index 9319a2e2327c..64c105e2539e 100644 --- a/airbyte-webapp/src/components/JobItem/components/Logs.tsx +++ b/airbyte-webapp/src/components/JobItem/components/Logs.tsx @@ -75,21 +75,18 @@ const getMatchingLineNumbers = (matchTimestamp: number | undefined, lines: strin if (!matchTimestamp || !lines || lines.length === 0) { return []; } - const flooredMatchTimestamp = Math.floor(matchTimestamp / 1000) * 1000; - - const resolutionOffset = 1000; // the resolution of the timestamps is in seconds + const matchTimestampSeconds = Math.floor(matchTimestamp / 1000); const matchingLineNumbers: number[] = []; let lineCounter = lines.length - 1; while (lineCounter >= 0) { const datetime = dayjs.utc(lines[lineCounter], DATE_TIME_FORMAT, false)?.toDate()?.getTime(); if (datetime) { - if ( - datetime - resolutionOffset <= flooredMatchTimestamp && - datetime + resolutionOffset >= flooredMatchTimestamp - ) { + // The resolution of the timestamps in the logs is seconds (no ms), so this will not need to be rounded + const datetimeSeconds = datetime / 1000; + if (datetimeSeconds + 1 === matchTimestampSeconds) { matchingLineNumbers.push(lineCounter + 1); - } else if (datetime - (resolutionOffset * 2 + 1) <= flooredMatchTimestamp) { + } else if (datetimeSeconds < matchTimestampSeconds) { break; // Once we've reached a timestamp earlier than our search, we can stop seeking } } From a001ed279ecc818aa7422f36d1fde442f79dffe7 Mon Sep 17 00:00:00 2001 From: evantahler Date: Wed, 25 May 2022 12:16:49 -0700 Subject: [PATCH 19/20] revert `dayJs` and use `Date.parse` --- .../src/components/JobItem/components/Logs.tsx | 18 +++++++++++------- airbyte-webapp/src/globals.ts | 12 ------------ airbyte-webapp/src/index.tsx | 2 -- 3 files changed, 11 insertions(+), 21 deletions(-) delete mode 100644 airbyte-webapp/src/globals.ts diff --git a/airbyte-webapp/src/components/JobItem/components/Logs.tsx b/airbyte-webapp/src/components/JobItem/components/Logs.tsx index 64c105e2539e..f3a48f97bbe6 100644 --- a/airbyte-webapp/src/components/JobItem/components/Logs.tsx +++ b/airbyte-webapp/src/components/JobItem/components/Logs.tsx @@ -1,10 +1,10 @@ -import dayjs from "dayjs"; import { useMemo } from "react"; import { FormattedMessage } from "react-intl"; import { LazyLog } from "react-lazylog"; import styled from "styled-components"; -const DATE_TIME_FORMAT = "YYYY-MM-DD HH:mm:ss"; +const TIMESTAMP_MATCHER = + /^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1]) (2[0-3]|[01][0-9]):[0-5][0-9]:[0-5][0-9]/; const LogsView = styled.div<{ isEmpty?: boolean }>` padding: 11px ${({ isEmpty }) => (isEmpty ? 42 : 12)}px 20px; @@ -75,19 +75,23 @@ const getMatchingLineNumbers = (matchTimestamp: number | undefined, lines: strin if (!matchTimestamp || !lines || lines.length === 0) { return []; } + const matchTimestampSeconds = Math.floor(matchTimestamp / 1000); const matchingLineNumbers: number[] = []; let lineCounter = lines.length - 1; while (lineCounter >= 0) { - const datetime = dayjs.utc(lines[lineCounter], DATE_TIME_FORMAT, false)?.toDate()?.getTime(); - if (datetime) { + const timeString = lines[lineCounter].match(TIMESTAMP_MATCHER); + if (timeString) { // The resolution of the timestamps in the logs is seconds (no ms), so this will not need to be rounded - const datetimeSeconds = datetime / 1000; - if (datetimeSeconds + 1 === matchTimestampSeconds) { + // Log timestamps are always in UTC + const datetimeSeconds = Date.parse(`${timeString[0].replace(" ", "T")}Z`) / 1000; + // Due to the fuzziness of timestamp matching, and how the platform may take some time to build the failureReason timestamp from the log line, we are matching line numbers with +/- 1 second from the failure timestamp + if (datetimeSeconds >= matchTimestampSeconds - 1 && datetimeSeconds <= matchTimestampSeconds + 1) { matchingLineNumbers.push(lineCounter + 1); } else if (datetimeSeconds < matchTimestampSeconds) { - break; // Once we've reached a timestamp earlier than our search, we can stop seeking + // Once we've reached a timestamp earlier than our search, we can stop seeking + break; } } lineCounter--; diff --git a/airbyte-webapp/src/globals.ts b/airbyte-webapp/src/globals.ts deleted file mode 100644 index a47c4100c47b..000000000000 --- a/airbyte-webapp/src/globals.ts +++ /dev/null @@ -1,12 +0,0 @@ -// This file should contain all stateful modification that need to be made to libraries. -// In general this is a bad pattern that should try to be avoided, but some libraries will -// require plugins to be registered to their global instance. This file encapsulates all those -// stateful modifications. - -import dayjs from "dayjs"; -import customParseFormat from "dayjs/plugin/customParseFormat"; -import utc from "dayjs/plugin/utc"; - -// Configure dayjs instance -dayjs.extend(customParseFormat); -dayjs.extend(utc); diff --git a/airbyte-webapp/src/index.tsx b/airbyte-webapp/src/index.tsx index c28ca7e21d12..c168556485de 100644 --- a/airbyte-webapp/src/index.tsx +++ b/airbyte-webapp/src/index.tsx @@ -4,8 +4,6 @@ import { lazy, Suspense } from "react"; import ReactDOM from "react-dom"; import "react-reflex/styles.css"; -import "./globals"; - // We do not follow default config approach since we want to init sentry asap Sentry.init({ dsn: process.env.REACT_APP_SENTRY_DSN || window.REACT_APP_SENTRY_DSN, From 27efa7b5260e7cb83e51316ac9cb09352d860d35 Mon Sep 17 00:00:00 2001 From: evantahler Date: Thu, 26 May 2022 11:11:42 -0700 Subject: [PATCH 20/20] Just show failure timestamp rather than scroll --- .../src/components/JobItem/JobItem.tsx | 6 +-- .../JobItem/components/ErrorDetails.tsx | 22 +++----- .../components/JobItem/components/JobLogs.tsx | 4 +- .../components/JobItem/components/Logs.tsx | 50 ++----------------- .../JobItem/components/LogsDetails.tsx | 5 +- airbyte-webapp/src/globals.ts | 12 +++++ airbyte-webapp/src/index.tsx | 2 + 7 files changed, 30 insertions(+), 71 deletions(-) create mode 100644 airbyte-webapp/src/globals.ts diff --git a/airbyte-webapp/src/components/JobItem/JobItem.tsx b/airbyte-webapp/src/components/JobItem/JobItem.tsx index 4d1403e56436..11543de2a0b5 100644 --- a/airbyte-webapp/src/components/JobItem/JobItem.tsx +++ b/airbyte-webapp/src/components/JobItem/JobItem.tsx @@ -55,10 +55,8 @@ export const JobItem: React.FC = ({ shortInfo, job }) => { const { jobId: linkedJobId } = useAttemptLink(); const [isOpen, setIsOpen] = useState(linkedJobId === getJobId(job)); const scrollAnchor = useRef(null); - const [logTimestamp, setLogTimestamp] = useState(); const onExpand = () => { setIsOpen(!isOpen); - setLogTimestamp(undefined); }; const didSucceed = didJobSucceed(job); @@ -93,8 +91,8 @@ export const JobItem: React.FC = ({ shortInfo, job }) => { > {isOpen && ( <> - - + + )} diff --git a/airbyte-webapp/src/components/JobItem/components/ErrorDetails.tsx b/airbyte-webapp/src/components/JobItem/components/ErrorDetails.tsx index e6a36f242435..ab6387df98b5 100644 --- a/airbyte-webapp/src/components/JobItem/components/ErrorDetails.tsx +++ b/airbyte-webapp/src/components/JobItem/components/ErrorDetails.tsx @@ -1,13 +1,11 @@ +import dayjs from "dayjs"; import { useIntl } from "react-intl"; import styled from "styled-components"; -import { Button } from "components/base"; - import { AttemptRead } from "core/request/AirbyteClient"; type IProps = { attempts?: AttemptRead[]; - setLogTimestamp: (t: number) => void; }; const ExpandedFailureContainer = styled.div` @@ -18,11 +16,15 @@ const ExpandedFailureContainer = styled.div` color: ${({ theme }) => theme.greyColor40}; `; +const FailureDateDisplay = styled.span` + font-style: italic; +`; + const getFailureFromAttempt = (attempt: AttemptRead) => { return attempt.failureSummary?.failures[0]; }; -const ErrorDetails: React.FC = ({ attempts, setLogTimestamp }) => { +const ErrorDetails: React.FC = ({ attempts }) => { const { formatMessage } = useIntl(); if (!attempts?.length) { @@ -45,21 +47,11 @@ const ErrorDetails: React.FC = ({ attempts, setLogTimestamp }) => { return null; } - const jumpToLogTimestamp = () => { - if (failure.timestamp) { - setLogTimestamp(failure.timestamp); - } - }; - const internalMessage = getInternalFailureMessage(attempt); return ( {!!failure.timestamp && ( - <> - {" "} - + {dayjs.utc(failure.timestamp).format("YYYY-MM-DD HH:mm:ss")} - )} {internalMessage} diff --git a/airbyte-webapp/src/components/JobItem/components/JobLogs.tsx b/airbyte-webapp/src/components/JobItem/components/JobLogs.tsx index d6a29f33784c..8df8ecbdc5d1 100644 --- a/airbyte-webapp/src/components/JobItem/components/JobLogs.tsx +++ b/airbyte-webapp/src/components/JobItem/components/JobLogs.tsx @@ -16,7 +16,6 @@ import Tabs, { TabsData } from "./Tabs"; type JobLogsProps = { jobIsFailed?: boolean; job: SynchronousJobReadWithStatus | JobsWithJobs; - logTimestamp?: number; }; const isPartialSuccess = (attempt: AttemptRead) => { @@ -29,7 +28,7 @@ const jobIsSynchronousJobRead = ( return !!(job as SynchronousJobReadWithStatus)?.logs?.logLines; }; -const JobLogs: React.FC = ({ jobIsFailed, job, logTimestamp }) => { +const JobLogs: React.FC = ({ jobIsFailed, job }) => { const isSynchronousJobRead = jobIsSynchronousJobRead(job); const id: number | string = (job as JobsWithJobs).job?.id ?? (job as SynchronousJobReadWithStatus).id; @@ -86,7 +85,6 @@ const JobLogs: React.FC = ({ jobIsFailed, job, logTimestamp }) => jobDebugInfo={debugInfo} showAttemptStats={attempts > 1} logs={debugInfo?.attempts[attemptNumber].logs.logLines} - logTimestamp={logTimestamp} /> ); diff --git a/airbyte-webapp/src/components/JobItem/components/Logs.tsx b/airbyte-webapp/src/components/JobItem/components/Logs.tsx index f3a48f97bbe6..3a9f4f6eb04d 100644 --- a/airbyte-webapp/src/components/JobItem/components/Logs.tsx +++ b/airbyte-webapp/src/components/JobItem/components/Logs.tsx @@ -1,11 +1,7 @@ -import { useMemo } from "react"; import { FormattedMessage } from "react-intl"; import { LazyLog } from "react-lazylog"; import styled from "styled-components"; -const TIMESTAMP_MATCHER = - /^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1]) (2[0-3]|[01][0-9]):[0-5][0-9]:[0-5][0-9]/; - const LogsView = styled.div<{ isEmpty?: boolean }>` padding: 11px ${({ isEmpty }) => (isEmpty ? 42 : 12)}px 20px; font-size: 12px; @@ -37,12 +33,10 @@ const LogsView = styled.div<{ isEmpty?: boolean }>` type LogsProps = { logsArray?: string[]; - logTimestamp?: number; }; -const Logs: React.FC = ({ logsArray, logTimestamp }) => { +const Logs: React.FC = ({ logsArray }) => { const logsJoin = logsArray?.length ? logsArray.join("\n") : "No logs available"; - const matchingLineNumbers = useMemo(() => getMatchingLineNumbers(logTimestamp, logsArray), [logsArray, logTimestamp]); return ( @@ -52,10 +46,10 @@ const Logs: React.FC = ({ logsArray, logTimestamp }) => { lineClassName="logLine" highlightLineClassName="highlightLogLine" selectableLines - follow={matchingLineNumbers.length === 0} + follow={true} style={{ background: "transparent" }} - scrollToLine={matchingLineNumbers.length > 0 ? matchingLineNumbers[0] - 1 : undefined} - highlight={matchingLineNumbers} + scrollToLine={undefined} + highlight={[]} /> ) : ( @@ -64,40 +58,4 @@ const Logs: React.FC = ({ logsArray, logTimestamp }) => { ); }; -/** - * Matching the log's line number by time makes the following assumptions: - * 1. The log's lines are already ordered by time - * 2. The timestamps used are in the same timezone - * 3. The error is closer to the end of the log file than the beginning - * 4. Lines are matched by the start of a line with "YYYY-MM-DD HH:mm:ss" format, like "2022-05-17 23:00:19 DEBUG I am a log message" - */ -const getMatchingLineNumbers = (matchTimestamp: number | undefined, lines: string[] | undefined) => { - if (!matchTimestamp || !lines || lines.length === 0) { - return []; - } - - const matchTimestampSeconds = Math.floor(matchTimestamp / 1000); - const matchingLineNumbers: number[] = []; - - let lineCounter = lines.length - 1; - while (lineCounter >= 0) { - const timeString = lines[lineCounter].match(TIMESTAMP_MATCHER); - if (timeString) { - // The resolution of the timestamps in the logs is seconds (no ms), so this will not need to be rounded - // Log timestamps are always in UTC - const datetimeSeconds = Date.parse(`${timeString[0].replace(" ", "T")}Z`) / 1000; - // Due to the fuzziness of timestamp matching, and how the platform may take some time to build the failureReason timestamp from the log line, we are matching line numbers with +/- 1 second from the failure timestamp - if (datetimeSeconds >= matchTimestampSeconds - 1 && datetimeSeconds <= matchTimestampSeconds + 1) { - matchingLineNumbers.push(lineCounter + 1); - } else if (datetimeSeconds < matchTimestampSeconds) { - // Once we've reached a timestamp earlier than our search, we can stop seeking - break; - } - } - lineCounter--; - } - - return [Math.min(...matchingLineNumbers), Math.max(...matchingLineNumbers)]; -}; - export default Logs; diff --git a/airbyte-webapp/src/components/JobItem/components/LogsDetails.tsx b/airbyte-webapp/src/components/JobItem/components/LogsDetails.tsx index 069ceedb4e08..07bd5682c928 100644 --- a/airbyte-webapp/src/components/JobItem/components/LogsDetails.tsx +++ b/airbyte-webapp/src/components/JobItem/components/LogsDetails.tsx @@ -34,8 +34,7 @@ export const LogsDetails: React.FC<{ jobDebugInfo?: JobDebugInfoRead; showAttemptStats: boolean; logs?: string[]; - logTimestamp?: number; -}> = ({ path, id, currentAttempt, jobDebugInfo, showAttemptStats, logs, logTimestamp }) => ( +}> = ({ path, id, currentAttempt, jobDebugInfo, showAttemptStats, logs }) => ( <> {currentAttempt && showAttemptStats && ( @@ -52,6 +51,6 @@ export const LogsDetails: React.FC<{ )} - + ); diff --git a/airbyte-webapp/src/globals.ts b/airbyte-webapp/src/globals.ts new file mode 100644 index 000000000000..a47c4100c47b --- /dev/null +++ b/airbyte-webapp/src/globals.ts @@ -0,0 +1,12 @@ +// This file should contain all stateful modification that need to be made to libraries. +// In general this is a bad pattern that should try to be avoided, but some libraries will +// require plugins to be registered to their global instance. This file encapsulates all those +// stateful modifications. + +import dayjs from "dayjs"; +import customParseFormat from "dayjs/plugin/customParseFormat"; +import utc from "dayjs/plugin/utc"; + +// Configure dayjs instance +dayjs.extend(customParseFormat); +dayjs.extend(utc); diff --git a/airbyte-webapp/src/index.tsx b/airbyte-webapp/src/index.tsx index c168556485de..c28ca7e21d12 100644 --- a/airbyte-webapp/src/index.tsx +++ b/airbyte-webapp/src/index.tsx @@ -4,6 +4,8 @@ import { lazy, Suspense } from "react"; import ReactDOM from "react-dom"; import "react-reflex/styles.css"; +import "./globals"; + // We do not follow default config approach since we want to init sentry asap Sentry.init({ dsn: process.env.REACT_APP_SENTRY_DSN || window.REACT_APP_SENTRY_DSN,