diff --git a/app/schema.graphql b/app/schema.graphql index 28bc42d657..7382aaa800 100644 --- a/app/schema.graphql +++ b/app/schema.graphql @@ -583,6 +583,11 @@ type Span { """ cumulativeTokenCountCompletion: Int + """ + Propagated status code that percolates up error status codes from descendant spans (children, grandchildren, etc.) + """ + propagatedStatusCode: SpanStatusCode! + """ Evaluations associated with the span, e.g. if the span is an LLM, an evaluation may assess the helpfulness of its response with respect to its input. """ diff --git a/app/src/pages/trace/TracePage.tsx b/app/src/pages/trace/TracePage.tsx index 1cc0afc370..ae79f978ed 100644 --- a/app/src/pages/trace/TracePage.tsx +++ b/app/src/pages/trace/TracePage.tsx @@ -149,7 +149,7 @@ export function TracePage() { } name spanKind - statusCode + statusCode: propagatedStatusCode startTime parentId latencyMs diff --git a/app/src/pages/trace/__generated__/TracePageQuery.graphql.ts b/app/src/pages/trace/__generated__/TracePageQuery.graphql.ts index 82a831aec7..ae709f482f 100644 --- a/app/src/pages/trace/__generated__/TracePageQuery.graphql.ts +++ b/app/src/pages/trace/__generated__/TracePageQuery.graphql.ts @@ -1,5 +1,5 @@ /** - * @generated SignedSource<<1eb72a3d680d71c670d7c4b1514ade00>> + * @generated SignedSource<<195b3d0d088479513351cc9f1315a80d>> * @lightSyntaxTransform * @nogrep */ @@ -130,10 +130,10 @@ v4 = { "storageKey": null }, v5 = { - "alias": null, + "alias": "statusCode", "args": null, "kind": "ScalarField", - "name": "statusCode", + "name": "propagatedStatusCode", "storageKey": null }, v6 = { @@ -440,16 +440,16 @@ return { ] }, "params": { - "cacheID": "2712afb2c9e3f823ca772e8c26b7f83d", + "cacheID": "31d1c1317e04312f8cd6b6856807d0c5", "id": null, "metadata": {}, "name": "TracePageQuery", "operationKind": "query", - "text": "query TracePageQuery(\n $traceId: ID!\n) {\n spans(traceIds: [$traceId], sort: {col: startTime, dir: asc}) {\n edges {\n span: node {\n context {\n spanId\n }\n name\n spanKind\n statusCode\n startTime\n parentId\n latencyMs\n tokenCountTotal\n tokenCountPrompt\n tokenCountCompletion\n input {\n value\n mimeType\n }\n output {\n value\n mimeType\n }\n attributes\n events {\n name\n message\n timestamp\n }\n spanEvaluations {\n name\n label\n score\n }\n documentEvaluations {\n documentPosition\n name\n label\n score\n explanation\n }\n ...SpanEvaluationsTable_evals\n }\n }\n }\n}\n\nfragment SpanEvaluationsTable_evals on Span {\n spanEvaluations {\n name\n label\n score\n explanation\n }\n}\n" + "text": "query TracePageQuery(\n $traceId: ID!\n) {\n spans(traceIds: [$traceId], sort: {col: startTime, dir: asc}) {\n edges {\n span: node {\n context {\n spanId\n }\n name\n spanKind\n statusCode: propagatedStatusCode\n startTime\n parentId\n latencyMs\n tokenCountTotal\n tokenCountPrompt\n tokenCountCompletion\n input {\n value\n mimeType\n }\n output {\n value\n mimeType\n }\n attributes\n events {\n name\n message\n timestamp\n }\n spanEvaluations {\n name\n label\n score\n }\n documentEvaluations {\n documentPosition\n name\n label\n score\n explanation\n }\n ...SpanEvaluationsTable_evals\n }\n }\n }\n}\n\nfragment SpanEvaluationsTable_evals on Span {\n spanEvaluations {\n name\n label\n score\n explanation\n }\n}\n" } }; })(); -(node as any).hash = "fa1253d695252b087e84e1ea06055595"; +(node as any).hash = "0774d4c7f66298e7a68e70354249c151"; export default node; diff --git a/app/src/pages/tracing/TracesTable.tsx b/app/src/pages/tracing/TracesTable.tsx index e7783bb0fc..b1f433c947 100644 --- a/app/src/pages/tracing/TracesTable.tsx +++ b/app/src/pages/tracing/TracesTable.tsx @@ -119,7 +119,7 @@ export function TracesTable(props: TracesTableProps) { rootSpan: node { spanKind name - statusCode + statusCode: propagatedStatusCode startTime latencyMs tokenCountTotal: cumulativeTokenCountTotal @@ -144,7 +144,7 @@ export function TracesTable(props: TracesTableProps) { descendants { spanKind name - statusCode + statusCode: propagatedStatusCode startTime latencyMs parentId diff --git a/app/src/pages/tracing/__generated__/TracesTableQuery.graphql.ts b/app/src/pages/tracing/__generated__/TracesTableQuery.graphql.ts index 7181998b3c..50e69c542e 100644 --- a/app/src/pages/tracing/__generated__/TracesTableQuery.graphql.ts +++ b/app/src/pages/tracing/__generated__/TracesTableQuery.graphql.ts @@ -1,5 +1,5 @@ /** - * @generated SignedSource<> + * @generated SignedSource<<59e432397316135817574c68975277b8>> * @lightSyntaxTransform * @nogrep */ @@ -108,10 +108,10 @@ v7 = { "storageKey": null }, v8 = { - "alias": null, + "alias": "statusCode", "args": null, "kind": "ScalarField", - "name": "statusCode", + "name": "propagatedStatusCode", "storageKey": null }, v9 = { @@ -414,16 +414,16 @@ return { ] }, "params": { - "cacheID": "548b60c4c1410666b4f0533db3394b05", + "cacheID": "fab5f025fcc472c6a2d5a9860ed94f8e", "id": null, "metadata": {}, "name": "TracesTableQuery", "operationKind": "query", - "text": "query TracesTableQuery(\n $after: String = null\n $filterCondition: String = null\n $first: Int = 100\n $sort: SpanSort = {col: startTime, dir: desc}\n) {\n ...TracesTable_spans_1XEuU\n}\n\nfragment TracesTable_spans_1XEuU on Query {\n rootSpans: spans(first: $first, after: $after, sort: $sort, rootSpansOnly: true, filterCondition: $filterCondition) {\n edges {\n rootSpan: node {\n spanKind\n name\n statusCode\n startTime\n latencyMs\n tokenCountTotal: cumulativeTokenCountTotal\n tokenCountPrompt: cumulativeTokenCountPrompt\n tokenCountCompletion: cumulativeTokenCountCompletion\n parentId\n input {\n value\n }\n output {\n value\n }\n context {\n spanId\n traceId\n }\n spanEvaluations {\n name\n label\n score\n }\n descendants {\n spanKind\n name\n statusCode\n startTime\n latencyMs\n parentId\n tokenCountTotal\n tokenCountPrompt\n tokenCountCompletion\n input {\n value\n }\n output {\n value\n }\n context {\n spanId\n traceId\n }\n spanEvaluations {\n name\n label\n score\n }\n }\n }\n cursor\n node {\n __typename\n }\n }\n pageInfo {\n endCursor\n hasNextPage\n }\n }\n}\n" + "text": "query TracesTableQuery(\n $after: String = null\n $filterCondition: String = null\n $first: Int = 100\n $sort: SpanSort = {col: startTime, dir: desc}\n) {\n ...TracesTable_spans_1XEuU\n}\n\nfragment TracesTable_spans_1XEuU on Query {\n rootSpans: spans(first: $first, after: $after, sort: $sort, rootSpansOnly: true, filterCondition: $filterCondition) {\n edges {\n rootSpan: node {\n spanKind\n name\n statusCode: propagatedStatusCode\n startTime\n latencyMs\n tokenCountTotal: cumulativeTokenCountTotal\n tokenCountPrompt: cumulativeTokenCountPrompt\n tokenCountCompletion: cumulativeTokenCountCompletion\n parentId\n input {\n value\n }\n output {\n value\n }\n context {\n spanId\n traceId\n }\n spanEvaluations {\n name\n label\n score\n }\n descendants {\n spanKind\n name\n statusCode: propagatedStatusCode\n startTime\n latencyMs\n parentId\n tokenCountTotal\n tokenCountPrompt\n tokenCountCompletion\n input {\n value\n }\n output {\n value\n }\n context {\n spanId\n traceId\n }\n spanEvaluations {\n name\n label\n score\n }\n }\n }\n cursor\n node {\n __typename\n }\n }\n pageInfo {\n endCursor\n hasNextPage\n }\n }\n}\n" } }; })(); -(node as any).hash = "601519f525b82c387835d8876bc3eab9"; +(node as any).hash = "0f56f1244730e44f83105f9f3214cb7c"; export default node; diff --git a/app/src/pages/tracing/__generated__/TracesTable_spans.graphql.ts b/app/src/pages/tracing/__generated__/TracesTable_spans.graphql.ts index b3e11d84f6..169a7acd0f 100644 --- a/app/src/pages/tracing/__generated__/TracesTable_spans.graphql.ts +++ b/app/src/pages/tracing/__generated__/TracesTable_spans.graphql.ts @@ -1,5 +1,5 @@ /** - * @generated SignedSource<<25f062cc55729b3b589d3bacc74dfe3c>> + * @generated SignedSource<<0543bdb45df4afc207b3fd254fcc84de>> * @lightSyntaxTransform * @nogrep */ @@ -95,10 +95,10 @@ v2 = { "storageKey": null }, v3 = { - "alias": null, + "alias": "statusCode", "args": null, "kind": "ScalarField", - "name": "statusCode", + "name": "propagatedStatusCode", "storageKey": null }, v4 = { @@ -431,6 +431,6 @@ return { }; })(); -(node as any).hash = "601519f525b82c387835d8876bc3eab9"; +(node as any).hash = "0f56f1244730e44f83105f9f3214cb7c"; export default node; diff --git a/app/src/pages/tracing/__generated__/TracingHomePageQuery.graphql.ts b/app/src/pages/tracing/__generated__/TracingHomePageQuery.graphql.ts index a4e061b2bc..ccf5e0030f 100644 --- a/app/src/pages/tracing/__generated__/TracingHomePageQuery.graphql.ts +++ b/app/src/pages/tracing/__generated__/TracingHomePageQuery.graphql.ts @@ -1,5 +1,5 @@ /** - * @generated SignedSource<> + * @generated SignedSource<<7fc3b94ce8df8f09f6dbeeade49778e4>> * @lightSyntaxTransform * @nogrep */ @@ -52,48 +52,41 @@ v4 = { "storageKey": null }, v5 = { - "alias": null, - "args": null, - "kind": "ScalarField", - "name": "statusCode", - "storageKey": null -}, -v6 = { "alias": null, "args": null, "kind": "ScalarField", "name": "startTime", "storageKey": null }, -v7 = { +v6 = { "alias": null, "args": null, "kind": "ScalarField", "name": "latencyMs", "storageKey": null }, -v8 = { +v7 = { "alias": null, "args": null, "kind": "ScalarField", "name": "tokenCountTotal", "storageKey": null }, -v9 = { +v8 = { "alias": null, "args": null, "kind": "ScalarField", "name": "tokenCountPrompt", "storageKey": null }, -v10 = { +v9 = { "alias": null, "args": null, "kind": "ScalarField", "name": "tokenCountCompletion", "storageKey": null }, -v11 = { +v10 = { "alias": null, "args": null, "concreteType": "SpanContext", @@ -118,15 +111,15 @@ v11 = { ], "storageKey": null }, -v12 = { +v11 = { "alias": null, "args": null, "kind": "ScalarField", "name": "value", "storageKey": null }, -v13 = [ - (v12/*: any*/), +v12 = [ + (v11/*: any*/), { "alias": null, "args": null, @@ -135,7 +128,7 @@ v13 = [ "storageKey": null } ], -v14 = { +v13 = { "alias": null, "args": null, "concreteType": "SpanEvaluation", @@ -161,14 +154,14 @@ v14 = { ], "storageKey": null }, -v15 = { +v14 = { "alias": null, "args": null, "kind": "ScalarField", "name": "cursor", "storageKey": null }, -v16 = { +v15 = { "alias": null, "args": null, "concreteType": "Span", @@ -186,7 +179,7 @@ v16 = { ], "storageKey": null }, -v17 = { +v16 = { "alias": null, "args": null, "concreteType": "PageInfo", @@ -211,16 +204,23 @@ v17 = { ], "storageKey": null }, -v18 = { +v17 = { "kind": "Literal", "name": "rootSpansOnly", "value": true }, -v19 = [ +v18 = [ (v0/*: any*/), - (v18/*: any*/), + (v17/*: any*/), (v1/*: any*/) ], +v19 = { + "alias": "statusCode", + "args": null, + "kind": "ScalarField", + "name": "propagatedStatusCode", + "storageKey": null +}, v20 = { "alias": null, "args": null, @@ -229,7 +229,7 @@ v20 = { "storageKey": null }, v21 = [ - (v12/*: any*/) + (v11/*: any*/) ], v22 = { "alias": null, @@ -252,7 +252,7 @@ v23 = { "storageKey": null }, v24 = [ - (v18/*: any*/) + (v17/*: any*/) ], v25 = [ { @@ -337,13 +337,19 @@ return { "selections": [ (v3/*: any*/), (v4/*: any*/), + { + "alias": null, + "args": null, + "kind": "ScalarField", + "name": "statusCode", + "storageKey": null + }, (v5/*: any*/), (v6/*: any*/), (v7/*: any*/), (v8/*: any*/), (v9/*: any*/), (v10/*: any*/), - (v11/*: any*/), { "alias": null, "args": null, @@ -351,7 +357,7 @@ return { "kind": "LinkedField", "name": "input", "plural": false, - "selections": (v13/*: any*/), + "selections": (v12/*: any*/), "storageKey": null }, { @@ -361,19 +367,19 @@ return { "kind": "LinkedField", "name": "output", "plural": false, - "selections": (v13/*: any*/), + "selections": (v12/*: any*/), "storageKey": null }, - (v14/*: any*/) + (v13/*: any*/) ], "storageKey": null }, - (v15/*: any*/), - (v16/*: any*/) + (v14/*: any*/), + (v15/*: any*/) ], "storageKey": null }, - (v17/*: any*/) + (v16/*: any*/) ], "storageKey": "spans(first:100,sort:{\"col\":\"startTime\",\"dir\":\"desc\"})" }, @@ -391,7 +397,7 @@ return { }, { "alias": "rootSpans", - "args": (v19/*: any*/), + "args": (v18/*: any*/), "concreteType": "SpanConnection", "kind": "LinkedField", "name": "spans", @@ -415,9 +421,9 @@ return { "selections": [ (v3/*: any*/), (v4/*: any*/), + (v19/*: any*/), (v5/*: any*/), (v6/*: any*/), - (v7/*: any*/), { "alias": "tokenCountTotal", "args": null, @@ -442,8 +448,8 @@ return { (v20/*: any*/), (v22/*: any*/), (v23/*: any*/), - (v11/*: any*/), - (v14/*: any*/), + (v10/*: any*/), + (v13/*: any*/), { "alias": null, "args": null, @@ -454,35 +460,35 @@ return { "selections": [ (v3/*: any*/), (v4/*: any*/), + (v19/*: any*/), (v5/*: any*/), (v6/*: any*/), - (v7/*: any*/), (v20/*: any*/), + (v7/*: any*/), (v8/*: any*/), (v9/*: any*/), - (v10/*: any*/), (v22/*: any*/), (v23/*: any*/), - (v11/*: any*/), - (v14/*: any*/) + (v10/*: any*/), + (v13/*: any*/) ], "storageKey": null } ], "storageKey": null }, - (v15/*: any*/), - (v16/*: any*/) + (v14/*: any*/), + (v15/*: any*/) ], "storageKey": null }, - (v17/*: any*/) + (v16/*: any*/) ], "storageKey": "spans(first:100,rootSpansOnly:true,sort:{\"col\":\"startTime\",\"dir\":\"desc\"})" }, { "alias": "rootSpans", - "args": (v19/*: any*/), + "args": (v18/*: any*/), "filters": [ "sort", "rootSpansOnly", @@ -511,7 +517,7 @@ return { "name": "traceDatasetInfo", "plural": false, "selections": [ - (v6/*: any*/), + (v5/*: any*/), { "alias": null, "args": null, @@ -519,7 +525,7 @@ return { "name": "endTime", "storageKey": null }, - (v8/*: any*/), + (v7/*: any*/), { "alias": null, "args": null, @@ -550,12 +556,12 @@ return { ] }, "params": { - "cacheID": "54108ef1d02b43bca1dc407ab4a8bd80", + "cacheID": "9d35e485fa98fc88c776cd938d1ae101", "id": null, "metadata": {}, "name": "TracingHomePageQuery", "operationKind": "query", - "text": "query TracingHomePageQuery {\n ...SpansTable_spans\n ...TracesTable_spans\n ...TracingHomePageHeader_stats\n ...StreamToggle_data\n}\n\nfragment SpansTable_spans on Query {\n spans(first: 100, sort: {col: startTime, dir: desc}) {\n edges {\n span: node {\n spanKind\n name\n statusCode\n startTime\n latencyMs\n tokenCountTotal\n tokenCountPrompt\n tokenCountCompletion\n context {\n spanId\n traceId\n }\n input {\n value\n mimeType\n }\n output {\n value\n mimeType\n }\n spanEvaluations {\n name\n label\n score\n }\n }\n cursor\n node {\n __typename\n }\n }\n pageInfo {\n endCursor\n hasNextPage\n }\n }\n}\n\nfragment StreamToggle_data on Query {\n traceCount: spans(rootSpansOnly: true) {\n pageInfo {\n totalCount\n }\n }\n}\n\nfragment TracesTable_spans on Query {\n rootSpans: spans(first: 100, sort: {col: startTime, dir: desc}, rootSpansOnly: true) {\n edges {\n rootSpan: node {\n spanKind\n name\n statusCode\n startTime\n latencyMs\n tokenCountTotal: cumulativeTokenCountTotal\n tokenCountPrompt: cumulativeTokenCountPrompt\n tokenCountCompletion: cumulativeTokenCountCompletion\n parentId\n input {\n value\n }\n output {\n value\n }\n context {\n spanId\n traceId\n }\n spanEvaluations {\n name\n label\n score\n }\n descendants {\n spanKind\n name\n statusCode\n startTime\n latencyMs\n parentId\n tokenCountTotal\n tokenCountPrompt\n tokenCountCompletion\n input {\n value\n }\n output {\n value\n }\n context {\n spanId\n traceId\n }\n spanEvaluations {\n name\n label\n score\n }\n }\n }\n cursor\n node {\n __typename\n }\n }\n pageInfo {\n endCursor\n hasNextPage\n }\n }\n}\n\nfragment TracingHomePageHeader_stats on Query {\n totalTraces: spans(rootSpansOnly: true) {\n pageInfo {\n totalCount\n }\n }\n traceDatasetInfo {\n startTime\n endTime\n tokenCountTotal\n latencyMsP50\n latencyMsP99\n }\n}\n" + "text": "query TracingHomePageQuery {\n ...SpansTable_spans\n ...TracesTable_spans\n ...TracingHomePageHeader_stats\n ...StreamToggle_data\n}\n\nfragment SpansTable_spans on Query {\n spans(first: 100, sort: {col: startTime, dir: desc}) {\n edges {\n span: node {\n spanKind\n name\n statusCode\n startTime\n latencyMs\n tokenCountTotal\n tokenCountPrompt\n tokenCountCompletion\n context {\n spanId\n traceId\n }\n input {\n value\n mimeType\n }\n output {\n value\n mimeType\n }\n spanEvaluations {\n name\n label\n score\n }\n }\n cursor\n node {\n __typename\n }\n }\n pageInfo {\n endCursor\n hasNextPage\n }\n }\n}\n\nfragment StreamToggle_data on Query {\n traceCount: spans(rootSpansOnly: true) {\n pageInfo {\n totalCount\n }\n }\n}\n\nfragment TracesTable_spans on Query {\n rootSpans: spans(first: 100, sort: {col: startTime, dir: desc}, rootSpansOnly: true) {\n edges {\n rootSpan: node {\n spanKind\n name\n statusCode: propagatedStatusCode\n startTime\n latencyMs\n tokenCountTotal: cumulativeTokenCountTotal\n tokenCountPrompt: cumulativeTokenCountPrompt\n tokenCountCompletion: cumulativeTokenCountCompletion\n parentId\n input {\n value\n }\n output {\n value\n }\n context {\n spanId\n traceId\n }\n spanEvaluations {\n name\n label\n score\n }\n descendants {\n spanKind\n name\n statusCode: propagatedStatusCode\n startTime\n latencyMs\n parentId\n tokenCountTotal\n tokenCountPrompt\n tokenCountCompletion\n input {\n value\n }\n output {\n value\n }\n context {\n spanId\n traceId\n }\n spanEvaluations {\n name\n label\n score\n }\n }\n }\n cursor\n node {\n __typename\n }\n }\n pageInfo {\n endCursor\n hasNextPage\n }\n }\n}\n\nfragment TracingHomePageHeader_stats on Query {\n totalTraces: spans(rootSpansOnly: true) {\n pageInfo {\n totalCount\n }\n }\n traceDatasetInfo {\n startTime\n endTime\n tokenCountTotal\n latencyMsP50\n latencyMsP99\n }\n}\n" } }; })(); diff --git a/src/phoenix/core/traces.py b/src/phoenix/core/traces.py index 5441f26637..9f1f00bc4b 100644 --- a/src/phoenix/core/traces.py +++ b/src/phoenix/core/traces.py @@ -61,6 +61,8 @@ class ComputedAttributes(Enum): CUMULATIVE_LLM_TOKEN_COUNT_TOTAL = COMPUTED_PREFIX + "cumulative_token_count.total" CUMULATIVE_LLM_TOKEN_COUNT_PROMPT = COMPUTED_PREFIX + "cumulative_token_count.prompt" CUMULATIVE_LLM_TOKEN_COUNT_COMPLETION = COMPUTED_PREFIX + "cumulative_token_count.completion" + ERROR_COUNT = COMPUTED_PREFIX + "error_count" + CUMULATIVE_ERROR_COUNT = COMPUTED_PREFIX + "cumulative_error_count" class ReadableSpan(ObjectProxy): # type: ignore @@ -272,6 +274,9 @@ def _process_span(self, span: pb.Span) -> None: if self._max_start_time is None else max(self._max_start_time, start_time) ) + new_span[ComputedAttributes.ERROR_COUNT.value] = int( + span.status.code is pb.Span.Status.Code.ERROR + ) # Update cumulative values for span's ancestors. for attribute_name, cumulative_attribute_name in ( (LLM_TOKEN_COUNT_TOTAL, ComputedAttributes.CUMULATIVE_LLM_TOKEN_COUNT_TOTAL.value), @@ -280,6 +285,10 @@ def _process_span(self, span: pb.Span) -> None: LLM_TOKEN_COUNT_COMPLETION, ComputedAttributes.CUMULATIVE_LLM_TOKEN_COUNT_COMPLETION.value, ), + ( + ComputedAttributes.ERROR_COUNT.value, + ComputedAttributes.CUMULATIVE_ERROR_COUNT.value, + ), ): existing_value = (existing_span[attribute_name] or 0) if existing_span else 0 new_value = new_span[attribute_name] or 0 diff --git a/src/phoenix/server/api/types/Span.py b/src/phoenix/server/api/types/Span.py index 4de44d14a6..877f0c45fa 100644 --- a/src/phoenix/server/api/types/Span.py +++ b/src/phoenix/server/api/types/Span.py @@ -123,6 +123,10 @@ class Span: description="Cumulative (completion) token count from self and all " "descendant spans (children, grandchildren, etc.)", ) + propagated_status_code: SpanStatusCode = strawberry.field( + description="Propagated status code that percolates up error status " + "codes from descendant spans (children, grandchildren, etc.)", + ) @strawberry.field( description="Evaluations associated with the span, e.g. if the span is " @@ -222,6 +226,11 @@ def to_gql_span(span: trace_schema.Span) -> "Span": Optional[int], span.attributes.get(ComputedAttributes.CUMULATIVE_LLM_TOKEN_COUNT_COMPLETION.value), ), + propagated_status_code=( + SpanStatusCode.ERROR + if span.attributes.get(ComputedAttributes.CUMULATIVE_ERROR_COUNT.value) + else SpanStatusCode(span.status_code) + ), events=events, input=( SpanIOValue(