Skip to content

Commit

Permalink
improve(logs): roll level into the log line a instead of its own label
Browse files Browse the repository at this point in the history 
This will help with storage & performance
  • Loading branch information
@sdnts
sdnts committed Sep 16, 2023
1 parent c58d594 commit bf353b0
Show file tree
Hide file tree
Showing 3 changed files with 60 additions and 96 deletions.
27 changes: 9 additions & 18 deletions src/endpoints/logs.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,24 +48,14 @@ export const endpoint: Endpoint<typeof schema> = {
// Serialize logs to the Loki log entry format
// https://grafana.com/docs/loki/latest/reference/api/#push-log-entries-to-loki

const levelBuckets = params.data.logs.reduce(
(acc, l) => {
if (acc[l.level]) acc[l.level].push(l);
else acc[l.level] = [l];
return acc;
},
{} as Record<LogLevel, Log[]>,
);

const body = JSON.stringify({
streams: Object.entries(levelBuckets).map(([level, logs]) => {
return {
streams: [
{
stream: {
environment: params.data.environment,
service: params.data.service,
level,
},
values: logs.map((l) => {
values: params.data.logs.map((l) => {
// Loki wants timestamps in nanoseconds
if (l.timestamp.p === "ms")
l.timestamp.v = `${l.timestamp.v}000000`;
Expand All @@ -80,21 +70,22 @@ export const endpoint: Endpoint<typeof schema> = {
// Reference: https://grafana.com/docs/loki/latest/get-started/labels/structured-metadata/
// TODO: We should just use structured metadata once it lands as stable

const log = Object.entries({
const line = Object.entries({
level: l.level,
...params.data.kv,
...l.kv,
msg: l.message,
message: l.message,
})
.filter(([_, v]) => v !== undefined) // Leave null values untouched
.map(([k, v]) => {
if (typeof v === "string") return `${k}="${v}"`;
return `${k}=${v}`;
})
.join(" ");
return [l.timestamp.v, log];
return [l.timestamp.v, line];
}),
};
}),
},
],
});

return fetch(env.lokiUrl, {
Expand Down
70 changes: 29 additions & 41 deletions tests/logs.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,8 @@ test("with environment", async () => {
stream: {
environment: "staging",
service: "blob-city",
level: "info",
},
values: [["001", 'msg="Incoming request"']],
values: [["001", 'level="info" message="Incoming request"']],
},
],
});
Expand Down Expand Up @@ -64,9 +63,8 @@ test("with level", async () => {
stream: {
environment: "production",
service: "blob-city",
level: "fatal",
},
values: [["001", 'msg="Incoming request"']],
values: [["001", 'level="fatal" message="Incoming request"']],
},
],
});
Expand Down Expand Up @@ -99,9 +97,8 @@ test("with ms precision", async () => {
stream: {
environment: "production",
service: "blob-city",
level: "info",
},
values: [["001000000", 'msg="Incoming request"']],
values: [["001000000", 'level="info" message="Incoming request"']],
},
],
});
Expand Down Expand Up @@ -133,9 +130,8 @@ test("with ns precision", async () => {
stream: {
environment: "production",
service: "blob-city",
level: "info",
},
values: [["001", 'msg="Incoming request"']],
values: [["001", 'level="info" message="Incoming request"']],
},
],
});
Expand Down Expand Up @@ -168,9 +164,8 @@ test("without kv", async () => {
stream: {
environment: "production",
service: "blob-city",
level: "info",
},
values: [["001", 'msg="Incoming request"']],
values: [["001", 'level="info" message="Incoming request"']],
},
],
});
Expand Down Expand Up @@ -204,9 +199,10 @@ test("with line kv", async () => {
stream: {
environment: "production",
service: "blob-city",
level: "info",
},
values: [["001", 'method="GET" msg="Incoming request"']],
values: [
["001", 'level="info" method="GET" message="Incoming request"'],
],
},
],
});
Expand Down Expand Up @@ -240,9 +236,10 @@ test("with common kv", async () => {
stream: {
environment: "production",
service: "blob-city",
level: "info",
},
values: [["001", 'rayId="1234" msg="Incoming request"']],
values: [
["001", 'level="info" rayId="1234" message="Incoming request"'],
],
},
],
});
Expand Down Expand Up @@ -277,9 +274,13 @@ test("with both kv", async () => {
stream: {
environment: "production",
service: "blob-city",
level: "info",
},
values: [["001", 'rayId="1234" method="GET" msg="Incoming request"']],
values: [
[
"001",
'level="info" rayId="1234" method="GET" message="Incoming request"',
],
],
},
],
});
Expand Down Expand Up @@ -314,9 +315,8 @@ test("with undefined kv values", async () => {
stream: {
environment: "production",
service: "blob-city",
level: "info",
},
values: [["001", 'msg="Incoming request"']],
values: [["001", 'level="info" message="Incoming request"']],
},
],
});
Expand Down Expand Up @@ -349,9 +349,8 @@ test("with no message", async () => {
stream: {
environment: "production",
service: "blob-city",
level: "info",
},
values: [["001", ""]],
values: [["001", 'level="info"']],
},
],
});
Expand Down Expand Up @@ -403,34 +402,23 @@ test("multiple logs", async () => {
stream: {
environment: "production",
service: "blob-city",
level: "info",
},
values: [
[
"001",
'rayId="abcd" method="GET" path="/tunnel" msg="Incoming request"',
'level="info" rayId="abcd" method="GET" path="/tunnel" message="Incoming request"',
],
["004", 'rayId="abcd" status=200 msg="Response"'],
],
},
{
stream: {
environment: "production",
service: "blob-city",
level: "debug",
},
values: [
["002", 'rayId="abcd" tunnelId="1234" msg="Forwarding to DO"'],
[
"002",
'level="debug" rayId="abcd" tunnelId="1234" message="Forwarding to DO"',
],
[
"003000000",
'level="trace" rayId="abcd" message="Creating WebSocketPair"',
],
["004", 'level="info" rayId="abcd" status=200 message="Response"'],
],
},
{
stream: {
environment: "production",
service: "blob-city",
level: "trace",
},
values: [["003000000", 'rayId="abcd" msg="Creating WebSocketPair"']],
},
],
});
});
59 changes: 22 additions & 37 deletions tests/tail.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,12 @@ test("shipping error logs", async () => {
expect(streams[0].stream).toStrictEqual({
environment: "production",
service: "ingest-worker",
level: "fatal",
});
expect(streams[0].values).toHaveLength(1);
expect(streams[0].values[0]).toHaveLength(2);
expect(streams[0].values[0][0]).toBe("1000000");
expect(streams[0].values[0][1]).toMatch(
/name="Error" stack="Error: Missing scriptName(.|\n)*" msg="Missing scriptName"/,
/level="fatal" name="Error" stack="Error: Missing scriptName(.|\n)*" message="Missing scriptName"/,
);

vi.useRealTimers();
Expand Down Expand Up @@ -86,9 +85,13 @@ test("extra log when outcome is not ok", async () => {
stream: {
environment: "production",
service: "blob-city-api",
level: "fatal",
},
values: [["1000000", 'outcome="cpuExceeded" msg="Fatal outcome"']],
values: [
[
"1000000",
'level="fatal" outcome="cpuExceeded" message="Fatal outcome"',
],
],
},
],
});
Expand Down Expand Up @@ -139,25 +142,15 @@ test("script logs", async () => {
stream: {
environment: "production",
service: "blob-city-api",
level: "info",
},
values: [["1000000", 'msg="log1"']],
},
{
stream: {
environment: "production",
service: "blob-city-api",
level: "warn",
},
values: [["2000000", 'a="b" c=9 d=false msg="log2 extra params"']],
},
{
stream: {
environment: "production",
service: "blob-city-api",
level: "fatal",
},
values: [["3000000", 'msg="log3 extra params"']],
values: [
["1000000", 'level="info" message="log1"'],
[
"2000000",
'level="warn" a="b" c=9 d=false message="log2 extra params"',
],
["3000000", 'level="fatal" message="log3 extra params"'],
],
},
],
});
Expand Down Expand Up @@ -203,11 +196,10 @@ test("script exceptions", async () => {
stream: {
environment: "production",
service: "blob-city-api",
level: "error",
},
values: [
["1000000", 'msg="exception1"'],
["2000000", 'msg="exception2"'],
["1000000", 'level="error" message="exception1"'],
["2000000", 'level="error" message="exception2"'],
],
},
],
Expand Down Expand Up @@ -275,17 +267,11 @@ test("event batch", async () => {
stream: {
environment: "production",
service: "blob-city-api",
level: "info",
},
values: [["1000000", 'msg="log1"']],
},
{
stream: {
environment: "production",
service: "blob-city-api",
level: "warn",
},
values: [["2000000", 'msg="log2"']],
values: [
["1000000", 'level="info" message="log1"'],
["2000000", 'level="warn" message="log2"'],
],
},
],
});
Expand All @@ -295,9 +281,8 @@ test("event batch", async () => {
stream: {
environment: "production",
service: "blob-city-api",
level: "error",
},
values: [["3000000", 'msg="exception1"']],
values: [["3000000", 'level="error" message="exception1"']],
},
],
});
Expand Down

0 comments on commit bf353b0

Please sign in to comment.