From 3f947db91c7935fb996ca3840b79769ee5993fe6 Mon Sep 17 00:00:00 2001 From: Ray Chen Date: Thu, 23 Mar 2023 04:49:59 +0800 Subject: [PATCH 1/7] Refactor --- .ignore | 2 + package.json | 6 +- src/{client/http.ts => api/http/client.ts} | 12 +- src/api/http/get-project-data.ts | 51 ++++++ src/{queries/gql.ts => api/http/queries.ts} | 44 +---- src/{client/ws.ts => api/websocket/client.ts} | 5 +- src/api/websocket/queries.ts | 43 +++++ .../websocket/subscribe-to-deployment-logs.ts | 21 +++ src/api/websocket/subscribe-to-plugin-logs.ts | 22 +++ src/api/websocket/subscribe.ts | 70 ++++++++ src/logger.ts | 116 -------------- src/main.ts | 132 +++++++++++---- src/push-deployment-logs.ts | 41 +++++ src/push-plugin-logs.ts | 42 +++++ src/queries/calls.ts | 150 ------------------ src/{types/queries.ts => types.d.ts} | 50 +++++- src/types/app.ts | 37 ----- src/utils.ts | 22 --- src/utils/bytes2utf8.ts | 5 + src/utils/get-env.ts | 6 + src/utils/parse-project-ids.ts | 6 + src/utils/require-env.ts | 9 ++ src/vector.ts | 35 ---- src/vector/configure.ts | 29 ++++ src/vector/sinks.ts | 25 +++ src/vector/sources.ts | 6 + src/vector/spawn.ts | 32 ++++ src/vector/write.ts | 9 ++ tsconfig.json | 5 +- yarn.lock | 120 ++++++++++++++ 30 files changed, 698 insertions(+), 455 deletions(-) create mode 100644 .ignore rename src/{client/http.ts => api/http/client.ts} (53%) create mode 100644 src/api/http/get-project-data.ts rename src/{queries/gql.ts => api/http/queries.ts} (52%) rename src/{client/ws.ts => api/websocket/client.ts} (80%) create mode 100644 src/api/websocket/queries.ts create mode 100644 src/api/websocket/subscribe-to-deployment-logs.ts create mode 100644 src/api/websocket/subscribe-to-plugin-logs.ts create mode 100644 src/api/websocket/subscribe.ts delete mode 100644 src/logger.ts create mode 100644 src/push-deployment-logs.ts create mode 100644 src/push-plugin-logs.ts delete mode 100644 src/queries/calls.ts rename src/{types/queries.ts => types.d.ts} (68%) delete mode 100644 src/types/app.ts delete mode 100644 src/utils.ts create mode 100644 src/utils/bytes2utf8.ts create mode 100644 src/utils/get-env.ts create mode 100644 src/utils/parse-project-ids.ts create mode 100644 src/utils/require-env.ts delete mode 100644 src/vector.ts create mode 100644 src/vector/configure.ts create mode 100644 src/vector/sinks.ts create mode 100644 src/vector/sources.ts create mode 100644 src/vector/spawn.ts create mode 100644 src/vector/write.ts diff --git a/.ignore b/.ignore new file mode 100644 index 0000000..85dcc16 --- /dev/null +++ b/.ignore @@ -0,0 +1,2 @@ +.git +node_modules diff --git a/package.json b/package.json index a936118..1e754ab 100644 --- a/package.json +++ b/package.json @@ -6,13 +6,15 @@ "version": "1.0.0", "license": "MIT", "scripts": { - "run-dev": "ts-node src/main.ts", + "run-dev": "ts-node -r tsconfig-paths/register src/main.ts", "build": "tsc --build --verbose" }, "devDependencies": { "@types/node": "^18.15.3", + "@types/tmp": "^0.2.3", "@types/ws": "^8.5.4", "ts-node": "^10.9.1", + "tsconfig-paths": "^4.1.2", "typescript": "^5.0.2" }, "dependencies": { @@ -20,6 +22,8 @@ "dotenv": "^16.0.3", "graphql": "^16.6.0", "graphql-ws": "^5.12.0", + "tmp": "^0.2.1", + "ts-pattern": "^4.2.1", "ws": "^8.13.0" } } diff --git a/src/client/http.ts b/src/api/http/client.ts similarity index 53% rename from src/client/http.ts rename to src/api/http/client.ts index dde6a2b..497737d 100644 --- a/src/client/http.ts +++ b/src/api/http/client.ts @@ -1,16 +1,10 @@ -import { - ApolloClient, - InMemoryCache, - NormalizedCacheObject, -} from '@apollo/client/core' +import { HttpClient } from '@/types' +import { ApolloClient, InMemoryCache } from '@apollo/client/core' /** * Creates an authenticated GQL HTTP client. */ -const createHttpClient = ( - endpoint: string, - apiToken: string, -): ApolloClient => { +const createHttpClient = (endpoint: string, apiToken: string): HttpClient => { return new ApolloClient({ uri: endpoint, cache: new InMemoryCache(), diff --git a/src/api/http/get-project-data.ts b/src/api/http/get-project-data.ts new file mode 100644 index 0000000..2e89940 --- /dev/null +++ b/src/api/http/get-project-data.ts @@ -0,0 +1,51 @@ +import { ProjectQuery } from '@/api/http/queries' +import { App, QueryResponse } from '@/types' +import { + ApolloClient as GqlHttpClient, + NormalizedCacheObject, +} from '@apollo/client/core' + +/** + * Returns a `projectId`'s plugins and its latest deployment of services (in + * any environment). + */ +const getProjectData = async ( + client: GqlHttpClient, + projectId: App.ProjectId, +): Promise => { + const res = await client.query({ + query: ProjectQuery, + variables: { + projectId, + }, + }) + const project = res.data.project + + // Each Plugin has its own Environment-specific instance, which makes this + // unique by a `(environmentId, pluginId)` tuple. + const plugins: App.Plugin[] = project.environments.edges.flatMap((e) => { + return project.plugins.edges.map((p) => { + return { + id: p.node.id, + name: p.node.name, + environmentId: e.node.id, + environmentName: e.node.name, + } + }) + }) + + // Latest Deployment of each Service in the Project. + const deployments: App.Deployment[] = project.services.edges.flatMap((s) => { + return s.node.deployments.edges.map((d) => { + return { + id: d.node.id, + staticUrl: d.node.staticUrl, + serviceId: s.node.id, + } + }) + }) + + return Promise.resolve({ projectId, plugins, deployments }) +} + +export default getProjectData diff --git a/src/queries/gql.ts b/src/api/http/queries.ts similarity index 52% rename from src/queries/gql.ts rename to src/api/http/queries.ts index 57f2726..1030187 100644 --- a/src/queries/gql.ts +++ b/src/api/http/queries.ts @@ -1,47 +1,5 @@ import { gql } from '@apollo/client/core' -// Subscribe to DeploymentLogs -const DeploymentLogs = ` -subscription DeploymentLogs( - $deploymentId: String! - $filter: String - $limit: Int -) { - deploymentLogs(deploymentId: $deploymentId, filter: $filter, limit: $limit) { - ...LogFields - } -} - -fragment LogFields on Log { - timestamp - message -} -` - -// Subscribe to PluginLogs -const PluginLogs = ` -subscription PluginLogs( - $pluginId: String! - $environmentId: String! - $filter: String - $limit: Int -) { - pluginLogs( - pluginId: $pluginId, - environmentId: $environmentId, - filter: $filter, - limit: $limit - ) { - ...LogFields - } -} - -fragment LogFields on Log { - timestamp - message -} -` - // Fetch information about a project, its plugins, services, and deployments const ProjectQuery = gql` query project($projectId: String!) { @@ -84,4 +42,4 @@ const ProjectQuery = gql` } ` -export { ProjectQuery, DeploymentLogs, PluginLogs } +export { ProjectQuery } diff --git a/src/client/ws.ts b/src/api/websocket/client.ts similarity index 80% rename from src/client/ws.ts rename to src/api/websocket/client.ts index 8ed1e40..91f5460 100644 --- a/src/client/ws.ts +++ b/src/api/websocket/client.ts @@ -1,10 +1,11 @@ -import { Client, createClient } from 'graphql-ws' +import { WsClient } from '@/types' +import { createClient } from 'graphql-ws' import WebSocket from 'ws' /** * Creates an authenticated GQL WebSocket client. */ -const createWsClient = (endpoint: string, apiToken: string): Client => { +const createWsClient = (endpoint: string, apiToken: string): WsClient => { class AuthenticatedWebSocket extends WebSocket { constructor(address: string, protocols: string[]) { super(address, protocols, { diff --git a/src/api/websocket/queries.ts b/src/api/websocket/queries.ts new file mode 100644 index 0000000..c1194b5 --- /dev/null +++ b/src/api/websocket/queries.ts @@ -0,0 +1,43 @@ +// Subscribe to DeploymentLogs +const DeploymentLogs = ` +subscription DeploymentLogs( + $deploymentId: String! + $filter: String + $limit: Int +) { + deploymentLogs(deploymentId: $deploymentId, filter: $filter, limit: $limit) { + ...LogFields + } +} + +fragment LogFields on Log { + timestamp + message +} +` + +// Subscribe to PluginLogs +const PluginLogs = ` +subscription PluginLogs( + $pluginId: String! + $environmentId: String! + $filter: String + $limit: Int +) { + pluginLogs( + pluginId: $pluginId, + environmentId: $environmentId, + filter: $filter, + limit: $limit + ) { + ...LogFields + } +} + +fragment LogFields on Log { + timestamp + message +} +` + +export { DeploymentLogs, PluginLogs } diff --git a/src/api/websocket/subscribe-to-deployment-logs.ts b/src/api/websocket/subscribe-to-deployment-logs.ts new file mode 100644 index 0000000..1a1d229 --- /dev/null +++ b/src/api/websocket/subscribe-to-deployment-logs.ts @@ -0,0 +1,21 @@ +import { DeploymentLogs } from '@/api/websocket/queries' +import wsSubscribe from '@/api/websocket/subscribe' +import { QueryResponse } from '@/types' +import { Client as GqlWsClient, ExecutionResult } from 'graphql-ws' + +/** + * Returns an async-iterable subscription to deployment logs. + */ +const subscribeToDeploymentLogs = ( + client: GqlWsClient, + deploymentId: string, +): AsyncGenerator< + ExecutionResult +> => { + return wsSubscribe(client, { + query: DeploymentLogs, + variables: { deploymentId }, + }) +} + +export default subscribeToDeploymentLogs diff --git a/src/api/websocket/subscribe-to-plugin-logs.ts b/src/api/websocket/subscribe-to-plugin-logs.ts new file mode 100644 index 0000000..7633994 --- /dev/null +++ b/src/api/websocket/subscribe-to-plugin-logs.ts @@ -0,0 +1,22 @@ +import { PluginLogs } from '@/api/websocket/queries' +import wsSubscribe from '@/api/websocket/subscribe' +import { QueryResponse } from '@/types' +import { Client as GqlWsClient, ExecutionResult } from 'graphql-ws' + +/** + * Returns an async-iterable subscription to plugin logs. + */ +const subscribeToPluginLogs = ( + client: GqlWsClient, + pluginId: string, + environmentId: string, +): AsyncGenerator< + ExecutionResult +> => { + return wsSubscribe(client, { + query: PluginLogs, + variables: { pluginId, environmentId }, + }) +} + +export default subscribeToPluginLogs diff --git a/src/api/websocket/subscribe.ts b/src/api/websocket/subscribe.ts new file mode 100644 index 0000000..10239d1 --- /dev/null +++ b/src/api/websocket/subscribe.ts @@ -0,0 +1,70 @@ +import { + Client as GqlWsClient, + ExecutionResult, + SubscribePayload, +} from 'graphql-ws' +import { Deferred } from '@/types' + +/** + * Returns an async-iterable stream of a GQL subscription. + * + * Usage: + * ```typescript + * const subscription = subscribe({ ... }) + * for await (const result of subscription) { + * ... + * } + * ``` + * + * Based on this example from `graphql-ws`: + * https://github.com/enisdenjo/graphql-ws#async-iterator + */ +const wsSubscribe = ( + client: GqlWsClient, + payload: SubscribePayload, +): AsyncGenerator> => { + const pending: ExecutionResult[] = [] + let deferred: Deferred | null = null + let throwMe: unknown = null, + done = false + + const dispose = client.subscribe(payload, { + next: (data) => { + pending.push(data) + deferred?.resolve(false) + }, + error: (err) => { + throwMe = err + deferred?.reject(throwMe) + }, + complete: () => { + done = true + deferred?.resolve(true) + }, + }) + + return { + [Symbol.asyncIterator]() { + return this + }, + async next() { + if (done) return { done: true, value: undefined } + if (throwMe) throw throwMe + if (pending.length) return { value: pending.shift()! } + return (await new Promise( + (resolve, reject) => (deferred = { resolve, reject }), + )) + ? { done: true, value: undefined } + : { value: pending.shift()! } + }, + async throw(err) { + throw err + }, + async return() { + dispose() + return { done: true, value: undefined } + }, + } +} + +export default wsSubscribe diff --git a/src/logger.ts b/src/logger.ts deleted file mode 100644 index 619d8c9..0000000 --- a/src/logger.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { - ApolloClient as GqlHttpClient, - NormalizedCacheObject, -} from '@apollo/client/core' -import { Client as GqlWsClient } from 'graphql-ws' -import { - getProjectData, - subscribeToDeploymentLogs, - subscribeToPluginLogs, -} from './queries/calls' -import App, { VectorProcess } from './types/app' -import { write } from './vector' - -/** - * Pushes the Deployment and Plugin logs of `projectIds` into Vector. - */ -const run = async ( - httpClient: GqlHttpClient, - wsClient: GqlWsClient, - vector: VectorProcess, - projectIds: App.ProjectId[], -) => { - console.info(`🔄 Refreshing projects!`) - const state = await Promise.all( - projectIds.map(async (id) => { - const project = await getProjectData(httpClient, id) - return { - projectId: id, - plugins: project.plugins, - deployments: project.deployments, - } - }), - ) - - console.info(`✅ Enabling for:`) - state.forEach(async ({ deployments, plugins, projectId }) => { - console.info(` > projectId=${projectId}`) - deployments.forEach(async (d) => { - console.info(` - deployment=${d.staticUrl}, deploymentId=${d.id}`) - pushDeploymentLogs(wsClient, vector, d) - }) - plugins.forEach(async (p) => { - console.info( - ` - plugin=${p.name}, pluginId=${p.id}, env=${p.environmentName}`, - ) - pushPluginLogs(wsClient, vector, p) - }) - }) -} - -/** - * Write Deployment logs to Vector. - */ -const pushDeploymentLogs = async ( - wsClient: GqlWsClient, - vector: VectorProcess, - deployment: App.Deployment, -) => { - try { - for await (const result of subscribeToDeploymentLogs( - wsClient, - deployment.id, - )) { - result.data?.deploymentLogs.forEach((log) => { - const out = { - railway: { - type: 'DEPLOYMENT', - name: deployment.staticUrl, - id: deployment.id, - environment: null, // @TODO - }, - ...log, - } - write(vector, JSON.stringify(out)) - }) - } - } catch (e) { - console.error('Error reading deployment logs', e) - process.exit(1) - } -} - -/** - * Write Plugin logs to Vector. - */ -const pushPluginLogs = async ( - wsClient: GqlWsClient, - vector: VectorProcess, - plugin: App.Plugin, -) => { - try { - for await (const result of subscribeToPluginLogs( - wsClient, - plugin.id, - plugin.environmentId, - )) { - result.data?.pluginLogs.forEach((log) => { - const out = { - railway: { - type: 'PLUGIN', - name: plugin.name, - id: plugin.id, - environment: plugin.environmentName, - }, - ...log, - } - write(vector, JSON.stringify(out)) - }) - } - } catch (e) { - console.error('Error reading plugin logs', e) - process.exit(1) - } -} - -export default run diff --git a/src/main.ts b/src/main.ts index 91e2226..2e1aa27 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,9 +1,15 @@ +import createHttpClient from '@/api/http/client' +import getProjectData from '@/api/http/get-project-data' +import createWsClient from '@/api/websocket/client' +import pushDeploymentLogs from '@/push-deployment-logs' +import pushPluginLogs from '@/push-plugin-logs' +import { App, HttpClient, VectorProcess, WsClient } from '@/types' +import getEnv from '@/utils/get-env' +import parseProjectIds from '@/utils/parse-project-ids' +import requireEnv from '@/utils/require-env' +import spawn from '@/vector/spawn' +import write from '@/vector/write' import dotenv from 'dotenv' -import createHttpClient from './client/http' -import createWsClient from './client/ws' -import run from './logger' -import { getEnv, parseProjectIds, requireEnv } from './utils' -import { spawn, write } from './vector' dotenv.config() @@ -15,66 +21,122 @@ const RAILWAY_API_WS_ENDPOINT = getEnv( 'RAILWAY_API_WS_ENDPOINT', 'wss://backboard.railway.app/graphql/v2', ) + const RAILWAY_PROJECT_IDS = requireEnv('RAILWAY_PROJECT_IDS') const RAILWAY_API_TOKEN = requireEnv('RAILWAY_API_TOKEN') const VECTOR_BIN_PATH = requireEnv('VECTOR_BIN_PATH') -const VECTOR_CFG_PATH = requireEnv('VECTOR_CFG_PATH') // @TODO: Move this to env? const REFRESH_INTERVAL_SECONDS = 60 * 15 +import configureVector from './vector/configure' + +/** + * This is the main event loop that refreshes a project's deployments and + * plugins every n seconds (where n=`REFRESH_INTERVAL_SECONDS`) and pushes + * the logs of each deployment/plugin into Vector. + * + * The major limitation of this approach is Railway's rate limit - each + * account is limited to 1k requests per day [0]. At the current default + * refresh interval of 15 mins, that works out to 96 requests every 24 + * hours for project data per-project. I'm not sure if the WS subscriptions + * are subject to this rate limit. In theory, rate limits should apply + * per-request instead of per-WS message, which implies each WS connection + * counts toward the rate limit. This effectively means that we're making + * `(1 + (services+plugins)) * (86400 / REFRESH_INTERVAL_SECONDS)` requests + * every 24hours for each project. + * + * An alternative/better way of doing this is through Railway's webhooks. + * We can subscribe to new deployments. However, this approach misses out + * on plugin creation/deletion: when a plugin is created, logs will only + * get pushed after restarting this service. Conversely, when a plugin is + * deleted, we'll have no way of knowing (would subscribing to logs of a + * deleted plugin error out?). + * + * [0] https://docs.railway.app/reference/public-api#rate-limits + */ const main = async () => { console.info(`⚡ railway-chord is starting!`) + // Vector sinks are configured dynamically based on the presence of a sink's + // API token in env. i.e. if there's a LOGTAIL_TOKEN provided, inject the + // Logtail sink into Vector config; if there's a DATADOG_TOKEN provided, + // inject the Datadog sink into Vector config; and so on. + const ENABLE_STDOUT = process.env.ENABLE_STDOUT === 'true' ? true : false + const LOGTAIL_TOKEN = process.env.LOGTAIL_TOKEN ?? null + const vectorCfg = configureVector(ENABLE_STDOUT, LOGTAIL_TOKEN) + // Start Vector first. We want to crash early; there's no point in making // network requests to Railway API if Vector can't start. console.info(`⚙️ Using Vector binary: ${VECTOR_BIN_PATH}`) - console.info(`⚙️ Using Vector config: ${VECTOR_CFG_PATH}`) - const vector = spawn(VECTOR_BIN_PATH, VECTOR_CFG_PATH) + const vector = spawn(VECTOR_BIN_PATH, vectorCfg.contents) + write(vector, '>>> ping from railway-chord') console.info(`✅ Vector started`) + console.info(`✅ Enabled sinks:`) + vectorCfg.enabled.forEach((s) => { + console.info(` - ${s}`) + }) + console.info(`⚙️ Using Railway HTTP endpoint: ${RAILWAY_API_HTTP_ENDPOINT}`) console.info( `⚙️ Using Railway WebSockets endpoint: ${RAILWAY_API_WS_ENDPOINT}`, ) const projectIds = parseProjectIds(RAILWAY_PROJECT_IDS) - const httpClient = createHttpClient( RAILWAY_API_HTTP_ENDPOINT, RAILWAY_API_TOKEN, ) const wsClient = createWsClient(RAILWAY_API_WS_ENDPOINT, RAILWAY_API_TOKEN) - /** - * This is the main event loop that refreshes a project's deployments and - * plugins every n seconds (where n=`REFRESH_INTERVAL_SECONDS`) and pushes - * the logs of each deployment/plugin into Vector. - * - * The major limitation of this approach is Railway's rate limit - each - * account is limited to 1k requests per day [0]. At the current default - * refresh interval of 15 mins, that works out to 96 requests every 24 - * hours for project data per-project. I'm not sure if the WS subscriptions - * are subject to this rate limit. In theory, rate limits should apply - * per-request instead of per-WS message, which implies each WS connection - * counts toward the rate limit. This effectively means that we're making - * `(1 + (services+plugins)) * (86400 / REFRESH_INTERVAL_SECONDS)` requests - * every 24hours for each project. - * - * An alternative/better way of doing this is through Railway's webhooks. - * We can subscribe to new deployments. However, this approach misses out - * on plugin creation/deletion: when a plugin is created, logs will only - * get pushed after restarting this service. Conversely, when a plugin is - * deleted, we'll have no way of knowing (would subscribing to logs of a - * deleted plugin error out?). - * - * [0] https://docs.railway.app/reference/public-api#rate-limits - */ - await run(httpClient, wsClient, vector, projectIds) + // Start event loop + await runEventLoop(httpClient, wsClient, vector, projectIds) setInterval(async () => { - await run(httpClient, wsClient, vector, projectIds) + await runEventLoop(httpClient, wsClient, vector, projectIds) }, REFRESH_INTERVAL_SECONDS * 1000) } +const refreshProjects = async ( + httpClient: HttpClient, + projectIds: App.ProjectId[], +) => { + return await Promise.all( + projectIds.map(async (id) => { + const project = await getProjectData(httpClient, id) + return { + projectId: id, + plugins: project.plugins, + deployments: project.deployments, + } + }), + ) +} + +const runEventLoop = async ( + httpClient: HttpClient, + wsClient: WsClient, + vector: VectorProcess, + projectIds: App.ProjectId[], +) => { + console.info(`🔄 Refreshing projects!`) + const projects = await refreshProjects(httpClient, projectIds) + + console.info(`✅ Enabling for:`) + projects.forEach(async ({ deployments, plugins, projectId }) => { + console.info(` > projectId=${projectId}`) + deployments.forEach(async (d) => { + console.info(` - deployment=${d.staticUrl}, deploymentId=${d.id}`) + pushDeploymentLogs(wsClient, vector, d) + }) + plugins.forEach(async (p) => { + console.info( + ` - plugin=${p.name}, pluginId=${p.id}, env=${p.environmentName}`, + ) + pushPluginLogs(wsClient, vector, p) + }) + }) +} + main() diff --git a/src/push-deployment-logs.ts b/src/push-deployment-logs.ts new file mode 100644 index 0000000..947e74e --- /dev/null +++ b/src/push-deployment-logs.ts @@ -0,0 +1,41 @@ +import subscribeToDeploymentLogs from '@/api/websocket/subscribe-to-deployment-logs' +import { App, VectorProcess } from '@/types' +import write from '@/vector/write' +import { Client as GqlWsClient } from 'graphql-ws' + +/** + * Opens a subscription to Railway's deployment logs API, and pushes the + * responses into Vector. + */ +const pushDeploymentLogs = async ( + wsClient: GqlWsClient, + vector: VectorProcess, + deployment: App.Deployment, +) => { + try { + for await (const result of subscribeToDeploymentLogs( + wsClient, + deployment.id, + )) { + result.data?.deploymentLogs.forEach((log) => { + const out = { + railway: { + type: 'DEPLOYMENT', + name: deployment.staticUrl, + id: deployment.id, + environment: null, // @TODO + }, + ...log, + } + write(vector, JSON.stringify(out)) + }) + } + } catch (e) { + // @TODO This needs some re-try logic. If there's a momentary API error, + // this will crash the service (intentional for now). + console.error('Error reading deployment logs', e) + process.exit(1) + } +} + +export default pushDeploymentLogs diff --git a/src/push-plugin-logs.ts b/src/push-plugin-logs.ts new file mode 100644 index 0000000..6d2c799 --- /dev/null +++ b/src/push-plugin-logs.ts @@ -0,0 +1,42 @@ +import subscribeToPluginLogs from '@/api/websocket/subscribe-to-plugin-logs' +import { App, VectorProcess } from '@/types' +import write from '@/vector/write' +import { Client as GqlWsClient } from 'graphql-ws' + +/** + * Opens a subscription to Railway's plugin logs API, and pushes the + * responses into Vector. + */ +const pushPluginLogs = async ( + wsClient: GqlWsClient, + vector: VectorProcess, + plugin: App.Plugin, +) => { + try { + for await (const result of subscribeToPluginLogs( + wsClient, + plugin.id, + plugin.environmentId, + )) { + result.data?.pluginLogs.forEach((log) => { + const out = { + railway: { + type: 'PLUGIN', + name: plugin.name, + id: plugin.id, + environment: plugin.environmentName, + }, + ...log, + } + write(vector, JSON.stringify(out)) + }) + } + } catch (e) { + // @TODO This needs some re-try logic. If there's a momentary API error, + // this will crash the service (intentional for now). + console.error('Error reading plugin logs', e) + process.exit(1) + } +} + +export default pushPluginLogs diff --git a/src/queries/calls.ts b/src/queries/calls.ts deleted file mode 100644 index 1c655cb..0000000 --- a/src/queries/calls.ts +++ /dev/null @@ -1,150 +0,0 @@ -import { - ApolloClient as GqlHttpClient, - NormalizedCacheObject, -} from '@apollo/client/core' -import { - Client as GqlWsClient, - ExecutionResult, - SubscribePayload, -} from 'graphql-ws' -import { DeploymentLogs, PluginLogs, ProjectQuery } from './gql' -import QueryResponse from '../types/queries' -import App, { Deferred } from '../types/app' - -/** - * Returns an async-iterable stream of a GQL subscription. - * - * Usage: - * ```typescript - * const subscription = subscribe({ ... }) - * for await (const result of subscription) { - * ... - * } - * ``` - * - * Based on this example from `graphql-ws`: - * https://github.com/enisdenjo/graphql-ws#async-iterator - */ -const wsSubscribe = ( - client: GqlWsClient, - payload: SubscribePayload, -): AsyncGenerator> => { - const pending: ExecutionResult[] = [] - let deferred: Deferred | null = null - let throwMe: unknown = null, - done = false - - const dispose = client.subscribe(payload, { - next: (data) => { - pending.push(data) - deferred?.resolve(false) - }, - error: (err) => { - throwMe = err - deferred?.reject(throwMe) - }, - complete: () => { - done = true - deferred?.resolve(true) - }, - }) - - return { - [Symbol.asyncIterator]() { - return this - }, - async next() { - if (done) return { done: true, value: undefined } - if (throwMe) throw throwMe - if (pending.length) return { value: pending.shift()! } - return (await new Promise( - (resolve, reject) => (deferred = { resolve, reject }), - )) - ? { done: true, value: undefined } - : { value: pending.shift()! } - }, - async throw(err) { - throw err - }, - async return() { - dispose() - return { done: true, value: undefined } - }, - } -} - -/** - * Returns a `projectId`'s plugins and its latest deployment of services (in - * any environment). - */ -const getProjectData = async ( - client: GqlHttpClient, - projectId: App.ProjectId, -): Promise => { - const res = await client.query({ - query: ProjectQuery, - variables: { - projectId, - }, - }) - const project = res.data.project - - // Each Plugin has its own Environment-specific instance, which makes this - // unique by a `(environmentId, pluginId)` tuple. - const plugins: App.Plugin[] = project.environments.edges.flatMap((e) => { - return project.plugins.edges.map((p) => { - return { - id: p.node.id, - name: p.node.name, - environmentId: e.node.id, - environmentName: e.node.name, - } - }) - }) - - // Latest Deployment of each Service in the Project. - const deployments: App.Deployment[] = project.services.edges.flatMap((s) => { - return s.node.deployments.edges.map((d) => { - return { - id: d.node.id, - staticUrl: d.node.staticUrl, - serviceId: s.node.id, - } - }) - }) - - return Promise.resolve({ projectId, plugins, deployments }) -} - -/** - * Returns an async-iterable subscription to deployment logs. - */ -const subscribeToDeploymentLogs = ( - client: GqlWsClient, - deploymentId: string, -): AsyncGenerator< - ExecutionResult -> => { - return wsSubscribe(client, { - query: DeploymentLogs, - variables: { deploymentId }, - }) -} - -/** - * Returns an async-iterable subscription to plugin logs. - */ -const subscribeToPluginLogs = ( - client: GqlWsClient, - pluginId: string, - environmentId: string, -): AsyncGenerator< - ExecutionResult -> => { - return wsSubscribe(client, { - query: PluginLogs, - variables: { pluginId, environmentId }, - }) -} - -export { subscribeToPluginLogs, subscribeToDeploymentLogs, getProjectData } diff --git a/src/types/queries.ts b/src/types.d.ts similarity index 68% rename from src/types/queries.ts rename to src/types.d.ts index 4a7de86..62008ee 100644 --- a/src/types/queries.ts +++ b/src/types.d.ts @@ -1,10 +1,56 @@ +import { ApolloClient, NormalizedCacheObject } from '@apollo/client/core' +import { ChildProcessWithoutNullStreams } from 'child_process' +import { Client } from 'graphql-ws' + +export type VectorProcess = ChildProcessWithoutNullStreams + +export type HttpClient = ApolloClient + +export type WsClient = Client + +export interface VectorConfiguration { + contents: string // This holds the actual Vector config in toml format. + enabled: string[] +} + +export interface Deferred { + resolve: (done: boolean) => void + reject: (err: unknown) => void +} + +/** + * Internal application state. + */ +export namespace App { + export type ProjectId = string + + export interface Plugin { + id: string + name: string + environmentId: string + environmentName: string + } + + export interface Deployment { + id: string + staticUrl: string + serviceId: string + } + + export interface State { + projectId: ProjectId + plugins: Plugin[] + deployments: Deployment[] + } +} + /** * Response types. * * @NOTE: I'm maintaining this manually because it's only three queries. If * the number of queries grow, look into using GQL client codegen tools. */ -namespace QueryResponse { +export namespace QueryResponse { export interface ProjectQueryResponse { project: Node.Project } @@ -97,5 +143,3 @@ namespace QueryResponse { } } } - -export default QueryResponse diff --git a/src/types/app.ts b/src/types/app.ts deleted file mode 100644 index 55471f2..0000000 --- a/src/types/app.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { ChildProcessWithoutNullStreams } from 'child_process' - -type VectorProcess = ChildProcessWithoutNullStreams - -interface Deferred { - resolve: (done: boolean) => void - reject: (err: unknown) => void -} - -/** - * Internal application state. - */ -namespace App { - export type ProjectId = string - - export interface Plugin { - id: string - name: string - environmentId: string - environmentName: string - } - - export interface Deployment { - id: string - staticUrl: string - serviceId: string - } - - export interface State { - projectId: ProjectId - plugins: Plugin[] - deployments: Deployment[] - } -} - -export { Deferred, VectorProcess } -export default App diff --git a/src/utils.ts b/src/utils.ts deleted file mode 100644 index e48ff61..0000000 --- a/src/utils.ts +++ /dev/null @@ -1,22 +0,0 @@ -const requireEnv = (key: string) => { - const env = process.env[key] - if (env === undefined) { - throw new Error(`Environment variable "${key}" is required`) - } - return env -} - -const getEnv = (key: string, defaultValue: string) => { - const env = process.env[key] - return env === undefined ? defaultValue : env -} - -const bytes2utf8 = (b: ArrayBuffer): string => { - return Buffer.from(b).toString('utf-8') -} - -const parseProjectIds = (id: string): string[] => { - return id.split(',') -} - -export { bytes2utf8, getEnv, parseProjectIds, requireEnv } diff --git a/src/utils/bytes2utf8.ts b/src/utils/bytes2utf8.ts new file mode 100644 index 0000000..4270346 --- /dev/null +++ b/src/utils/bytes2utf8.ts @@ -0,0 +1,5 @@ +const bytes2utf8 = (b: ArrayBuffer): string => { + return Buffer.from(b).toString('utf-8') +} + +export default bytes2utf8 diff --git a/src/utils/get-env.ts b/src/utils/get-env.ts new file mode 100644 index 0000000..42bc888 --- /dev/null +++ b/src/utils/get-env.ts @@ -0,0 +1,6 @@ +const getEnv = (key: string, defaultValue: string) => { + const env = process.env[key] + return env ?? defaultValue +} + +export default getEnv diff --git a/src/utils/parse-project-ids.ts b/src/utils/parse-project-ids.ts new file mode 100644 index 0000000..7a1d7a6 --- /dev/null +++ b/src/utils/parse-project-ids.ts @@ -0,0 +1,6 @@ +const parseProjectIds = (id: string): string[] => { + // @TODO: This needs better error handling. + return id.split(',') +} + +export default parseProjectIds diff --git a/src/utils/require-env.ts b/src/utils/require-env.ts new file mode 100644 index 0000000..a568c5b --- /dev/null +++ b/src/utils/require-env.ts @@ -0,0 +1,9 @@ +const requireEnv = (key: string) => { + const env = process.env[key] + if (env === undefined) { + throw new Error(`Environment variable "${key}" is required`) + } + return env +} + +export default requireEnv diff --git a/src/vector.ts b/src/vector.ts deleted file mode 100644 index d680f69..0000000 --- a/src/vector.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { spawn as nodeSpawn } from 'child_process' -import { VectorProcess } from './types/app' -import { bytes2utf8 } from './utils' - -const spawn = ( - binPath: string, - cfgPath: string, - stdout: boolean = false, -): VectorProcess => { - const vector = nodeSpawn(binPath, ['--config', cfgPath]) - if (stdout === true) { - vector.stdout.on('data', (data) => { - console.log(`[process::vector/stdout] ${bytes2utf8(data)}`) - }) - } - vector.stderr.on('data', (data) => { - console.error(`[process::vector/stderr] ${bytes2utf8(data)}`) - }) - vector.on('error', (error) => { - console.error(`[process::vector/error]`, error) - }) - vector.on('close', (exitCode) => { - console.error(`[process::vector/exited] Exited with ${exitCode}`) - process.exit(1) - }) - return vector -} - -const write = (process: VectorProcess, data: string) => { - process.stdin.cork() // create buffer - process.stdin.write(`${data}\n`) - process.stdin.uncork() // flush buffer -} - -export { spawn, write } diff --git a/src/vector/configure.ts b/src/vector/configure.ts new file mode 100644 index 0000000..a483309 --- /dev/null +++ b/src/vector/configure.ts @@ -0,0 +1,29 @@ +import { VectorConfiguration } from '@/types' +import { LOGTAIL, STDOUT } from './sinks' +import { STDIN } from './sources' + +const configure = ( + enableStdout: boolean, + logtail: string | null, +): VectorConfiguration => { + const enabled = [] + let cfg = '' + cfg += STDIN + + // Append the vector sink config for each enabled sink + if (enableStdout === true) { + enabled.push('stdout') + cfg += STDOUT + } + if (logtail !== null) { + enabled.push('logtail') + cfg += LOGTAIL(logtail) + } + + return { + contents: cfg, + enabled, + } +} + +export default configure diff --git a/src/vector/sinks.ts b/src/vector/sinks.ts new file mode 100644 index 0000000..4c9ae1d --- /dev/null +++ b/src/vector/sinks.ts @@ -0,0 +1,25 @@ +const STDOUT = ` +[sinks.out] +inputs = ["*"] +type = "console" +encoding.codec = "text"` + +const LOGTAIL = (token: string) => ` +[transforms.logtail_transform] +type = "remap" +inputs = [ "*" ] +source = ''' +.dt = del(.timestamp) +.railway = del(.railway) +''' + +[sinks.logtail_sink] +type = "http" +method = "post" +inputs = [ "logtail_transform" ] +uri = "https://in.logtail.com/" +encoding.codec = "json" +auth.strategy = "bearer" +auth.token = "${token}"` + +export { STDOUT, LOGTAIL } diff --git a/src/vector/sources.ts b/src/vector/sources.ts new file mode 100644 index 0000000..ec447eb --- /dev/null +++ b/src/vector/sources.ts @@ -0,0 +1,6 @@ +const STDIN = ` +[sources.in] +type = "stdin" +` + +export { STDIN } diff --git a/src/vector/spawn.ts b/src/vector/spawn.ts new file mode 100644 index 0000000..be444e7 --- /dev/null +++ b/src/vector/spawn.ts @@ -0,0 +1,32 @@ +import { VectorProcess } from '@/types' +import bytes2utf8 from '@/utils/bytes2utf8' +import { spawn as nodeSpawn } from 'child_process' +import { writeFileSync } from 'fs' +import tmp from 'tmp' + +const spawn = (binPath: string, cfg: string): VectorProcess => { + // Write config to a tmp file. Vector's config flags seem to fopen a file, + // so passing in a buffer doesn't help. + const cfgFile = tmp.fileSync() + writeFileSync(cfgFile.name, cfg) + + const vector = nodeSpawn(binPath, ['--config', cfgFile.name]) + + vector.stdout.on('data', (data) => { + console.log(`[process::vector/stdout] ${bytes2utf8(data)}`) + }) + vector.stderr.on('data', (data) => { + console.error(`[process::vector/stderr] ${bytes2utf8(data)}`) + }) + vector.on('error', (error) => { + console.error(`[process::vector/error]`, error) + }) + vector.on('close', (exitCode) => { + console.error(`[process::vector/exited] Exited with ${exitCode}`) + process.exit(1) + }) + + return vector +} + +export default spawn diff --git a/src/vector/write.ts b/src/vector/write.ts new file mode 100644 index 0000000..a90383e --- /dev/null +++ b/src/vector/write.ts @@ -0,0 +1,9 @@ +import { VectorProcess } from '@/types' + +const write = (process: VectorProcess, data: string) => { + process.stdin.cork() // create buffer + process.stdin.write(`${data}\n`) + process.stdin.uncork() // flush buffer +} + +export default write diff --git a/tsconfig.json b/tsconfig.json index 6bc9e20..087f683 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -28,8 +28,9 @@ "module": "commonjs", /* Specify what module code is generated. */ // "rootDir": "./", /* Specify the root folder within your source files. */ // "moduleResolution": "node10", /* Specify how TypeScript looks up a file from a given module specifier. */ - // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ - // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ + "paths": { + "@/*": ["./src/*"] + }, // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ // "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */ // "types": [], /* Specify type package names to be included without being referenced in a source file. */ diff --git a/yarn.lock b/yarn.lock index 89bb5b0..516cf17 100644 --- a/yarn.lock +++ b/yarn.lock @@ -76,6 +76,11 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-18.15.3.tgz#f0b991c32cfc6a4e7f3399d6cb4b8cf9a0315014" integrity sha512-p6ua9zBxz5otCmbpb5D3U4B5Nanw6Pk3PPyX05xnxbB/fRv71N7CPmORg7uAD5P70T0xmx1pzAx/FUfa5X+3cw== +"@types/tmp@^0.2.3": + version "0.2.3" + resolved "https://registry.yarnpkg.com/@types/tmp/-/tmp-0.2.3.tgz#908bfb113419fd6a42273674c00994d40902c165" + integrity sha512-dDZH/tXzwjutnuk4UacGgFRwV+JSLaXL1ikvidfJprkb7L9Nx1njcRHHmi3Dsvt7pgqqTEeucQuOrWHPFgzVHA== + "@types/ws@^8.5.4": version "8.5.4" resolved "https://registry.yarnpkg.com/@types/ws/-/ws-8.5.4.tgz#bb10e36116d6e570dd943735f86c933c1587b8a5" @@ -119,6 +124,24 @@ arg@^4.1.0: resolved "https://registry.yarnpkg.com/arg/-/arg-4.1.3.tgz#269fc7ad5b8e42cb63c896d5666017261c144089" integrity sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA== +balanced-match@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" + integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== + +brace-expansion@^1.1.7: + version "1.1.11" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" + integrity sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA== + dependencies: + balanced-match "^1.0.0" + concat-map "0.0.1" + +concat-map@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" + integrity sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg== + create-require@^1.1.0: version "1.1.1" resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" @@ -134,6 +157,23 @@ dotenv@^16.0.3: resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.0.3.tgz#115aec42bac5053db3c456db30cc243a5a836a07" integrity sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ== +fs.realpath@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" + integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== + +glob@^7.1.3: + version "7.2.3" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b" + integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q== + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.1.1" + once "^1.3.0" + path-is-absolute "^1.0.0" + graphql-tag@^2.12.6: version "2.12.6" resolved "https://registry.yarnpkg.com/graphql-tag/-/graphql-tag-2.12.6.tgz#d441a569c1d2537ef10ca3d1633b48725329b5f1" @@ -158,11 +198,29 @@ hoist-non-react-statics@^3.3.2: dependencies: react-is "^16.7.0" +inflight@^1.0.4: + version "1.0.6" + resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" + integrity sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA== + dependencies: + once "^1.3.0" + wrappy "1" + +inherits@2: + version "2.0.4" + resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" + integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== + "js-tokens@^3.0.0 || ^4.0.0": version "4.0.0" resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ== +json5@^2.2.2: + version "2.2.3" + resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283" + integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg== + loose-envify@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf" @@ -175,11 +233,30 @@ make-error@^1.1.1: resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2" integrity sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw== +minimatch@^3.1.1: + version "3.1.2" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" + integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw== + dependencies: + brace-expansion "^1.1.7" + +minimist@^1.2.6: + version "1.2.8" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" + integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== + object-assign@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" integrity sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg== +once@^1.3.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" + integrity sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w== + dependencies: + wrappy "1" + optimism@^0.16.1: version "0.16.2" resolved "https://registry.yarnpkg.com/optimism/-/optimism-0.16.2.tgz#519b0c78b3b30954baed0defe5143de7776bf081" @@ -188,6 +265,11 @@ optimism@^0.16.1: "@wry/context" "^0.7.0" "@wry/trie" "^0.3.0" +path-is-absolute@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" + integrity sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg== + prop-types@^15.7.2: version "15.8.1" resolved "https://registry.yarnpkg.com/prop-types/-/prop-types-15.8.1.tgz#67d87bf1a694f48435cf332c24af10214a3140b5" @@ -207,11 +289,30 @@ response-iterator@^0.2.6: resolved "https://registry.yarnpkg.com/response-iterator/-/response-iterator-0.2.6.tgz#249005fb14d2e4eeb478a3f735a28fd8b4c9f3da" integrity sha512-pVzEEzrsg23Sh053rmDUvLSkGXluZio0qu8VT6ukrYuvtjVfCbDZH9d6PGXb8HZfzdNZt8feXv/jvUzlhRgLnw== +rimraf@^3.0.0: + version "3.0.2" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" + integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA== + dependencies: + glob "^7.1.3" + +strip-bom@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-3.0.0.tgz#2334c18e9c759f7bdd56fdef7e9ae3d588e68ed3" + integrity sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA== + symbol-observable@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/symbol-observable/-/symbol-observable-4.0.0.tgz#5b425f192279e87f2f9b937ac8540d1984b39205" integrity sha512-b19dMThMV4HVFynSAM1++gBHAbk2Tc/osgLIBZMKsyqh34jb2e8Os7T6ZW/Bt3pJFdBTd2JwAnAAEQV7rSNvcQ== +tmp@^0.2.1: + version "0.2.1" + resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.2.1.tgz#8457fc3037dcf4719c251367a1af6500ee1ccf14" + integrity sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ== + dependencies: + rimraf "^3.0.0" + ts-invariant@^0.10.3: version "0.10.3" resolved "https://registry.yarnpkg.com/ts-invariant/-/ts-invariant-0.10.3.tgz#3e048ff96e91459ffca01304dbc7f61c1f642f6c" @@ -238,6 +339,20 @@ ts-node@^10.9.1: v8-compile-cache-lib "^3.0.1" yn "3.1.1" +ts-pattern@^4.2.1: + version "4.2.1" + resolved "https://registry.yarnpkg.com/ts-pattern/-/ts-pattern-4.2.1.tgz#d626da4c5755d78c1ae62b8f2675c94884a31a8c" + integrity sha512-lXCmHZb01QOM9HdCLvisCGUH9ATdKPON9UaUvwe007gJAhuSBhRWIAIowys5QqNxEq6odWctfMIdI96vzjnOMQ== + +tsconfig-paths@^4.1.2: + version "4.1.2" + resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-4.1.2.tgz#4819f861eef82e6da52fb4af1e8c930a39ed979a" + integrity sha512-uhxiMgnXQp1IR622dUXI+9Ehnws7i/y6xvpZB9IbUVOPy0muvdvgXeZOn88UcGPiT98Vp3rJPTa8bFoalZ3Qhw== + dependencies: + json5 "^2.2.2" + minimist "^1.2.6" + strip-bom "^3.0.0" + tslib@^2.1.0, tslib@^2.3.0: version "2.5.0" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.5.0.tgz#42bfed86f5787aeb41d031866c8f402429e0fddf" @@ -253,6 +368,11 @@ v8-compile-cache-lib@^3.0.1: resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf" integrity sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg== +wrappy@1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" + integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ== + ws@^8.13.0: version "8.13.0" resolved "https://registry.yarnpkg.com/ws/-/ws-8.13.0.tgz#9a9fb92f93cf41512a0735c8f4dd09b8a1211cd0" From 64736313c95d741bcd069895d799162c0d3952f2 Mon Sep 17 00:00:00 2001 From: Ray Chen Date: Thu, 23 Mar 2023 05:09:47 +0800 Subject: [PATCH 2/7] Cleanup: Remove Vector config --- Dockerfile | 2 -- vector.toml | 24 ------------------------ 2 files changed, 26 deletions(-) delete mode 100644 vector.toml diff --git a/Dockerfile b/Dockerfile index e7a1ace..0cb453a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,5 @@ WORKDIR /app COPY --chown=runner:runner --from=base /root/.vector ./vector COPY --chown=runner:runner --from=build /build/node_modules ./node_modules COPY --chown=runner:runner --from=build /build/dist ./dist -COPY --chown=runner:runner vector.toml ./vector.toml ENV VECTOR_BIN_PATH=/app/vector/bin/vector -ENV VECTOR_CFG_PATH=/app/vector.toml ENTRYPOINT ["node", "dist/main.js"] diff --git a/vector.toml b/vector.toml deleted file mode 100644 index 300a7bc..0000000 --- a/vector.toml +++ /dev/null @@ -1,24 +0,0 @@ -[sources.in] -type = "stdin" - -[sinks.out] -inputs = ["in"] -type = "console" -encoding.codec = "text" - -[transforms.logtail_transform] -type = "remap" -inputs = [ "*" ] -source = ''' -.dt = del(.timestamp) -.railway = del(.railway) -''' - -[sinks.logtail_sink] -type = "http" -method = "post" -inputs = [ "logtail_transform" ] -uri = "https://in.logtail.com/" -encoding.codec = "json" -auth.strategy = "bearer" -auth.token = "${LOGTAIL_TOKEN}" From 2637ebf50922e40af7b59b7a5098d8f12f0b6021 Mon Sep 17 00:00:00 2001 From: Ray Chen Date: Thu, 23 Mar 2023 05:10:00 +0800 Subject: [PATCH 3/7] Cleanup: Console logs --- src/main.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main.ts b/src/main.ts index 2e1aa27..141b4ef 100644 --- a/src/main.ts +++ b/src/main.ts @@ -70,19 +70,15 @@ const main = async () => { // network requests to Railway API if Vector can't start. console.info(`⚙️ Using Vector binary: ${VECTOR_BIN_PATH}`) const vector = spawn(VECTOR_BIN_PATH, vectorCfg.contents) - write(vector, '>>> ping from railway-chord') console.info(`✅ Vector started`) - console.info(`✅ Enabled sinks:`) vectorCfg.enabled.forEach((s) => { console.info(` - ${s}`) }) console.info(`⚙️ Using Railway HTTP endpoint: ${RAILWAY_API_HTTP_ENDPOINT}`) - console.info( - `⚙️ Using Railway WebSockets endpoint: ${RAILWAY_API_WS_ENDPOINT}`, - ) + console.info(`⚙️ Using Railway WS endpoint: ${RAILWAY_API_WS_ENDPOINT}`) const projectIds = parseProjectIds(RAILWAY_PROJECT_IDS) const httpClient = createHttpClient( From 0d9ad554faec0cc70579153d40276982ac6133fa Mon Sep 17 00:00:00 2001 From: Ray Chen Date: Thu, 23 Mar 2023 05:47:22 +0800 Subject: [PATCH 4/7] feat(vector): Add Datadog sink --- src/main.ts | 9 ++++++++- src/vector/configure.ts | 16 ++++++++++++---- src/vector/sinks.ts | 10 +++++++++- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/main.ts b/src/main.ts index 141b4ef..5a3aa90 100644 --- a/src/main.ts +++ b/src/main.ts @@ -64,7 +64,14 @@ const main = async () => { // inject the Datadog sink into Vector config; and so on. const ENABLE_STDOUT = process.env.ENABLE_STDOUT === 'true' ? true : false const LOGTAIL_TOKEN = process.env.LOGTAIL_TOKEN ?? null - const vectorCfg = configureVector(ENABLE_STDOUT, LOGTAIL_TOKEN) + const DATADOG_TOKEN = process.env.DATADOG_TOKEN ?? null + const DATADOG_SITE = process.env.DATADOG_SITE ?? null + const vectorCfg = configureVector( + ENABLE_STDOUT, + LOGTAIL_TOKEN, + DATADOG_TOKEN, + DATADOG_SITE, + ) // Start Vector first. We want to crash early; there's no point in making // network requests to Railway API if Vector can't start. diff --git a/src/vector/configure.ts b/src/vector/configure.ts index a483309..5f48b55 100644 --- a/src/vector/configure.ts +++ b/src/vector/configure.ts @@ -1,10 +1,12 @@ import { VectorConfiguration } from '@/types' -import { LOGTAIL, STDOUT } from './sinks' +import { DATADOG, LOGTAIL, STDOUT } from './sinks' import { STDIN } from './sources' const configure = ( enableStdout: boolean, - logtail: string | null, + logtailToken: string | null, + datadogToken: string | null, + datadogSite: string | null, ): VectorConfiguration => { const enabled = [] let cfg = '' @@ -15,9 +17,15 @@ const configure = ( enabled.push('stdout') cfg += STDOUT } - if (logtail !== null) { + if (logtailToken !== null) { enabled.push('logtail') - cfg += LOGTAIL(logtail) + cfg += LOGTAIL(logtailToken) + } + if (datadogToken !== null) { + enabled.push('datadog') + cfg += datadogSite + ? DATADOG(datadogToken, datadogSite) + : DATADOG(datadogToken) } return { diff --git a/src/vector/sinks.ts b/src/vector/sinks.ts index 4c9ae1d..0cfb135 100644 --- a/src/vector/sinks.ts +++ b/src/vector/sinks.ts @@ -22,4 +22,12 @@ encoding.codec = "json" auth.strategy = "bearer" auth.token = "${token}"` -export { STDOUT, LOGTAIL } +const DATADOG = (token: string, site?: string) => ` +[sinks.datadog] +type = "datadog_logs" +inputs = [ "*" ] +compression = "gzip" +site = "${site ?? "datadoghq.com"}" +default_api_key = "${token}"` + +export { DATADOG, STDOUT, LOGTAIL } From 70007814ae5d41a4e531c978cf4cd93aecd95bd7 Mon Sep 17 00:00:00 2001 From: Ray Chen Date: Thu, 23 Mar 2023 06:03:59 +0800 Subject: [PATCH 5/7] feat(docs): Add a section on adding Vector sinks --- README.md | 50 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 30837df..eb11a81 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,55 @@ Railway's Dashboard -> Project -> Settings -> General. RAILWAY_PROJECT_IDS="XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX,XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX" ``` -## Adding log destinations (Vector sinks) +## Adding Vector sinks + +### Request for new sinks + +To request for a new Vector sink, please [open a GitHub issue](https://github.com/half0wl/railway-chord/issues/new). + +### Add your own sink + +**See this [pull request](https://github.com/half0wl/railway-chord/pull/8) for an +example**. + +Before adding a new Vector sink, check the authentication mechanism of the +provider you're using. There is usually an API key required. + +1. In [`src/main.ts`](src/main.ts), pass the required API key/token into +`configureVector()`: + + ```typescript + const PROVIDER_TOKEN = process.env.PROVIDER_TOKEN ?? null + const vectorCfg = configureVector( + ..., + PROVIDER_TOKEN + ) + ``` +2. Create a new [Vector sink configuration](https://vector.dev/docs/reference/configuration/sinks/) +in [`src/vector/sinks.ts`](src/vector/sinks.ts) (TOML format). + + ```typescript + const PROVIDER = (token: string) => ` + [sinks.PROVIDER] + ... + token = "${token}" + ` + ``` +3. In [`src/vector/configure.ts`](src/vector/configure.ts), import and append +the newly-created config created above, passing the required API key into it: + + ```typescript + import { PROVIDER } from './sinks' + const configure = ( ..., providerToken: string | null ) => { + ... + if (providerToken !== null) { + enabled.push('provider') + cfg += PROVIDER(providerToken) + } + ... + } + ``` -(...wip) ## Configuration From 4b45418fef841cbc43cc1508bdb6ce1c95bde1d2 Mon Sep 17 00:00:00 2001 From: Ray Chen Date: Thu, 23 Mar 2023 06:46:42 +0800 Subject: [PATCH 6/7] Revamp docs --- README.md | 47 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index eb11a81..38457b2 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # railway-chord -Centralized logging for [Railway.app](https://railway.app) projects, powered -by [Vector](https://vector.dev/). +[Vector](https://vector.dev/) log egress for [Railway.app](https://railway.app) +projects. _railway-chord_ pipes the log stream from Railway's GraphQL API into Vector. This gives you **project-wide** centralized logging on Railway. For each Railway @@ -20,7 +20,25 @@ the instructions in the link. The button above will deploy _railway-chord_ on Railway. You will be prompted to set some required environment variables; there is no additional configuration -required beyond this. +required beyond this. Log sinks are automatically configured based on the +presence of required environment variables of each provider, i.e. setting a +`DATADOG_TOKEN` will enable Datadog, and so on. + +### Datadog + +To enable Datadog logs, set the `DATADOG_TOKEN` service variable. + +There is an optional `DATADOG_SITE` setting if your Datadog account is hosted +on a different Datadog instance (defaults to `datadoghq.com`). + +### Logtail + +To enable Logtail, set the `LOGTAIL_TOKEN` service variable. + +### `stdout` + +To enable logging to `stdout`, set `ENABLE_STDOUT=true`. This will output all +enabled project's logs into _railway-chord_'s `stdout`. ### Project IDs @@ -34,6 +52,23 @@ Railway's Dashboard -> Project -> Settings -> General. RAILWAY_PROJECT_IDS="XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX,XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX" ``` +### Log Enrichment + +_railway-chord_ will inject a `railway` object into the logs sent to the provider +containing the deployment/plugin ID and name. + +#### Example: + +```json +{ + "railway": { + "id": "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX", + "name": "app.up.railway.app", + "type": "DEPLOYMENT" + } +} +``` + ## Adding Vector sinks ### Request for new sinks @@ -90,11 +125,13 @@ the newly-created config created above, passing the required API key into it: | ----------- | ----------- | | `RAILWAY_API_TOKEN` | **Required**. Railway API token. | | `RAILWAY_PROJECT_IDS` | **Required**. A comma-separated list of Railway Project IDs. | -| `LOGTAIL_TOKEN` | **Required**. Logtail token. | +| `LOGTAIL_TOKEN` | Optional. Logtail token. | +| `DATADOG_TOKEN` | Optional. Datadog API token. | +| `DATADOG_SITE` | Optional. Datadog site setting. Defaults to `datadoghq.com`. | +| `ENABLE_STDOUT` | Optional. Enable Vector logging to stdout. | | `RAILWAY_API_HTTP_ENDPOINT` | Optional. Railway's HTTP GQL Endpoint. Defaults to `https://backboard.railway.app/graphql/v2`. | | `RAILWAY_WS_HTTP_ENDPOINT` | Optional. Railway's WebSockets GQL Endpoint. Defaults to `wss://backboard.railway.app/graphql/v2`. | | `VECTOR_BIN_PATH` | Optional. Path to Vector binary. Defaults to path defined in Docker build. You do not need to set this. | -| `VECTOR_CFG_PATH` | Optional. Path to Vector configuration. Defaults to path defined in Docker build. You do not need to set this. | ## License From 2e4ce66ac00283de57300e7c16f02077afdd3d87 Mon Sep 17 00:00:00 2001 From: Ray Chen Date: Thu, 23 Mar 2023 07:31:20 +0800 Subject: [PATCH 7/7] Add a section about Railway API rate limit --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 38457b2..d6e4e37 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,15 @@ containing the deployment/plugin ID and name. } ``` +### Railway API Rate Limit + +Railway's Public API currently has a [rate limit of 1,000 requests daily](https://docs.railway.app/reference/public-api#rate-limits). _railway-chord_ will make ~100 requests every 24 hours for each project enabled. +Work is underway to reduce the amount of requests _railway-chord_ is making (see +[railway-chord#3](https://github.com/half0wl/railway-chord/issues/3)). + +Be careful about this if you're using Railway's API for something else - _railway-chord_ +will eat into your rate limit! + ## Adding Vector sinks ### Request for new sinks