diff --git a/packages/ttsl-lang/src/language/builtins/ttsl-ds-functions.ts b/packages/ttsl-lang/src/language/builtins/ttsl-functions.ts similarity index 100% rename from packages/ttsl-lang/src/language/builtins/ttsl-ds-functions.ts rename to packages/ttsl-lang/src/language/builtins/ttsl-functions.ts diff --git a/packages/ttsl-lang/src/language/communication/commands.ts b/packages/ttsl-lang/src/language/communication/commands.ts new file mode 100644 index 00000000..73be6b26 --- /dev/null +++ b/packages/ttsl-lang/src/language/communication/commands.ts @@ -0,0 +1,3 @@ +export const COMMAND_PRINT_VALUE = 'ttsl.printValue'; +export const COMMAND_RUN_PIPELINE = 'ttsl.runPipeline'; +export const COMMAND_SHOW_IMAGE = 'ttsl.showImage'; diff --git a/packages/ttsl-lang/src/language/communication/rpc.ts b/packages/ttsl-lang/src/language/communication/rpc.ts new file mode 100644 index 00000000..22e4e59a --- /dev/null +++ b/packages/ttsl-lang/src/language/communication/rpc.ts @@ -0,0 +1,59 @@ +import { MessageDirection, NotificationType0, RequestType0 } from 'vscode-languageserver'; +import { NotificationType } from 'vscode-languageserver-protocol'; + +export namespace InstallRunnerNotification { + export const method = 'runner/install' as const; + export const messageDirection = MessageDirection.serverToClient; + export const type = new NotificationType0(method); +} + +export namespace StartRunnerNotification { + export const method = 'runner/start' as const; + export const messageDirection = MessageDirection.clientToServer; + export const type = new NotificationType0(method); +} + +export namespace RunnerStartedNotification { + export const method = 'runner/started' as const; + export const messageDirection = MessageDirection.serverToClient; + export const type = new NotificationType(method); +} + +export interface RunnerStartedParams { + /** + * The port the runner is listening on. + */ + port: number; +} + +export namespace UpdateRunnerNotification { + export const method = 'runner/update' as const; + export const messageDirection = MessageDirection.serverToClient; + export const type = new NotificationType0(method); +} + +export namespace ShowImageNotification { + export const method = 'runner/showImage' as const; + export const messageDirection = MessageDirection.serverToClient; + export const type = new NotificationType(method); +} + +export interface ShowImageParams { + image: { + /** + * The format of the image. + */ + format: 'png'; + + /** + * The Base64-encoded image. + */ + bytes: string; + }; +} + +export namespace IsRunnerReadyRequest { + export const method = 'runner/isReady' as const; + export const messageDirection = MessageDirection.clientToServer; + export const type = new RequestType0(method); +} diff --git a/packages/ttsl-lang/src/language/communication/ttsl-messaging-provider.ts b/packages/ttsl-lang/src/language/communication/ttsl-messaging-provider.ts new file mode 100644 index 00000000..82bbdb13 --- /dev/null +++ b/packages/ttsl-lang/src/language/communication/ttsl-messaging-provider.ts @@ -0,0 +1,541 @@ +import { TTSLServices } from '../ttsl-module.js'; +import { + CancellationToken, + Connection, + Disposable, + MessageActionItem, + NotificationHandler, + NotificationHandler0, + NotificationType, + NotificationType0, + RequestHandler, + RequestHandler0, + RequestType, + RequestType0, + WorkDoneProgressReporter, +} from 'vscode-languageserver'; +import { GenericRequestHandler } from 'vscode-jsonrpc/lib/common/connection.js'; +import { GenericNotificationHandler } from 'vscode-languageserver-protocol'; + +/** + * Log or show messages in the language client or otherwise communicate with it. + */ +export class TTSLMessagingProvider { + private readonly connection: Connection | undefined; + private logger: Partial | undefined = undefined; + private userInteractionProvider: Partial | undefined = undefined; + private messageBroker: Partial | undefined = undefined; + + constructor(services: TTSLServices) { + this.connection = services.shared.lsp.Connection; + } + + // Logging --------------------------------------------------------------------------------------------------------- + + /** + * Create a logger that prepends all messages with the given tag. + */ + createTaggedLogger(tag: string): TTSLLogger { + return { + trace: (message: string, verbose?: string) => this.trace(tag, message, verbose), + debug: (message: string) => this.debug(tag, message), + info: (message: string) => this.info(tag, message), + warn: (message: string) => this.warn(tag, message), + error: (message: string) => this.error(tag, message), + result: (message: string) => this.result(message), + }; + } + + /** + * Log the given data to the trace log. + */ + trace(tag: string, message: string, verbose?: string): void { + const text = this.formatLogMessage(tag, message); + if (this.logger?.trace) { + this.logger.trace(text, verbose); + } else if (this.connection) { + /* c8 ignore next 2 */ + this.connection.tracer.log(text, verbose); + } + } + + /** + * Log a debug message. + */ + debug(tag: string, message: string): void { + const text = this.formatLogMessage(tag, message); + if (this.logger?.debug) { + this.logger.debug(text); + } else if (this.connection) { + /* c8 ignore next 2 */ + this.connection.console.debug(text); + } + } + + /** + * Log an information message. + */ + info(tag: string, message: string): void { + const text = this.formatLogMessage(tag, message); + if (this.logger?.info) { + this.logger.info(text); + } else if (this.connection) { + /* c8 ignore next 2 */ + this.connection.console.info(text); + } + } + + /** + * Log a warning message. + */ + warn(tag: string, message: string): void { + const text = this.formatLogMessage(tag, message); + if (this.logger?.warn) { + this.logger.warn(text); + } else if (this.connection) { + /* c8 ignore next 2 */ + this.connection.console.warn(text); + } + } + + /** + * Log an error message. + */ + error(tag: string, message: string): void { + const text = this.formatLogMessage(tag, message); + if (this.logger?.error) { + this.logger.error(text); + } else if (this.connection) { + /* c8 ignore next 2 */ + this.connection.console.error(text); + } + } + + /** + * Show a result to the user. + */ + result(message: string): void { + const text = this.formatLogMessage('Result', message) + '\n'; + if (this.logger?.result) { + this.logger.result(text); + } else if (this.connection) { + /* c8 ignore next 2 */ + this.connection.console.log(text); + } + } + + private formatLogMessage(tag: string, message: string): string { + return tag ? `[${tag}] ${message}` : message; + } + + // User interaction ------------------------------------------------------------------------------------------------ + + /** + * Shows an information message in the client's user interface. + * + * Depending on the client this might be a modal dialog with a confirmation button or a notification in a + * notification center. + * + * @param message The message to show. + */ + showInformationMessage(message: string): void; + /** + * Shows an information message in the client's user interface. + * + * Depending on the client this might be a modal dialog with a confirmation button or a notification in a + * notification center. + * + * @param message The message to show. + * @param actions The actions to show. + * + * @returns A promise that resolves to the selected action. + */ + showInformationMessage(message: string, ...actions: T[]): Promise; + async showInformationMessage( + message: string, + ...actions: T[] + ): Promise { + if (this.userInteractionProvider?.showInformationMessage) { + return this.userInteractionProvider.showInformationMessage(message, ...actions); + } /* c8 ignore start */ else if (this.connection) { + return this.connection.window.showInformationMessage(message, ...actions); + } else { + return undefined; + } /* c8 ignore stop */ + } + + /** + * Shows a warning message in the client's user interface. + * + * Depending on the client this might be a modal dialog with a confirmation button or a notification in a + * notification center. + * + * @param message The message to show. + * + * @returns A promise that resolves to the selected action. + */ + showWarningMessage(message: string): void; + /** + * Shows a warning message in the client's user interface. + * + * Depending on the client this might be a modal dialog with a confirmation button or a notification in a + * notification center. + * + * @param message The message to show. + * @param actions The actions to show. + * + * @returns A promise that resolves to the selected action. + */ + showWarningMessage(message: string, ...actions: T[]): Promise; + async showWarningMessage(message: string, ...actions: T[]): Promise { + if (this.userInteractionProvider?.showWarningMessage) { + return this.userInteractionProvider.showWarningMessage(message, ...actions); + } /* c8 ignore start */ else if (this.connection) { + return this.connection.window.showWarningMessage(message, ...actions); + } else { + return undefined; + } /* c8 ignore stop */ + } + + /** + * Shows an error message in the client's user interface. + * + * Depending on the client this might be a modal dialog with a confirmation button or a notification in a + * notification center. + * + * @param message The message to show. + * + * @returns A promise that resolves to the selected action. + */ + showErrorMessage(message: string): void; + /** + * Shows an error message in the client's user interface. + * + * Depending on the client this might be a modal dialog with a confirmation button or a notification in a + * notification center. + * + * @param message The message to show. + * @param actions The actions to show. + * + * @returns A promise that resolves to the selected action. + */ + showErrorMessage(message: string, ...actions: T[]): Promise; + async showErrorMessage(message: string, ...actions: T[]): Promise { + if (this.userInteractionProvider?.showErrorMessage) { + return this.userInteractionProvider.showErrorMessage(message, ...actions); + } /* c8 ignore start */ else if (this.connection) { + return this.connection.window.showErrorMessage(message, ...actions); + } else { + return undefined; + } /* c8 ignore stop */ + } + + /** + * Shows a progress indicator in the client's user interface. + * + * @param title + * The title of the progress indicator. + * + * @param message + * An optional message to indicate what is currently being done. + * + * @param cancellable + * Whether the progress indicator should be cancellable. Observe the `token` inside the returned reporter to check + * if the user has cancelled the progress indicator. + * + * @returns + * A promise that resolves to the progress reporter. Use this reporter to update the progress indicator. + */ + async showProgress( + title: string, + message?: string, + cancellable: boolean = false, + ): Promise { + if (this.userInteractionProvider?.showProgress) { + return this.userInteractionProvider.showProgress(title, 0, message, cancellable); + } /* c8 ignore start */ else if (this.connection) { + const reporter = await this.connection.window.createWorkDoneProgress(); + reporter?.begin(title, 0, message, cancellable); + return reporter; + } else { + return NOOP_PROGRESS_REPORTER; + } /* c8 ignore stop */ + } + + // Message broker -------------------------------------------------------------------------------------------------- + + /** + * Installs a notification handler for the given method. + * + * @param type The method to register a request handler for. + * @param handler The handler to install. + */ + onNotification(type: NotificationType0, handler: NotificationHandler0): Disposable; + /** + * Installs a notification handler for the given method. + * + * @param type The method to register a request handler for. + * @param handler The handler to install. + */ + onNotification

(type: NotificationType

, handler: NotificationHandler

): Disposable; + onNotification

( + type: NotificationType0 | NotificationType

, + handler: NotificationHandler0 | NotificationHandler

, + ): Disposable { + if (this.messageBroker?.onNotification) { + return this.messageBroker.onNotification(type.method, handler); + } else if (this.connection) { + /* c8 ignore next 2 */ + return this.connection.onNotification(type.method, handler); + } else { + return NOOP_DISPOSABLE; + } + } + + /** + * Send a notification to the client. + * + * @param type The method to invoke on the client. + */ + sendNotification(type: NotificationType0): Promise; + /** + * Send a notification to the client. + * + * @param type The method to invoke on the client. + * @param args The arguments. + */ + sendNotification

(type: NotificationType

, args: P): Promise; + async sendNotification

(type: NotificationType0 | NotificationType

, args?: P): Promise { + if (this.messageBroker?.sendNotification) { + await this.messageBroker.sendNotification(type.method, args); + } else if (this.connection) { + /* c8 ignore next 2 */ + await this.connection.sendNotification(type.method, args); + } + } + + /** + * Installs a request handler for the given method. + * + * @param type The method to register a request handler for. + * @param handler The handler to install. + */ + onRequest(type: RequestType0, handler: RequestHandler0): Disposable; + /** + * Installs a request handler for the given method. + * + * @param type The method to register a request handler for. + * @param handler The handler to install. + */ + onRequest(type: RequestType, handler: RequestHandler): Disposable; + onRequest( + type: RequestType0 | RequestType, + handler: RequestHandler0 | RequestHandler, + ): Disposable { + if (this.messageBroker?.onRequest) { + return this.messageBroker.onRequest(type.method, handler); + } else if (this.connection) { + /* c8 ignore next 2 */ + return this.connection.onRequest(type.method, handler); + } else { + return NOOP_DISPOSABLE; + } + } + + /** + * Send a request to the client. + * + * @param type The method to register a request handler for. + * @param token A cancellation token that can be used to cancel the request. + * + * @returns A promise that resolves to the response. + */ + sendRequest(type: RequestType0, token?: CancellationToken): Promise; + /** + * Send a request to the client. + * + * @param type The method to register a request handler for. + * @param args The arguments. + * @param token A cancellation token that can be used to cancel the request. + * + * @returns A promise that resolves to the response. + */ + sendRequest(type: RequestType, args: P, token?: CancellationToken): Promise; + async sendRequest( + type: RequestType0 | RequestType, + argsOrToken?: P | CancellationToken, + token?: CancellationToken, + ): Promise { + if (this.messageBroker?.sendRequest) { + if (CancellationToken.is(argsOrToken) && !token) { + return this.messageBroker.sendRequest(type.method, undefined, argsOrToken); + } else { + return this.messageBroker.sendRequest(type.method, argsOrToken, token); + } + } else if (this.connection) { + /* c8 ignore next 2 */ + return this.connection.sendRequest(type.method, argsOrToken, token); + } else { + /* c8 ignore next 2 */ + return undefined; + } + } + + // Configuration --------------------------------------------------------------------------------------------------- + + /** + * Set the logger to use for logging messages. + */ + setLogger(logger: Partial) { + this.logger = logger; + } + + /** + * Set the service to interact with the user. + */ + setUserInteractionProvider(userInteractionProvider: Partial) { + this.userInteractionProvider = userInteractionProvider; + } + + /** + * Set the message broker to use for communicating with the client. + */ + setMessageBroker(messageBroker: Partial) { + this.messageBroker = messageBroker; + } +} + +/** + * A logging provider. + */ +export interface TTSLLogger { + /** + * Log the given data to the trace log. + */ + trace: (message: string, verbose?: string) => void; + + /** + * Log a debug message. + */ + debug: (message: string) => void; + + /** + * Log an information message. + */ + info: (message: string) => void; + + /** + * Log a warning message. + */ + warn: (message: string) => void; + + /** + * Log an error message. + */ + error: (message: string) => void; + + /** + * Show a result to the user. + */ + result: (message: string) => void; +} + +/** + * A service for showing messages to the user. + */ +export interface TTSLUserInteractionProvider { + /** + * Prominently show an information message. The message should be short and human-readable. + * + * @returns + * A thenable that resolves to the selected action. + */ + showInformationMessage: (message: string, ...actions: T[]) => Thenable; + + /** + * Prominently show a warning message. The message should be short and human-readable. + * + * @returns + * A thenable that resolves to the selected action. + */ + showWarningMessage: (message: string, ...actions: T[]) => Thenable; + + /** + * Prominently show an error message. The message should be short and human-readable. + * + * @returns + * A thenable that resolves to the selected action. + */ + showErrorMessage: (message: string, ...actions: T[]) => Thenable; + + /** + * Shows a progress indicator in the client's user interface. + * + * @param title + * The title of the progress indicator. + * + * @param message + * An optional message to indicate what is currently being done. + * + * @param cancellable + * Whether the progress indicator should be cancellable. Observe the `token` inside the returned reporter to check + * if the user has cancelled the progress indicator. + * + * @returns + * A thenable that resolves to the progress reporter. Use this reporter to update the progress indicator. + */ + showProgress: ( + title: string, + percentage?: number, + message?: string, + cancellable?: boolean, + ) => Thenable; +} + +/** + * A message broker for communicating with the client. + */ +export interface TTSLMessageBroker { + /** + * Installs a notification handler for the given method. + * + * @param method The method to register a request handler for. + * @param handler The handler to install. + */ + onNotification: (method: string, handler: GenericNotificationHandler) => Disposable; + + /** + * Send a notification to the client. + * + * @param method The method to invoke on the client. + * @param args The arguments. + */ + sendNotification: (method: string, args?: any) => Promise; + + /** + * Installs a request handler for the given method. + * + * @param method The method to register a request handler for. + * @param handler The handler to install. + */ + onRequest: (method: string, handler: GenericRequestHandler) => Disposable; + + /** + * Send a request to the client. + * + * @param method The method to register a request handler for. + * @param args The arguments. + * @param token A cancellation token that can be used to cancel the request. + * + * @returns A promise that resolves to the response. + */ + sendRequest(method: string, args: any, token?: CancellationToken): Promise; +} + +const NOOP_PROGRESS_REPORTER: WorkDoneProgressReporter = { + begin() {}, + report() {}, + done() {}, +}; + +const NOOP_DISPOSABLE: Disposable = Disposable.create(() => {}); diff --git a/packages/ttsl-lang/src/language/flow/ttsl-slicer.ts b/packages/ttsl-lang/src/language/flow/ttsl-slicer.ts new file mode 100644 index 00000000..47ec0b9c --- /dev/null +++ b/packages/ttsl-lang/src/language/flow/ttsl-slicer.ts @@ -0,0 +1,63 @@ +import { TTSLServices } from '../ttsl-module.js'; +import { isTslAssignment, isTslPlaceholder, isTslReference, TslPlaceholder, TslStatement } from '../generated/ast.js'; +import { AstUtils, Stream } from 'langium'; +import { getAssignees } from '../helpers/nodeProperties.js'; + +export class TTSLSlicer { + + constructor(services: TTSLServices) {} + + /** + * Computes the subset of the given statements that are needed to calculate the target placeholders. + */ + computeBackwardSlice(statements: TslStatement[], targets: TslPlaceholder[]): TslStatement[] { + const aggregator = new BackwardSliceAggregator(targets); + + for (const statement of statements.reverse()) { + // Keep if it declares a target + if ( + isTslAssignment(statement) && + getAssignees(statement).some((it) => isTslPlaceholder(it) && aggregator.targets.has(it)) + ) { + aggregator.addStatement(statement); + } + } + + return aggregator.statements; + } +} + +class BackwardSliceAggregator { + /** + * The statements that are needed to calculate the target placeholders. + */ + readonly statements: TslStatement[] = []; + + /** + * The target placeholders that should be calculated. + */ + readonly targets: Set; + + constructor(initialTargets: TslPlaceholder[]) { + this.targets = new Set(initialTargets); + } + + addStatement(statement: TslStatement): void { + this.statements.unshift(statement); + + // Remember all referenced placeholders + this.getReferencedPlaceholders(statement).forEach((it) => { + this.targets.add(it); + }); + } + + private getReferencedPlaceholders(node: TslStatement): Stream { + return AstUtils.streamAllContents(node).flatMap((it) => { + if (isTslReference(it) && isTslPlaceholder(it.target.ref)) { + return [it.target.ref]; + } else { + return []; + } + }); + } +} diff --git a/packages/ttsl-lang/src/language/generation/ttsl-python-generator.ts b/packages/ttsl-lang/src/language/generation/ttsl-python-generator.ts index 1b6f6577..eb26ee87 100644 --- a/packages/ttsl-lang/src/language/generation/ttsl-python-generator.ts +++ b/packages/ttsl-lang/src/language/generation/ttsl-python-generator.ts @@ -97,7 +97,8 @@ import { import { TTSLNodeMapper } from '../helpers/ttsl-node-mapper.js'; import { TTSLPartialEvaluator } from '../partialEvaluation/ttsl-partial-evaluator.js'; import { TTSLServices } from '../ttsl-module.js'; -import { TTSLFunction } from '../builtins/ttsl-ds-functions.js'; +import { TTSLFunction } from '../builtins/ttsl-functions.js'; +import { TTSLSlicer } from '../flow/ttsl-slicer.js'; export const CODEGEN_PREFIX = '__gen_'; @@ -317,11 +318,13 @@ export class TTSLPythonGenerator { private readonly builtinFunction: TTSLFunction; private readonly nodeMapper: TTSLNodeMapper; private readonly partialEvaluator: TTSLPartialEvaluator; + private readonly slicer: TTSLSlicer; constructor(services: TTSLServices) { this.builtinFunction = services.builtins.Functions; this.nodeMapper = services.helpers.NodeMapper; this.partialEvaluator = services.evaluation.PartialEvaluator; + this.slicer = services.flow.Slicer; } generate(document: LangiumDocument, generateOptions: GenerateOptions): TextDocument[] { @@ -553,10 +556,12 @@ export class TTSLPythonGenerator { frame: GenerationInfoFrame, timeunit: TslTimeunit | undefined, ): CompositeGeneratorNode { - const targetPlaceholder = getPlaceholderByName(block, frame.targetPlaceholder); let statements = getStatements(block); - if (targetPlaceholder) { - statements = this.getStatementsNeededForPartialExecution(targetPlaceholder, statements); + if (frame.targetPlaceholder) { + const targetPlaceholders = frame.targetPlaceholder.flatMap((it) => getPlaceholderByName(block, it) ?? []); + if (!isEmpty(targetPlaceholders)) { + statements = this.slicer.computeBackwardSlice(statements, targetPlaceholders); + } } let resultBlock = new CompositeGeneratorNode(); let returnStatement = statements.filter(isTslReturnStatement).at(0); @@ -758,10 +763,12 @@ export class TTSLPythonGenerator { block: TslBlock, frame: GenerationInfoFrame, ): CompositeGeneratorNode { - const targetPlaceholder = getPlaceholderByName(block, frame.targetPlaceholder); let statements = getStatements(block); - if (targetPlaceholder) { - statements = this.getStatementsNeededForPartialExecution(targetPlaceholder, statements); + if (frame.targetPlaceholder) { + const targetPlaceholders = frame.targetPlaceholder.flatMap((it) => getPlaceholderByName(block, it) ?? []); + if (!isEmpty(targetPlaceholders)) { + statements = this.slicer.computeBackwardSlice(statements, targetPlaceholders); + } } if (statements.length === 0 && !isTslForLoop(block.$container)) { return traceToNode(block)('pass'); @@ -1335,7 +1342,7 @@ class GenerationInfoFrame { private readonly utilitySet: Set; private readonly typeVariableSet: Set; public readonly isInsideFunction: boolean; - public readonly targetPlaceholder: string | undefined; + public readonly targetPlaceholder: string[] | undefined; public readonly disableRunnerIntegration: boolean; constructor( @@ -1343,7 +1350,7 @@ class GenerationInfoFrame { utilitySet: Set = new Set(), typeVariableSet: Set = new Set(), insideFunction: boolean = false, - targetPlaceholder: string | undefined = undefined, + targetPlaceholder: string[] | undefined = undefined, disableRunnerIntegration: boolean = false, ) { this.importSet = importSet; @@ -1385,6 +1392,6 @@ class GenerationInfoFrame { export interface GenerateOptions { destination: URI; createSourceMaps: boolean; - targetPlaceholder: string | undefined; + targetPlaceholder: string[] | undefined; disableRunnerIntegration: boolean; } diff --git a/packages/ttsl-lang/src/language/index.ts b/packages/ttsl-lang/src/language/index.ts index 542d597f..d021a84f 100644 --- a/packages/ttsl-lang/src/language/index.ts +++ b/packages/ttsl-lang/src/language/index.ts @@ -1,3 +1,5 @@ +import { pipVersionRange } from './runtime/ttsl-python-server.js'; + // Services export type { TTSLServices } from './ttsl-module.js'; export { createTTSLServices } from './ttsl-module.js'; @@ -14,3 +16,12 @@ export * from './helpers/nodeProperties.js'; // Location export { locationToString, positionToString, rangeToString } from '../helpers/locations.js'; + +// RPC +export * as rpc from './communication/rpc.js'; + +export const dependencies = { + 'ttsl-runner': { + pipVersionRange, + }, +}; \ No newline at end of file diff --git a/packages/ttsl-lang/src/language/runtime/messages.ts b/packages/ttsl-lang/src/language/runtime/messages.ts new file mode 100644 index 00000000..c3bb1a85 --- /dev/null +++ b/packages/ttsl-lang/src/language/runtime/messages.ts @@ -0,0 +1,282 @@ +/** + * Any message that can be sent or received by the runner. + * + * The type field identifies the type of the message. + * + * The id field is a unique identifier to track messages to their origin. + * A program message contains the id. A response message containing an error, progress or a placeholder then contains the same id. + */ +export type PythonServerMessage = + | ProgramMessage + | PlaceholderQueryMessage + | PlaceholderTypeMessage + | PlaceholderValueMessage + | RuntimeErrorMessage + | RuntimeProgressMessage + | ShutdownMessage; + +export type RuntimeProgress = 'done'; + +// Extension to Runner +/** + * Message that contains a fully executable compiled TTSL pipeline. + */ +export interface ProgramMessage { + type: 'program'; + id: string; + data: ProgramPackageMap; +} + +/** + * Contains code and the description of the main entry point of a pipeline. + */ +export interface ProgramPackageMap { + code: ProgramCodeMap; + main: ProgramMainInformation; + cwd?: string; +} + +/** + * Contains python modules grouped by a virtual directory structure. The key is a path, directories are separated by '.'. + */ +export interface ProgramCodeMap { + [key: string]: ProgramModuleMap; +} + +/** + * Contains python module code identified by the module name. + */ +export interface ProgramModuleMap { + [key: string]: string; +} + +/** + * Contains execution information about a pipeline. + */ +export interface ProgramMainInformation { + /** + * The path to the current module. + */ + modulepath: string; + + /** + * The current module name. + */ + module: string; + + /** + * The function name. + */ + funct: string; +} + +// Extension to Runner +/** + * Message that contains a request to send back the value of a specified placeholder + */ +export interface PlaceholderQueryMessage { + type: 'placeholder_query'; + id: string; + data: PlaceholderQuery; +} + +/** + * A query on a placeholder value. + */ +export interface PlaceholderQuery { + /** + * The name of the requested placeholder. + */ + name: string; + + /** + * Optional windowing information to request a subset of the available data. + */ + window: PlaceholderQueryWindow; +} + +/** + * Windowing information for the placeholder query. + */ +export interface PlaceholderQueryWindow { + /** + * The offset of the requested data. + */ + begin?: number; + + /** + * The size of the requested data. + */ + size?: number; +} + +// Runner to Extension +/** + * Message that contains information about a calculated placeholder. + */ +export interface PlaceholderTypeMessage { + type: 'placeholder_type'; + id: string; + data: PlaceholderDescription; +} + +/** + * Contains the description of a calculated placeholder. + */ +export interface PlaceholderDescription { + /** + * Name of the calculated placeholder. + */ + name: string; + + /** + * Type of the calculated placeholder + */ + type: string; +} + +/** + * Message that contains the value of a calculated placeholder. + */ +export interface PlaceholderValueMessage { + type: 'placeholder_value'; + id: string; + data: PlaceholderValue; +} + +/** + * Contains the description and the value of a calculated placeholder. + */ +export interface PlaceholderValue { + /** + * Name of the calculated placeholder. + */ + name: string; + + /** + * Type of the calculated placeholder. + */ + type: string; + + /** + * Actual value of the calculated placeholder. + */ + value: string; + + /** + * Optional windowing information when only a subset of the data was requested. This may be different from the requested bounds. + */ + window?: PlaceholderValueWindow; +} + +/** + * Windowing information for a placeholder value response. + */ +export interface PlaceholderValueWindow { + /** + * Index offset of the requested data subset. + */ + begin: number; + + /** + * Size of the requested data subset. + */ + size: number; + + /** + * Max. amount of elements available. + */ + max: number; +} + +// Runner to Extension +/** + * Message that contains information about a runtime error that occurred during execution. + */ +export interface RuntimeErrorMessage { + type: 'runtime_error'; + id: string; + data: RuntimeErrorDescription; +} + +/** + * Error description for runtime errors. + */ +export interface RuntimeErrorDescription { + /** + * Error Message + */ + message: string; + + /** + * Array of stackframes at the moment of raising the error. + */ + backtrace: RuntimeErrorBacktraceFrame[]; +} + +/** + * Contains debugging information about a stackframe. + */ +export interface RuntimeErrorBacktraceFrame { + /** + * Python module name (or file name). + */ + file: string; + + /** + * Line number where the error occurred. + */ + line: number; +} + +// Runner to Extension +/** + * Message that contains information about the current execution progress. + * Field data currently supports on of the following: 'done' + * + * A progress value of 'done' means that the pipeline execution completed. + */ +export interface RuntimeProgressMessage { + type: 'runtime_progress'; + id: string; + data: RuntimeProgress; +} + +export const createProgramMessage = function (id: string, data: ProgramPackageMap): PythonServerMessage { + return { type: 'program', id, data }; +}; + +export const createPlaceholderQueryMessage = function ( + id: string, + placeholderName: string, + windowBegin: number | undefined = undefined, + windowSize: number | undefined = undefined, +): PythonServerMessage { + return { + type: 'placeholder_query', + id, + data: { + name: placeholderName, + window: { + begin: !windowBegin ? undefined : Math.round(windowBegin), + size: !windowSize ? undefined : Math.round(windowSize), + }, + }, + }; +}; + +// Extension to Runner +/** + * Message that instructs the runner to shut itself down as soon as possible. + * + * There will be no response to this message, data and id fields are therefore empty. + */ +export interface ShutdownMessage { + type: 'shutdown'; + id: ''; + data: ''; +} + +export const createShutdownMessage = function (): PythonServerMessage { + return { type: 'shutdown', id: '', data: '' }; +}; diff --git a/packages/ttsl-lang/src/language/runtime/ttsl-python-server.ts b/packages/ttsl-lang/src/language/runtime/ttsl-python-server.ts new file mode 100644 index 00000000..2f4673df --- /dev/null +++ b/packages/ttsl-lang/src/language/runtime/ttsl-python-server.ts @@ -0,0 +1,666 @@ +import { TTSLServices } from '../ttsl-module.js'; +import treeKill from 'tree-kill'; +import { TTSLLogger, TTSLMessagingProvider } from '../communication/ttsl-messaging-provider.js'; +import child_process from 'child_process'; +import WebSocket from 'ws'; +import { createShutdownMessage, PythonServerMessage } from './messages.js'; +import { Disposable } from 'langium'; +import { TTSLSettingsProvider } from '../workspace/ttsl-settings-provider.js'; +import semver from 'semver'; +import net, { AddressInfo } from 'node:net'; +import { ChildProcessWithoutNullStreams } from 'node:child_process'; +import { + InstallRunnerNotification, + RunnerStartedNotification, + StartRunnerNotification, + UpdateRunnerNotification, +} from '../communication/rpc.js'; + +const LOWEST_SUPPORTED_RUNNER_VERSION = '0.18.0'; +const LOWEST_UNSUPPORTED_RUNNER_VERSION = '0.19.0'; +const npmVersionRange = `>=${LOWEST_SUPPORTED_RUNNER_VERSION} <${LOWEST_UNSUPPORTED_RUNNER_VERSION}`; +export const pipVersionRange = `>=${LOWEST_SUPPORTED_RUNNER_VERSION},<${LOWEST_UNSUPPORTED_RUNNER_VERSION}`; + +/* c8 ignore start */ +export class TTSLPythonServer { + private readonly logger: TTSLLogger; + private readonly messaging: TTSLMessagingProvider; + private readonly settingsProvider: TTSLSettingsProvider; + + private state: State = stopped; + private restartTracker = new RestartTracker(); + private messageCallbacks: Map void)[]> = new Map(); + + constructor(services: TTSLServices) { + this.logger = services.communication.MessagingProvider.createTaggedLogger('Python Server'); + this.messaging = services.communication.MessagingProvider; + this.settingsProvider = services.workspace.SettingsProvider; + + // Restart if the runner command changes + services.workspace.SettingsProvider.onRunnerCommandUpdate(async () => { + await this.start(); + }); + + // Start if specifically requested. This can happen if the updater installed a new version of the runner but the + // runner command did not have to be changed. + this.messaging.onNotification(StartRunnerNotification.type, async () => { + await this.start(); + }); + + // Stop the Python server when the language server is shut down + services.shared.lsp.Connection?.onShutdown(async () => { + await this.stop(); + }); + } + + // Lifecycle methods ----------------------------------------------------------------------------------------------- + + /** + * Whether the Python server is started and ready to accept requests. + */ + get isStarted(): boolean { + return isStarted(this.state); + } + + /** + * Start the Python server and connect to it. + */ + private async start(): Promise { + if (!isStopped(this.state)) { + return; + } + this.state = starting(); + this.logger.info('Starting...'); + + // Get the runner command + const command = await this.getValidRunnerCommand(); + if (!command) { + this.state = stopped; + return; + } + + // Start the server at a free port + const port = await this.getFreePort(); + this.startServerProcess(command, port); + + // Connect to the server + await this.connectToServer(port); + + // Notify the services in the language client that the process has started. + // TODO: Removed once all the execution logic is in the language server. + if (isStarted(this.state)) { + this.logger.info('Started successfully.'); + await this.messaging.sendNotification(RunnerStartedNotification.type, { port }); + } + } + + /** + * Stop the Python server. + */ + // TODO make private once the execution logic is fully handled in the language server + async stop(): Promise { + if (!isStarting(this.state) && !isStarted(this.state)) { + return; + } + this.state = stopping(this.state?.serverProcess, this.state?.serverConnection); + this.logger.info('Stopping...'); + + // Attempt a graceful shutdown first + await this.stopServerProcessGracefully(2500); + if (isStopped(this.state)) { + this.logger.info('Stopped successfully.'); + return; + } + + // If the graceful shutdown failed, kill the server process + this.logger.debug('Graceful shutdown failed. Killing the server process...'); + await this.killServerProcess(); + if (isStopped(this.state)) { + this.logger.info('Stopped successfully.'); + return; + } + + // The server could not be stopped + this.logger.error('Could not stop the server.'); + this.state = failed; + } + + /** + * Stop the Python server and start it again. + * + * @param shouldBeTracked Whether the restart should be tracked. If `false`, the restart will always be executed. + */ + private async restart(shouldBeTracked: boolean): Promise { + if (shouldBeTracked && !this.restartTracker.shouldRestart()) { + this.logger.error('Restarting too frequently. Aborting.'); + return; + } + + await this.stop(); + await this.start(); + } + + // Command handling ------------------------------------------------------------------------------------------------ + + /** + * Get the runner command from the settings provider and check whether it is valid. + * + * @returns The runner command if it is valid, otherwise `undefined`. + */ + private async getValidRunnerCommand(): Promise { + const command = this.settingsProvider.getRunnerCommand(); + this.logger.debug(`Using runner command "${command}".`); + + // Check whether the runner command is set properly and get the runner version + let installedVersion: string; + try { + installedVersion = await this.getInstalledRunnerVersion(await command); + this.logger.debug(`Found ttsl-runner with version "${installedVersion}".`); + } catch (error) { + await this.reportBadRunnerCommand(await command, error); + return undefined; + } + + // Check whether the runner version is supported + if (!this.isValidVersion(installedVersion)) { + await this.reportInvalidRunnerVersion(installedVersion); + return undefined; + } + + // Check whether a new version of the runner is available + const latestVersion = await this.getLatestMatchingRunnerVersion(); + if (latestVersion && semver.gt(latestVersion, installedVersion)) { + if (await this.reportOutdatedRunner(installedVersion, latestVersion)) { + // Abort the start process if the user wants to update the runner + return undefined; + } + } + + return command; + } + + /** + * Attempt to get the version of the runner command. + * + * @returns A promise that resolves to the version of the runner if it could be determined, otherwise the promise is + * rejected. + */ + private async getInstalledRunnerVersion(command: string): Promise { + const versionProcess = child_process.spawn(command, ['-V']); + + return new Promise((resolve, reject) => { + versionProcess.stdout.on('data', (data: Buffer) => { + const version = data.toString().trim().split(/\s/u)[1]; + if (version !== undefined) { + resolve(version); + } + }); + versionProcess.on('error', (err) => { + reject(new Error(`The subprocess could not be started (${err.message}).`)); + }); + versionProcess.on('close', (code) => { + reject(new Error(`The subprocess closed with code ${code}.`)); + }); + }); + } + + /** + * Get the latest version of the runner in the required version range. + */ + private async getLatestMatchingRunnerVersion(): Promise { + // Get information about `ttsl-runner` from Pypi + const response = await fetch('https://pypi.org/pypi/ttsl-runner/json', { + signal: AbortSignal.timeout(2000), + }); + if (!response.ok) { + this.logger.error(`Could not fetch the latest version of ttsl-runner: ${response.statusText}`); + return undefined; + } + + // Parse the response + try { + const jsonData = await response.json(); + const allReleases = Object.keys(jsonData.releases); + return semver.maxSatisfying(allReleases, `>=0.13.0 <0.14.0`) ?? undefined; + } catch (error) { + this.logger.error(`Could not parse the response from PyPI: ${error}`); + return undefined; + } + } + + /** + * Check whether the available runner is supported. + */ + private isValidVersion(version: string): boolean { + return semver.satisfies(version, npmVersionRange); + } + + // Port handling --------------------------------------------------------------------------------------------------- + + /** + * Get a random free port on the local machine. + */ + private async getFreePort(): Promise { + return new Promise((resolve) => { + const server = net.createServer(); + server.listen(0, '127.0.0.1', () => { + const port = (server.address() as AddressInfo).port; + server.close(() => resolve(port)); + }); + }); + } + + // Process handling ------------------------------------------------------------------------------------------------ + + /** + * Starts the server using the given command and port. + */ + private startServerProcess(command: string, port: number) { + if (!isStarting(this.state)) { + return; + } + + // Spawn the server process + const args = ['start', '--port', String(port)]; + this.logger.debug(`Running "${command} ${args.join(' ')}".`); + const serverProcess = child_process.spawn(command, args); + + // Log the output of the server process + serverProcess.stdout.on('data', (data: Buffer) => { + this.logger.debug(`[Stdout] ${data.toString().trim()}`); + }); + serverProcess.stderr.on('data', (data: Buffer) => { + this.logger.debug(`[Stderr] ${data.toString().trim()}`); + }); + + // Handle the termination of the server process + serverProcess.on('close', (code) => { + this.logger.debug(`Process exited with code ${code}.`); + this.state = stopped; + }); + + // Update the state + this.state = starting(serverProcess); + } + + /** + * Request a graceful shutdown of the server process. + */ + private async stopServerProcessGracefully(maxTimeoutMs: number): Promise { + if (!isStopping(this.state) || !this.state.serverConnection) { + return; + } + this.logger.debug('Trying graceful shutdown...'); + + return new Promise((resolve) => { + // Always resolve after a certain time + const cancelToken = setTimeout(resolve, maxTimeoutMs); + + // Wait for the server process to close + this.state.serverProcess?.on('close', () => { + clearTimeout(cancelToken); + resolve(); + }); + + // Send a shutdown message to the server. Do this last, so we don't miss the close event. + this.sendMessageToPythonServer(createShutdownMessage()); + }); + } + + /** + * Kill the server process forcefully. + */ + private async killServerProcess(): Promise { + if (!isStopping(this.state) || !this.state.serverProcess) { + return; + } + this.logger.debug('Killing process...'); + + // Get the process ID + const pid = this.state.serverProcess?.pid; + if (!pid) { + return; + } + + // Kill the process + await new Promise((resolve) => { + treeKill(pid, (error) => { + if (error) { + this.logger.error(`Error while killing process: ${error}`); + } + + resolve(); + }); + }); + } + + // Socket handling ------------------------------------------------------------------------------------------------- + + /** + * Connect to the server using the given port. + */ + private async connectToServer(port: number): Promise { + try { + await this.doConnectToServer(port); + } catch { + await this.stop(); + } + } + + private async doConnectToServer(port: number): Promise { + if (!isStarting(this.state)) { + return; + } + this.logger.debug(`Connecting to server at port ${port}...`); + + const baseTimeoutMs = 200; + const maxConnectionTries = 8; + let currentTry = 0; + + return new Promise((resolve, reject) => { + const tryConnect = () => { + const serverConnection = new WebSocket(`ws://127.0.0.1:${port}/WSMain`, { + handshakeTimeout: 10 * 1000, + }); + + // Connected successfully + serverConnection.onopen = () => { + this.logger.debug(`Connected successfully.`); + this.state = started(this.state.serverProcess, serverConnection); + resolve(); + }; + + // Handle connection errors + serverConnection.onerror = (event) => { + currentTry += 1; + + // Retry if the connection was refused with exponential backoff + if (event.message.includes('ECONNREFUSED')) { + serverConnection.terminate(); + + if (currentTry > maxConnectionTries) { + this.logger.error('Max retries reached. No further attempt at connecting is made.'); + } else { + this.logger.debug(`Not yet up. Retrying...`); + setTimeout(tryConnect, baseTimeoutMs * 2 ** (currentTry - 1)); // use exponential backoff + } + return; + } + + // Log other errors and reject if the server is not started + this.logger.error(`An error occurred: ${event.type} ${event.message}`); + if (!isStarted(this.state)) { + reject(); + } + }; + + // Handle incoming messages + serverConnection.onmessage = (event) => { + if (typeof event.data !== 'string') { + this.logger.trace(`Message received: (${event.type}, ${typeof event.data}) ${event.data}`); + return; + } + this.logger.trace( + `Message received: '${ + event.data.length > 128 ? event.data.substring(0, 128) + '' : event.data + }'`, + ); + + const pythonServerMessage: PythonServerMessage = JSON.parse(event.data); + if (!this.messageCallbacks.has(pythonServerMessage.type)) { + this.logger.trace(`Message type '${pythonServerMessage.type}' is not handled`, undefined); + return; + } + for (const callback of this.messageCallbacks.get(pythonServerMessage.type)!) { + callback(pythonServerMessage); + } + }; + + // Handle the server closing the connection + serverConnection.onclose = () => { + if ( + isStarted(this.state) && + this.state.serverProcess && + this.state.serverConnection === serverConnection + ) { + this.logger.error('Connection was unexpectedly closed'); + this.restart(true); + } + }; + }; + tryConnect(); + }); + } + + // User interaction ------------------------------------------------------------------------------------------------ + + /** + * Report to the user that the runner cannot be started with the configured command. + */ + private async reportBadRunnerCommand(command: string, error: unknown): Promise { + const message = error instanceof Error ? error.message : String(error); + this.logger.error(`Could not start runner with command "${command}": ${message}`); + + // Show an error message to the user and offer to install the runner + const action = await this.messaging.showErrorMessage(`The runner could not be started.`, { + title: 'Install runner', + }); + if (action?.title === 'Install runner') { + await this.messaging.sendNotification(InstallRunnerNotification.type); + } + } + + /** + * Report to the user that the runner version does not match the required version range. + */ + private async reportInvalidRunnerVersion(version: string): Promise { + this.logger.error(`Installed runner version ${version} is not in range "${pipVersionRange}".`); + + // Show an error message to the user and offer to update the runner + const action = await this.messaging.showErrorMessage( + `The runner must be updated to a version in the range "${pipVersionRange}".`, + { title: 'Update runner' }, + ); + if (action?.title === 'Update runner') { + await this.messaging.sendNotification(UpdateRunnerNotification.type); + } + } + + /** + * Report to the user that the installed runner is outdated. + * + * @returns Whether the user decided to update the runner. Returning `true` aborts the start process. + */ + private async reportOutdatedRunner(installedVersion: string, availableVersion: string): Promise { + this.logger.info( + `Installed runner version ${installedVersion} is outdated. Latest version is ${availableVersion}.`, + ); + + // Show an error message to the user and offer to update the runner + const action = await this.messaging.showInformationMessage(`A new version of the runner is available.`, { + title: 'Update runner', + }); + if (action?.title === 'Update runner') { + await this.messaging.sendNotification(UpdateRunnerNotification.type); + return true; + } else { + return false; + } + } + + // TODO ------------------------------------------------------------------------------------------------------------ + + /** + * Send a message to the python server using the websocket connection. + * + * @param message Message to be sent to the python server. This message should be serializable to JSON. + */ + public sendMessageToPythonServer(message: PythonServerMessage): void { + if (!this.state.serverConnection) { + return; + } + + const messageString = JSON.stringify(message); + this.logger.trace(`Sending message to python server: ${messageString}`); + this.state.serverConnection.send(messageString); + } + + /** + * Register a callback to execute when a message from the python server arrives. + * + * @param messageType Message type to register the callback for. + * @param callback Callback to execute + */ + public addMessageCallback( + messageType: M, + callback: (message: Extract) => void, + ): Disposable { + if (!this.messageCallbacks.has(messageType)) { + this.messageCallbacks.set(messageType, []); + } + this.messageCallbacks.get(messageType)!.push(<(message: PythonServerMessage) => void>callback); + return { + dispose: () => { + if (!this.messageCallbacks.has(messageType)) { + return; + } + this.messageCallbacks.set( + messageType, + this.messageCallbacks.get(messageType)!.filter((storedCallback) => storedCallback !== callback), + ); + }, + }; + } + + /** + * Remove a previously registered callback from being called when a message from the python server arrives. + * + * @param messageType Message type the callback was registered for. + * @param callback Callback to remove + */ + public removeMessageCallback( + messageType: M, + callback: (message: Extract) => void, + ): void { + if (!this.messageCallbacks.has(messageType)) { + return; + } + this.messageCallbacks.set( + messageType, + this.messageCallbacks.get(messageType)!.filter((storedCallback) => storedCallback !== callback), + ); + } + + async connectToPort(port: number): Promise { + if (!isStopped(this.state)) { + return; + } + this.state = starting(); + + try { + await this.doConnectToServer(port); + } catch (error) { + await this.stop(); + } + } +} + +// State --------------------------------------------------------------------------------------------------------------- + +/** + * The Python server process is stopped. + */ +const stopped = { + type: 'stopped', + serverProcess: undefined, + serverConnection: undefined, +} as const; + +const isStopped = (state: State): state is typeof stopped => state === stopped; + +/** + * The Python server process is being started. + */ +interface Starting { + type: 'starting'; + serverProcess?: ChildProcessWithoutNullStreams; + serverConnection: undefined; +} + +const starting = (serverProcess?: ChildProcessWithoutNullStreams): Starting => ({ + type: 'starting', + serverProcess, + serverConnection: undefined, +}); + +const isStarting = (state: State): state is Starting => state.type === 'starting'; + +/** + * The Python server process is started, and we are connected to it. + */ +interface Started { + type: 'started'; + // TODO: Should always be defined once the execution is fully handled in the language server + serverProcess?: ChildProcessWithoutNullStreams; + serverConnection: WebSocket; +} + +const started = (serverProcess: ChildProcessWithoutNullStreams | undefined, serverConnection: WebSocket): Started => ({ + type: 'started', + serverProcess, + serverConnection, +}); + +const isStarted = (state: State): state is Started => state.type === 'started'; + +/** + * The Python server process is being stopped. + */ +interface Stopping { + type: 'stopping'; + serverProcess?: ChildProcessWithoutNullStreams; + serverConnection?: WebSocket; +} + +const stopping = ( + serverProcess: ChildProcessWithoutNullStreams | undefined, + serverConnection: WebSocket | undefined, +): Stopping => ({ + type: 'stopping', + serverProcess, + serverConnection, +}); + +const isStopping = (state: State): state is Stopping => state.type === 'stopping'; + +/** + * Something went wrong. + */ +const failed = { + type: 'failed', + serverProcess: undefined, + serverConnection: undefined, +} as const; + +type State = typeof stopped | Starting | Started | Stopping | typeof failed; + +// Restart tracking ---------------------------------------------------------------------------------------------------- + +/** + * Tracks restarts of the Python server. + */ +class RestartTracker { + private timestamps: number[] = []; + + /** + * Add a timestamp to the tracker and check whether the server should be restarted. + */ + shouldRestart(): boolean { + const now = Date.now(); + this.timestamps.push(now); + this.timestamps = this.timestamps.filter((timestamp) => now - timestamp < 60_000); + return this.timestamps.length <= 5; + } +} + +/* c8 ignore stop */ diff --git a/packages/ttsl-lang/src/language/runtime/ttsl-runner.ts b/packages/ttsl-lang/src/language/runtime/ttsl-runner.ts new file mode 100644 index 00000000..2d82e702 --- /dev/null +++ b/packages/ttsl-lang/src/language/runtime/ttsl-runner.ts @@ -0,0 +1,533 @@ +import { TTSLServices } from '../ttsl-module.js'; +import { AstNodeLocator, AstUtils, LangiumDocument, LangiumDocuments, URI } from 'langium'; +import path from 'path'; +import { + createPlaceholderQueryMessage, + createProgramMessage, + PlaceholderValueMessage, + ProgramCodeMap, + RuntimeErrorBacktraceFrame, + RuntimeErrorMessage, +} from './messages.js'; +import { SourceMapConsumer } from 'source-map-js'; +import { TTSLPythonGenerator } from '../generation/ttsl-python-generator.js'; +import { isTslFunction, isTslModule, isTslPlaceholder } from '../generated/ast.js'; +import { TTSLLogger, TTSLMessagingProvider } from '../communication/ttsl-messaging-provider.js'; +import crypto from 'crypto'; +import { TTSLPythonServer } from './ttsl-python-server.js'; +import { IsRunnerReadyRequest, ShowImageNotification } from '../communication/rpc.js'; +import { expandToStringLF, joinToNode } from 'langium/generate'; +import { TTSLFunction } from '../builtins/ttsl-functions.js'; + +// Most of the functionality cannot be tested automatically as a functioning runner setup would always be required + +const RUNNER_TAG = 'Runner'; + +/* c8 ignore start */ +export class TTSLRunner { + private readonly astNodeLocator: AstNodeLocator; + private readonly generator: TTSLPythonGenerator; + private readonly langiumDocuments: LangiumDocuments; + private readonly logger: TTSLLogger; + private readonly messaging: TTSLMessagingProvider; + private readonly pythonServer: TTSLPythonServer; + private readonly ttslFunct: TTSLFunction; + + constructor(services: TTSLServices) { + this.astNodeLocator = services.workspace.AstNodeLocator; + this.generator = services.generation.PythonGenerator; + this.langiumDocuments = services.shared.workspace.LangiumDocuments; + this.logger = services.communication.MessagingProvider.createTaggedLogger(RUNNER_TAG); + this.messaging = services.communication.MessagingProvider; + this.pythonServer = services.runtime.PythonServer; + this.ttslFunct = services.builtins.Functions + + this.registerMessageLoggingCallbacks(); + + this.messaging.onRequest(IsRunnerReadyRequest.type, () => { + return this.isReady(); + }); + } + + /** + * Check if the runner is ready to execute functions. + */ + isReady(): boolean { + return this.pythonServer.isStarted; + } + + async runFunction(documentUri: string, nodePath: string) { + const uri = URI.parse(documentUri); + const document = this.langiumDocuments.getDocument(uri); + if (!document) { + this.messaging.showErrorMessage('Could not find document.'); + return; + } + + const root = document.parseResult.value; + const funct = this.astNodeLocator.getAstNode(root, nodePath); + if (!isTslFunction(funct)) { + this.messaging.showErrorMessage('Selected node is not a function.'); + return; + } + + const functExecutionId = crypto.randomUUID(); + + const start = Date.now(); + const progress = await this.messaging.showProgress('TTSL Runner', 'Starting...'); + this.logger.info(`[${functExecutionId}] Running function "${funct.name}" in ${documentUri}.`); + + const disposables = [ + this.pythonServer.addMessageCallback('placeholder_type', (message) => { + if (message.id === functExecutionId) { + progress.report(`Computed ${message.data.name}`); + } + }), + + this.pythonServer.addMessageCallback('runtime_error', (message) => { + if (message.id === functExecutionId) { + progress?.done(); + disposables.forEach((it) => { + it.dispose(); + }); + this.messaging.showErrorMessage('An error occurred during function execution.'); + } + progress.done(); + disposables.forEach((it) => { + it.dispose(); + }); + }), + + this.pythonServer.addMessageCallback('runtime_progress', (message) => { + if (message.id === functExecutionId) { + progress.done(); + const timeElapsed = Date.now() - start; + this.logger.info( + `[${functExecutionId}] Finished running function "${funct.name}" in ${timeElapsed}ms.`, + ); + disposables.forEach((it) => { + it.dispose(); + }); + } + }), + ]; + + await this.executeFunction(functExecutionId, document, funct.name); + } + + async printValue(documentUri: string, nodePath: string) { + const uri = URI.parse(documentUri); + const document = this.langiumDocuments.getDocument(uri); + if (!document) { + this.messaging.showErrorMessage('Could not find document.'); + return; + } + + const root = document.parseResult.value; + const placeholder = this.astNodeLocator.getAstNode(root, nodePath); + if (!isTslPlaceholder(placeholder)) { + this.messaging.showErrorMessage('Selected node is not a placeholder.'); + return; + } + + const funct = AstUtils.getContainerOfType(placeholder, isTslFunction); + if (!funct) { + this.messaging.showErrorMessage('Could not find function.'); + return; + } + + const functExecutionId = crypto.randomUUID(); + + const start = Date.now(); + + const progress = await this.messaging.showProgress('TTSL Runner', 'Starting...'); + + this.logger.info( + `[${functExecutionId}] Printing value "${funct.name}/${placeholder.name}" in ${documentUri}.`, + ); + + const disposables = [ + this.pythonServer.addMessageCallback('runtime_error', (message) => { + if (message.id === functExecutionId) { + progress?.done(); + disposables.forEach((it) => { + it.dispose(); + }); + this.messaging.showErrorMessage('An error occurred during function execution.'); + } + progress.done(); + disposables.forEach((it) => { + it.dispose(); + }); + }), + + this.pythonServer.addMessageCallback('placeholder_type', async (message) => { + if (message.id === functExecutionId && message.data.name === placeholder.name) { + const data = await this.getPlaceholderValue(placeholder.name, functExecutionId); + this.logger.result(`val ${placeholder.name} = ${JSON.stringify(data, null, 2)};`); + } + }), + + this.pythonServer.addMessageCallback('runtime_progress', (message) => { + if (message.id === functExecutionId) { + progress.done(); + const timeElapsed = Date.now() - start; + this.logger.info( + `[${functExecutionId}] Finished printing value "${funct.name}/${placeholder.name}" in ${timeElapsed}ms.`, + ); + disposables.forEach((it) => { + it.dispose(); + }); + } + }), + ]; + + await this.executeFunction(functExecutionId, document, funct.name, [placeholder.name]); + } + + async showImage(documentUri: string, nodePath: string) { + const uri = URI.parse(documentUri); + const document = this.langiumDocuments.getDocument(uri); + if (!document) { + this.messaging.showErrorMessage('Could not find document.'); + return; + } + + const root = document.parseResult.value; + const placeholder = this.astNodeLocator.getAstNode(root, nodePath); + if (!isTslPlaceholder(placeholder)) { + this.messaging.showErrorMessage('Selected node is not a placeholder.'); + return; + } + + const funct = AstUtils.getContainerOfType(placeholder, isTslFunction); + if (!funct) { + this.messaging.showErrorMessage('Could not find function.'); + return; + } + + const functExecutionId = crypto.randomUUID(); + + const start = Date.now(); + + const progress = await this.messaging.showProgress('TTSL Runner', 'Starting...'); + + this.logger.info( + `[${functExecutionId}] Showing image "${funct.name}/${placeholder.name}" in ${documentUri}.`, + ); + + const disposables = [ + this.pythonServer.addMessageCallback('runtime_error', (message) => { + if (message.id === functExecutionId) { + progress?.done(); + disposables.forEach((it) => { + it.dispose(); + }); + this.messaging.showErrorMessage('An error occurred during function execution.'); + } + progress.done(); + disposables.forEach((it) => { + it.dispose(); + }); + }), + + this.pythonServer.addMessageCallback('placeholder_type', async (message) => { + if (message.id === functExecutionId && message.data.name === placeholder.name) { + const data = await this.getPlaceholderValue(placeholder.name, functExecutionId); + await this.messaging.sendNotification(ShowImageNotification.type, { image: data }); + } + }), + + this.pythonServer.addMessageCallback('runtime_progress', (message) => { + if (message.id === functExecutionId) { + progress.done(); + const timeElapsed = Date.now() - start; + this.logger.info( + `[${functExecutionId}] Finished showing image "${funct.name}/${placeholder.name}" in ${timeElapsed}ms.`, + ); + disposables.forEach((it) => { + it.dispose(); + }); + } + }), + ]; + + await this.executeFunction(functExecutionId, document, funct.name, [placeholder.name]); + } + + private async getPlaceholderValue(placeholder: string, functExecutionId: string): Promise { + return new Promise((resolve) => { + if (placeholder === '') { + resolve(undefined); + } + + const placeholderValueCallback = (message: PlaceholderValueMessage) => { + if (message.id !== functExecutionId || message.data.name !== placeholder) { + return; + } + this.pythonServer.removeMessageCallback('placeholder_value', placeholderValueCallback); + resolve(message.data.value); + }; + + this.pythonServer.addMessageCallback('placeholder_value', placeholderValueCallback); + this.logger.info('Getting placeholder from Runner ...'); + this.pythonServer.sendMessageToPythonServer( + createPlaceholderQueryMessage(functExecutionId, placeholder), + ); + + setTimeout(() => { + resolve(undefined); + }, 30000); + }); + } + + /** + * Map that contains information about an execution keyed by the execution id. + */ + public executionInformation: Map = new Map< + string, + FunctionExecutionInformation + >(); + + /** + * Get information about a function execution. + * + * @param functId Unique id that identifies a function execution + * @return Execution context assigned to the provided id. + */ + public getExecutionContext(functId: string): FunctionExecutionInformation | undefined { + return this.executionInformation.get(functId); + } + + /** + * Remove information from a function execution, when it is no longer needed. + * + * @param functId Unique id that identifies a function execution + */ + public dropFunctionExecutionContext(functId: string) { + this.executionInformation.delete(functId); + } + + /** + * Remove information from all previous function executions. + */ + public dropAllFunctionExecutionContexts() { + this.executionInformation.clear(); + } + + /** + * Execute a TTSL function on the python runner. + * If a valid target placeholder is provided, the function is only executed partially, to calculate the result of the placeholder. + * + * @param id A unique id that is used in further communication with this function. + * @param functDocument Document containing the main TTSL function to execute. + * @param functName Name of the function that should be run + * @param targetPlaceholders The names of the target placeholders, used to do partial execution. If undefined is provided, the entire function is run. + */ + public async executeFunction( + id: string, + functDocument: LangiumDocument, + functName: string, + targetPlaceholders: string[] | undefined = undefined, + ) { + const node = functDocument.parseResult.value; + if (!isTslModule(node)) { + return; + } + // Function / Module name handling + const mainPythonModuleName = this.ttslFunct.getPythonModule(node); + const mainPackage = mainPythonModuleName === undefined ? node.name.split('.') : [mainPythonModuleName]; + const mainModuleName = this.getMainModuleName(functDocument); + // Code generation + const [codeMap, lastGeneratedSources] = this.generateCodeForRunner(functDocument, targetPlaceholders); + // Store information about the run + this.executionInformation.set(id, { + generatedSource: lastGeneratedSources, + sourceMappings: new Map(), + path: functDocument.uri.fsPath, + source: functDocument.textDocument.getText(), + calculatedPlaceholders: new Map(), + }); + // Code execution + this.pythonServer.sendMessageToPythonServer( + createProgramMessage(id, { + code: codeMap, + main: { + modulepath: mainPackage.join('.'), + module: mainModuleName, + funct: functName, + }, + cwd: path.parse(functDocument.uri.fsPath).dir, + }), + ); + } + + private registerMessageLoggingCallbacks() { + this.pythonServer.addMessageCallback('placeholder_value', (message) => { + this.logger.trace( + `Placeholder value is (${message.id}): ${message.data.name} of type ${message.data.type} = ${message.data.value}`, + undefined, + ); + }); + this.pythonServer.addMessageCallback('placeholder_type', (message) => { + this.logger.trace( + `Placeholder was calculated (${message.id}): ${message.data.name} of type ${message.data.type}`, + undefined, + ); + const execInfo = this.getExecutionContext(message.id); + execInfo?.calculatedPlaceholders.set(message.data.name, message.data.type); + // this.sendMessageToPythonServer( + // messages.createPlaceholderQueryMessage(message.id, message.data.name), + //); + }); + this.pythonServer.addMessageCallback('runtime_progress', (message) => { + this.logger.trace(`Runner-Progress (${message.id}): ${message.data}`, undefined); + }); + this.pythonServer.addMessageCallback('runtime_error', async (message) => { + let readableStacktraceTTSL: string[] = []; + const execInfo = this.getExecutionContext(message.id)!; + const readableStacktracePython = await Promise.all( + (message).data.backtrace.map(async (frame) => { + const mappedFrame = await this.tryMapToSafeDSSource(message.id, frame); + if (mappedFrame) { + readableStacktraceTTSL.push( + `\tat ${URI.file(execInfo.path)}#${mappedFrame.line} (${execInfo.path} line ${ + mappedFrame.line + })`, + ); + return `\tat ${frame.file} line ${frame.line} (mapped to '${mappedFrame.file}' line ${mappedFrame.line})`; + } + return `\tat ${frame.file} line ${frame.line}`; + }), + ); + this.logger.debug( + `[${message.id}] ${ + (message).data.message + }\n${readableStacktracePython.join('\n')}`, + ); + + this.prettyPrintRuntimeError(message, readableStacktraceTTSL); + }); + } + + private prettyPrintRuntimeError(message: RuntimeErrorMessage, readableStacktraceTTSL: string[]) { + const lines = [...message.data.message.split('\n'), ...readableStacktraceTTSL.reverse()].map((it) => + it.replace('\t', ' '), + ); + + this.logger.result( + expandToStringLF` + // ----- Runtime Error --------------------------------------------------------- + ${joinToNode(lines, { prefix: '// ', appendNewLineIfNotEmpty: true, skipNewLineAfterLastItem: true })} + // ----------------------------------------------------------------------------- + `, + ); + } + + /** + * Map a stack frame from python to TTSL. + * Uses generated sourcemaps to do this. + * If such a mapping does not exist, this function returns undefined. + * + * @param executionId Id that uniquely identifies the execution that produced this stack frame + * @param frame Stack frame from the python execution + */ + private async tryMapToSafeDSSource( + executionId: string, + frame: RuntimeErrorBacktraceFrame | undefined, + ): Promise { + if (!frame) { + return undefined; + } + if (!this.executionInformation.has(executionId)) { + return undefined; + } + const execInfo = this.executionInformation.get(executionId)!; + let sourceMapKeys = Array.from(execInfo.generatedSource.keys() || []).filter((value) => + value.endsWith(`${frame.file}.py.map`), + ); + if (sourceMapKeys.length === 0) { + return undefined; + } + let sourceMapKey = sourceMapKeys[0]!; + if (!execInfo.sourceMappings.has(sourceMapKey)) { + const sourceMapObject = JSON.parse(execInfo.generatedSource.get(sourceMapKey)!); + sourceMapObject.sourcesContent = [execInfo.source]; + const consumer = new SourceMapConsumer(sourceMapObject); + execInfo.sourceMappings.set(sourceMapKey, consumer); + } + const outputPosition = execInfo.sourceMappings.get(sourceMapKey)!.originalPositionFor({ + line: Number(frame.line), + column: 0, + bias: SourceMapConsumer.LEAST_UPPER_BOUND, + }); + return { file: outputPosition.source || '', line: outputPosition.line || 0 }; + } + + public generateCodeForRunner( + functDocument: LangiumDocument, + targetPlaceholder: string[] | undefined, + ): [ProgramCodeMap, Map] { + const rootGenerationDir = path.parse(functDocument.uri.fsPath).dir; + const generatedDocuments = this.generator.generate(functDocument, { + destination: URI.file(rootGenerationDir), // actual directory of main module file + createSourceMaps: true, + targetPlaceholder, + disableRunnerIntegration: false, + }); + const lastGeneratedSources = new Map(); + let codeMap: ProgramCodeMap = {}; + for (const generatedDocument of generatedDocuments) { + const fsPath = URI.parse(generatedDocument.uri).fsPath; + const workspaceRelativeFilePath = path.relative(rootGenerationDir, path.dirname(fsPath)); + const sdsFileName = path.basename(fsPath); + const sdsNoExtFilename = + path.extname(sdsFileName).length > 0 + ? sdsFileName.substring(0, sdsFileName.length - path.extname(sdsFileName).length) + : /* c8 ignore next */ + sdsFileName; + // Put code in map for further use in the extension (e.g. to remap errors) + lastGeneratedSources.set( + path.join(workspaceRelativeFilePath, sdsFileName).replaceAll('\\', '/'), + generatedDocument.getText(), + ); + // Check for sourcemaps after they are already added to the function context + // This needs to happen after lastGeneratedSources.set, as errors would not get mapped otherwise + if (fsPath.endsWith('.map')) { + // exclude sourcemaps from sending to runner + continue; + } + let modulePath = workspaceRelativeFilePath.replaceAll('/', '.').replaceAll('\\', '.'); + if (!codeMap.hasOwnProperty(modulePath)) { + codeMap[modulePath] = {}; + } + // Put code in object for runner + codeMap[modulePath]![sdsNoExtFilename] = generatedDocument.getText(); + } + return [codeMap, lastGeneratedSources]; + } + + public getMainModuleName(functDocument: LangiumDocument): string { + if (functDocument.uri.fsPath.endsWith('.ttsl')) { + return this.generator.sanitizeModuleNameForPython(path.basename(functDocument.uri.fsPath, '.ttsl')); + } else { + return this.generator.sanitizeModuleNameForPython(path.basename(functDocument.uri.fsPath)); + } + } +} + +/** + * Context containing information about the execution of a function. + */ +export interface FunctionExecutionInformation { + source: string; + generatedSource: Map; + sourceMappings: Map; + path: string; + /** + * Maps placeholder name to placeholder type + */ + calculatedPlaceholders: Map; +} + +/* c8 ignore stop */ diff --git a/packages/ttsl-lang/src/language/ttsl-module.ts b/packages/ttsl-lang/src/language/ttsl-module.ts index 85a7a507..03511cae 100644 --- a/packages/ttsl-lang/src/language/ttsl-module.ts +++ b/packages/ttsl-lang/src/language/ttsl-module.ts @@ -33,7 +33,11 @@ import { TTSLPackageManager } from './workspace/ttsl-package-manager.js'; import { TTSLWorkspaceManager } from './workspace/ttsl-workspace-manager.js'; import { TTSLSettingsProvider } from './workspace/ttsl-settings-provider.js'; import { TTSLRenameProvider } from './lsp/ttsl-rename-provider.js'; -import { TTSLFunction } from './builtins/ttsl-ds-functions.js'; +import { TTSLFunction } from './builtins/ttsl-functions.js'; +import { TTSLMessagingProvider } from './communication/ttsl-messaging-provider.js'; +import { TTSLPythonServer } from './runtime/ttsl-python-server.js'; +import { TTSLRunner } from './runtime/ttsl-runner.js'; +import { TTSLSlicer } from './flow/ttsl-slicer.js'; /** * Declaration of custom services - add your own service classes here. @@ -42,11 +46,15 @@ export type TTSLAddedServices = { builtins: { Functions: TTSLFunction; }; + communication: { + MessagingProvider: TTSLMessagingProvider; + }; evaluation: { PartialEvaluator: TTSLPartialEvaluator; }; flow: { CallGraphComputer: TTSLCallGraphComputer; + Slicer: TTSLSlicer; }; generation: { PythonGenerator: TTSLPythonGenerator; @@ -57,6 +65,10 @@ export type TTSLAddedServices = { lsp: { NodeInfoProvider: TTSLNodeInfoProvider; }; + runtime: { + PythonServer: TTSLPythonServer; + Runner: TTSLRunner; + }; types: { TypeChecker: TTSLTypeChecker; TypeComputer: TTSLTypeComputer; @@ -82,6 +94,9 @@ export const TTSLModule: Module new TTSLFunction(services), }, + communication: { + MessagingProvider: (services) => new TTSLMessagingProvider(services), + }, documentation: { CommentProvider: (services) => new TTSLCommentProvider(services), DocumentationProvider: (services) => new TTSLDocumentationProvider(services), @@ -91,6 +106,7 @@ export const TTSLModule: Module new TTSLCallGraphComputer(services), + Slicer: (services) => new TTSLSlicer(services), }, generation: { PythonGenerator: (services) => new TTSLPythonGenerator(services), @@ -115,6 +131,10 @@ export const TTSLModule: Module new TTSLScopeComputation(services), ScopeProvider: (services) => new TTSLScopeProvider(services), }, + runtime: { + PythonServer: (services) => new TTSLPythonServer(services), + Runner: (services) => new TTSLRunner(services), + }, types: { TypeChecker: () => new TTSLTypeChecker(), TypeComputer: (services) => new TTSLTypeComputer(services), diff --git a/packages/ttsl-lang/src/language/workspace/ttsl-settings-provider.ts b/packages/ttsl-lang/src/language/workspace/ttsl-settings-provider.ts index 173e08d7..60f5bea1 100644 --- a/packages/ttsl-lang/src/language/workspace/ttsl-settings-provider.ts +++ b/packages/ttsl-lang/src/language/workspace/ttsl-settings-provider.ts @@ -1,14 +1,35 @@ -import { ConfigurationProvider } from 'langium'; +import { ConfigurationProvider, DeepPartial, Disposable } from 'langium'; import { TTSLServices } from '../ttsl-module.js'; import { TTSLLanguageMetaData } from '../generated/module.js'; export class TTSLSettingsProvider { private readonly configurationProvider: ConfigurationProvider; + private watchers = new Set>(); + constructor(services: TTSLServices) { this.configurationProvider = services.shared.workspace.ConfigurationProvider; } + async getRunnerCommand(): Promise { + /* c8 ignore next 2 */ + return (await this.getRunnerSettings()).command ?? 'ttsl-runner'; + } + + async onRunnerCommandUpdate(callback: (newValue: string | undefined) => void): Promise { + const watcher: SettingsWatcher = { + accessor: (settings) => settings.runner?.command, + callback, + }; + + this.watchers.add(watcher); + + return Disposable.create(() => { + /* c8 ignore next */ + this.watchers.delete(watcher); + }); + } + async shouldValidateCodeStyle(): Promise { return (await this.getValidationSettings()).codeStyle ?? true; } @@ -28,6 +49,15 @@ export class TTSLSettingsProvider { private async getValidationSettings(): Promise> { return (await this.configurationProvider.getConfiguration(TTSLLanguageMetaData.languageId, 'validation')) ?? {}; } + + private async getRunnerSettings(): Promise> { + return (await this.configurationProvider.getConfiguration(TTSLLanguageMetaData.languageId, 'runner')) ?? {}; + } +} + +export interface Settings { + runner: RunnerSettings; + validation: ValidationSettings; } interface ValidationSettings { @@ -36,3 +66,12 @@ interface ValidationSettings { experimentalLibraryElement: boolean; nameConvention: boolean; } + +export interface RunnerSettings { + command: string; +} + +interface SettingsWatcher { + accessor: (settings: DeepPartial) => T; + callback: (newValue: T) => void; +} \ No newline at end of file diff --git a/packages/ttsl-lang/tests/language/generation/safe-ds-python-generator.test.ts b/packages/ttsl-lang/tests/language/generation/safe-ds-python-generator.test.ts index c5cad105..3c3d3dc2 100644 --- a/packages/ttsl-lang/tests/language/generation/safe-ds-python-generator.test.ts +++ b/packages/ttsl-lang/tests/language/generation/safe-ds-python-generator.test.ts @@ -22,10 +22,10 @@ describe('generation', async () => { const documents = await loadDocuments(services, test.inputUris); // Get target placeholder name for "run until" - let runUntilPlaceholderName: string | undefined = undefined; + let runUntilPlaceholderName: string[] | undefined = undefined; if (test.runUntil) { const document = langiumDocuments.getDocument(URI.parse(test.runUntil.uri))!; - runUntilPlaceholderName = document.textDocument.getText(test.runUntil.range); + runUntilPlaceholderName = [document.textDocument.getText(test.runUntil.range)]; } // Generate code for all documents diff --git a/packages/ttsl-lang/tests/language/runtime/messages.test.ts b/packages/ttsl-lang/tests/language/runtime/messages.test.ts new file mode 100644 index 00000000..24242d61 --- /dev/null +++ b/packages/ttsl-lang/tests/language/runtime/messages.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from 'vitest'; +import { ToStringTest } from '../../helpers/testDescription.js'; +import { + createPlaceholderQueryMessage, + createProgramMessage, + createShutdownMessage, + PythonServerMessage, +} from '../../../src/language/runtime/messages.js'; + +describe('runner messages', async () => { + const toStringTests: ToStringTest<() => PythonServerMessage>[] = [ + { + value: () => + createProgramMessage('abcdefgh', { + code: { + a: { + gen_test_a: 'def pipe():\n\tpass\n', + gen_test_a_pipe: "from gen_test_a import pipe\n\nif __name__ == '__main__':\n\tpipe()", + }, + }, + main: { modulepath: 'a', module: 'test_a', pipeline: 'pipe' }, + }), + expectedString: + '{"type":"program","id":"abcdefgh","data":{"code":{"a":{"gen_test_a":"def pipe():\\n\\tpass\\n","gen_test_a_pipe":"from gen_test_a import pipe\\n\\nif __name__ == \'__main__\':\\n\\tpipe()"}},"main":{"modulepath":"a","module":"test_a","pipeline":"pipe"}}}', + }, + { + value: () => createPlaceholderQueryMessage('abcdefg', 'value1', 2, 1), + expectedString: + '{"type":"placeholder_query","id":"abcdefg","data":{"name":"value1","window":{"begin":2,"size":1}}}', + }, + { + value: () => createPlaceholderQueryMessage('abcdefg', 'value1', 1), + expectedString: '{"type":"placeholder_query","id":"abcdefg","data":{"name":"value1","window":{"begin":1}}}', + }, + { + value: () => createPlaceholderQueryMessage('abcdefg', 'value1', undefined, 1), + expectedString: '{"type":"placeholder_query","id":"abcdefg","data":{"name":"value1","window":{"size":1}}}', + }, + { + value: () => createPlaceholderQueryMessage('abcdefg', 'value1'), + expectedString: '{"type":"placeholder_query","id":"abcdefg","data":{"name":"value1","window":{}}}', + }, + { + value: () => createShutdownMessage(), + expectedString: '{"type":"shutdown","id":"","data":""}', + }, + ]; + + describe.each(toStringTests)('stringify', ({ value, expectedString }) => { + it(`should return the expected JSON representation of runner message (type: ${JSON.parse(expectedString).type})`, () => { + expect(JSON.stringify(value())).toStrictEqual(expectedString); + }); + }); +}); diff --git a/packages/ttsl-lang/tests/language/runtime/ttsl-runner.test.ts b/packages/ttsl-lang/tests/language/runtime/ttsl-runner.test.ts new file mode 100644 index 00000000..7765292e --- /dev/null +++ b/packages/ttsl-lang/tests/language/runtime/ttsl-runner.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from 'vitest'; +import { NodeFileSystem } from 'langium/node'; +import { URI } from 'langium'; +import { createTTSLServices } from '../../../src/language/index.js'; + +const services = (await createTTSLServices(NodeFileSystem)).TTSL; +const runner = services.runtime.Runner; + +describe('TTSLRunner', async () => { + describe('getMainModuleName', async () => { + it('sds', async () => { + const document = services.shared.workspace.LangiumDocumentFactory.fromString('', URI.file('/a-b c.ttsl')); + const mainModuleName = runner.getMainModuleName(document); + expect(mainModuleName).toBe('a_b_c'); + }); + it('other', async () => { + const document = services.shared.workspace.LangiumDocumentFactory.fromString( + '', + URI.file('/a-b c.sdsdev2'), + ); + const mainModuleName = runner.getMainModuleName(document); + expect(mainModuleName).toBe('a_b_c_sdsdev2'); + }); + }); + describe('generateCodeForRunner', async () => { + it('generateCodeForRunner', async () => { + const document = services.shared.workspace.LangiumDocumentFactory.fromString( + 'package a\n\nfunction mainFunction() {}', + URI.file('/b.ttsl'), + ); + const [programCodeMap] = runner.generateCodeForRunner(document, undefined); + expect(JSON.stringify(programCodeMap).replaceAll('\\r\\n', '\\n')).toBe( + '{"a":{"gen_b":"# Functions --------------------------------------------------------------------\\n\\ndef mainFunction():\\n pass\\n","gen_b_mainFunction":"from .gen_b import mainFunction\\n\\nif __name__ == \'__main__\':\\n mainFunction()\\n"}}', + ); + }); + }); +}); diff --git a/packages/ttsl-lang/tests/resources/typing/expressions/aggregation/main.ttsl b/packages/ttsl-lang/tests/resources/typing/expressions/aggregation/main.ttsl index b79d949d..ba03f647 100644 --- a/packages/ttsl-lang/tests/resources/typing/expressions/aggregation/main.ttsl +++ b/packages/ttsl-lang/tests/resources/typing/expressions/aggregation/main.ttsl @@ -8,12 +8,12 @@ id data testID: Int function testFunction() groupedBy testID { # $TEST$ serialization Int - var int = aggregate "sum" of IntData groupedBy testID; + var »int« = aggregate "sum" of IntData groupedBy testID; # $TEST$ serialization Float - var float = aggregate "sum" of FloatData groupedBy testID; + var »float« = aggregate "sum" of FloatData groupedBy testID; # $TEST$ serialization String - var string = aggregate "append" of StringData groupedBy testID; + var »string« = aggregate "append" of StringData groupedBy testID; # $TEST$ serialization Boolean - var boolean = aggregate "and" of BooleanData groupedBy testID; + var »bool« = aggregate "and" of BooleanData groupedBy testID; } diff --git a/packages/ttsl-vscode/package.json b/packages/ttsl-vscode/package.json index d5cb7c27..6265cfe5 100644 --- a/packages/ttsl-vscode/package.json +++ b/packages/ttsl-vscode/package.json @@ -109,10 +109,25 @@ "title": "Dump Diagnostics to JSON", "category": "TTSL" }, + { + "command": "ttsl.installRunner", + "title": "Install the TTSL Runner", + "category": "TTSL" + }, { "command": "ttsl.openDiagnosticsDumps", "title": "Open Diagnostics Dumps in New VS Code Window", "category": "TTSL" + }, + { + "command": "ttsl.refreshWebview", + "title": "Refresh Webview", + "category": "TTSL" + }, + { + "command": "ttsl.updateRunner", + "title": "Update the TTSL Runner", + "category": "TTSL" } ], "snippets": [ diff --git a/packages/ttsl-vscode/src/extension/actions/installRunner.ts b/packages/ttsl-vscode/src/extension/actions/installRunner.ts new file mode 100644 index 00000000..6b5f90ca --- /dev/null +++ b/packages/ttsl-vscode/src/extension/actions/installRunner.ts @@ -0,0 +1,189 @@ +import vscode, { Uri } from 'vscode'; +import child_process from 'node:child_process'; +import semver from 'semver'; +import { dependencies, rpc } from '@ttsl/lang'; +import { LanguageClient } from 'vscode-languageclient/node.js'; +import { ttslLogger } from '../helpers/logging.js'; + +const pythonCommandCandidates = ['python3', 'python', 'py']; + +const LOWEST_SUPPORTED_PYTHON_VERSION = '3.11.0'; +const LOWEST_UNSUPPORTED_PYTHON_VERSION = '3.13.0'; +const npmVersionRange = `>=${LOWEST_SUPPORTED_PYTHON_VERSION} <${LOWEST_UNSUPPORTED_PYTHON_VERSION}`; + +export const installRunner = (client: LanguageClient) => { + return async () => { + // If the runner is already started, do nothing + if (await client.sendRequest(rpc.IsRunnerReadyRequest.type)) { + vscode.window.showInformationMessage('The runner is already installed and running.'); + return; + } + + // Ask the user where the virtual environment should be created + const runnerVirtualEnvironmentUris = await vscode.window.showOpenDialog({ + canSelectFolders: true, + canSelectMany: false, + title: 'Location for the runner installation', + }); + + if (!runnerVirtualEnvironmentUris || runnerVirtualEnvironmentUris.length === 0) { + return; + } + const runnerVirtualEnvironmentUri = runnerVirtualEnvironmentUris[0]!; + + // Install the runner if it is not already installed + const success = await doInstallRunner(runnerVirtualEnvironmentUri); + if (!success) { + return; + } + + // Set the runner command in the configuration + await vscode.workspace + .getConfiguration() + .update( + 'ttsl.runner.command', + getRunnerCommand(runnerVirtualEnvironmentUri), + vscode.ConfigurationTarget.Global, + ); + + // Start the runner (needed if the configuration did not change, so no event is fired) + await client.sendNotification(rpc.StartRunnerNotification.type); + + // Inform the user + vscode.window.showInformationMessage('The runner has been installed successfully.'); + }; +}; + +/** + * Installs the runner in a virtual environment. Returns true if the installation was successful. + */ +const doInstallRunner = async (runnerVirtualEnvironmentUri: Uri): Promise => { + // Check if a matching Python interpreter is available + const pythonCommand = await getPythonCommand(); + if (!pythonCommand) { + vscode.window.showErrorMessage('Could not find a matching Python interpreter.'); + ttslLogger.error('Could not find a matching Python interpreter.'); + return false; + } + + // Create a virtual environment for the runner + let success = await vscode.window.withProgress( + { + location: vscode.ProgressLocation.Window, + title: 'Creating a virtual environment...', + }, + async () => { + try { + await createRunnerVirtualEnvironment(runnerVirtualEnvironmentUri, pythonCommand); + return true; + } catch (error) { + vscode.window.showErrorMessage('Failed to create a virtual environment.'); + ttslLogger.error(String(error)); + return false; + } + }, + ); + if (!success) { + return false; + } + + // Install the runner in the virtual environment + success = await vscode.window.withProgress( + { + location: vscode.ProgressLocation.Window, + title: 'Installing the runner (this may take a few minutes)...', + }, + async () => { + try { + await installRunnerInVirtualEnvironment(getPipCommand(runnerVirtualEnvironmentUri)); + return true; + } catch (error) { + vscode.window.showErrorMessage('Failed to install the runner.'); + ttslLogger.error(String(error)); + return false; + } + }, + ); + return success; +}; + +const getPythonCommand = async (): Promise => { + for (const candidate of pythonCommandCandidates) { + if (await isMatchingPython(candidate)) { + return candidate; + } + } + + return undefined; +}; + +const isMatchingPython = async (pythonCommand: string): Promise => { + return new Promise((resolve) => { + child_process.exec( + `${pythonCommand} -c "import platform; print(platform.python_version())"`, + (error, stdout) => { + if (!error && semver.satisfies(stdout, npmVersionRange)) { + resolve(true); + } else { + resolve(false); + } + }, + ); + }); +}; + +const createRunnerVirtualEnvironment = async ( + runnerVirtualEnvironmentUri: Uri, + pythonCommand: string, +): Promise => { + return new Promise((resolve, reject) => { + child_process.exec(`"${pythonCommand}" -m venv "${runnerVirtualEnvironmentUri.fsPath}"`, (error) => { + if (error) { + reject(error); + } else { + resolve(); + } + }); + }); +}; + +export const installRunnerInVirtualEnvironment = async (pipCommand: string): Promise => { + return new Promise((resolve, reject) => { + const installCommand = `"${pipCommand}" install --upgrade "ttsl-runner${dependencies['ttsl-runner'].pipVersionRange}"`; + const process = child_process.spawn(installCommand, { shell: true }); + + process.stdout.on('data', (data: Buffer) => { + ttslLogger.debug(data.toString().trim()); + }); + process.stderr.on('data', (data: Buffer) => { + ttslLogger.error(data.toString().trim()); + }); + + process.on('error', (error) => { + reject(error); + }); + process.on('close', (code) => { + if (code === 0) { + resolve(); + } else { + reject(`Runner installation failed with code ${code}.`); + } + }); + }); +}; + +const getPipCommand = (runnerVirtualEnvironmentUri: Uri): string => { + if (process.platform === 'win32') { + return `${runnerVirtualEnvironmentUri.fsPath}\\Scripts\\pip.exe`; + } else { + return `${runnerVirtualEnvironmentUri.fsPath}/bin/pip`; + } +}; + +const getRunnerCommand = (runnerVirtualEnvironmentUri: Uri): string => { + if (process.platform === 'win32') { + return `${runnerVirtualEnvironmentUri.fsPath}\\Scripts\\ttsl-runner.exe`; + } else { + return `${runnerVirtualEnvironmentUri.fsPath}/bin/ttsl-runner`; + } +}; diff --git a/packages/ttsl-vscode/src/extension/actions/showImage.ts b/packages/ttsl-vscode/src/extension/actions/showImage.ts new file mode 100644 index 00000000..52b7ed96 --- /dev/null +++ b/packages/ttsl-vscode/src/extension/actions/showImage.ts @@ -0,0 +1,22 @@ +import vscode, { ExtensionContext, Uri } from 'vscode'; +import { rpc } from '@ttsl/lang'; + +export const showImage = (context: ExtensionContext) => { + return async ({ image }: rpc.ShowImageParams) => { + // Write the image to a file + const uri = imageUri(context); + await vscode.workspace.fs.writeFile(uri, Buffer.from(image.bytes, 'base64')); + + // Open the image in a preview editor + vscode.commands.executeCommand('vscode.openWith', uri, 'imagePreview.previewEditor', { + viewColumn: vscode.ViewColumn.Beside, + preview: true, + preserveFocus: true, + }); + }; +}; + +const imageUri = (context: ExtensionContext): Uri => { + const storageUri = context.storageUri ?? context.globalStorageUri; + return vscode.Uri.joinPath(storageUri, 'results', 'image.png'); +}; diff --git a/packages/ttsl-vscode/src/extension/actions/updateRunner.ts b/packages/ttsl-vscode/src/extension/actions/updateRunner.ts new file mode 100644 index 00000000..2cf83942 --- /dev/null +++ b/packages/ttsl-vscode/src/extension/actions/updateRunner.ts @@ -0,0 +1,82 @@ +import vscode, { ExtensionContext } from 'vscode'; +import { LanguageClient } from 'vscode-languageclient/node.js'; +import { rpc } from '@ttsl/lang'; +import fs from 'node:fs'; +import path from 'node:path'; +import { installRunner, installRunnerInVirtualEnvironment } from './installRunner.js'; +import { platform } from 'node:os'; +import { ttslLogger } from '../helpers/logging.js'; + +export const updateRunner = (context: ExtensionContext, client: LanguageClient) => { + return async () => { + // If the runner is already started, do nothing + if (await client.sendRequest(rpc.IsRunnerReadyRequest.type)) { + vscode.window.showInformationMessage('The runner is already installed and running.'); + return; + } + + // If the runner executable cannot be found at all, install it from scratch + if (!fs.existsSync(await getRunnerCommand())) { + await installRunner(client)(); + return; + } + + // Update the runner if it is already installed + const success = await doUpdateRunner(); + if (!success) { + return; + } + + // Start the runner (needed if the configuration did not change, so no event is fired) + await client.sendNotification(rpc.StartRunnerNotification.type); + + // Inform the user + vscode.window.showInformationMessage('The runner has been updated successfully.'); + }; +}; + +const doUpdateRunner = async (): Promise => { + // Check if pip is available + const pipCommand = await getPipCommand(); + if (!pipCommand) { + vscode.window.showErrorMessage('Failed to find pip.'); + ttslLogger.error('Failed to find pip.'); + return false; + } + + // Install the runner in the virtual environment + return vscode.window.withProgress( + { + location: vscode.ProgressLocation.Window, + title: 'Installing the runner (this may take a few minutes)...', + }, + async () => { + try { + await installRunnerInVirtualEnvironment(pipCommand); + return true; + } catch (error) { + vscode.window.showErrorMessage('Failed to install the runner.'); + ttslLogger.error(String(error)); + return false; + } + }, + ); +}; + +const getRunnerCommand = async (): Promise => { + return vscode.workspace.getConfiguration('ttsl.runner').get('command') ?? ''; +}; + +const getPipCommand = async (): Promise => { + const runnerCommand = await getRunnerCommand(); + if (!runnerCommand) { + return; + } + + const runnerDir = path.dirname(runnerCommand); + if (platform() === 'win32') { + return path.join(runnerDir, 'pip.exe'); + } else { + return path.join(runnerDir, 'pip'); + } +}; diff --git a/packages/ttsl-vscode/src/extension/helpers/logging.ts b/packages/ttsl-vscode/src/extension/helpers/logging.ts new file mode 100644 index 00000000..992cf53a --- /dev/null +++ b/packages/ttsl-vscode/src/extension/helpers/logging.ts @@ -0,0 +1,137 @@ +import vscode, { LogLevel, LogOutputChannel, OutputChannel, ViewColumn } from 'vscode'; + +const TRACE_PREFIX = /^\[Trace.*?\] /iu; +const DEBUG_PREFIX = /^\[Debug.*?\] /iu; +const INFO_PREFIX = /^\[Info.*?\] /iu; +const WARN_PREFIX = /^\[Warn.*?\] /iu; +const ERROR_PREFIX = /^\[Error.*?\] /iu; + +const RESULT_PREFIX = /^.*\[Result\] /iu; + +const RUNNER_TRACE_PREFIX = /^.*\[Python Server\].*\[?INFO\]?:?/iu; +const RUNNER_DEBUG_PREFIX = /^.*\[Python Server\].*\[?DEBUG\]?:?/iu; +const RUNNER_INFO_PREFIX = /^.*\[Python Server\].*\[?INFO\]?:?/iu; +const RUNNER_WARN_PREFIX = /^.*\[Python Server\].*\[?WARNING\]?:?/iu; +const RUNNER_ERROR_PREFIX = /^.*\[Python Server\].*\[?ERROR\]?:?/iu; + +class TTSLLogger implements LogOutputChannel { + private readonly languageServer: LogOutputChannel; + private readonly results: OutputChannel; + private readonly runner: LogOutputChannel; + + constructor() { + this.languageServer = vscode.window.createOutputChannel('Safe-DS', { log: true }); + this.results = vscode.window.createOutputChannel('Safe-DS Results', 'safe-ds'); + this.runner = vscode.window.createOutputChannel('Safe-DS Runner', { log: true }); + } + + get logLevel(): LogLevel { + return this.languageServer.logLevel; + } + + get name(): string { + return this.languageServer.name; + } + + get onDidChangeLogLevel(): vscode.Event { + return this.languageServer.onDidChangeLogLevel; + } + + append(value: string): void { + this.languageServer.append(value); + } + + appendLine(value: string): void { + if (RUNNER_TRACE_PREFIX.test(value)) { + this.runner.trace(value.replace(RUNNER_TRACE_PREFIX, '')); + } else if (RUNNER_DEBUG_PREFIX.test(value)) { + this.runner.debug(value.replace(RUNNER_DEBUG_PREFIX, '')); + } else if (RUNNER_INFO_PREFIX.test(value)) { + this.runner.info(value.replace(RUNNER_INFO_PREFIX, '')); + } else if (RUNNER_WARN_PREFIX.test(value)) { + this.runner.warn(value.replace(RUNNER_WARN_PREFIX, '')); + } else if (RUNNER_ERROR_PREFIX.test(value)) { + this.runner.error(value.replace(RUNNER_ERROR_PREFIX, '')); + } else if (RESULT_PREFIX.test(value)) { + this.results.appendLine(value.replace(RESULT_PREFIX, '')); + this.results.show(true); + } else if (TRACE_PREFIX.test(value)) { + this.languageServer.trace(value.replace(TRACE_PREFIX, '')); + } else if (DEBUG_PREFIX.test(value)) { + this.languageServer.debug(value.replace(DEBUG_PREFIX, '')); + } else if (INFO_PREFIX.test(value)) { + this.languageServer.info(value.replace(INFO_PREFIX, '')); + } else if (WARN_PREFIX.test(value)) { + this.languageServer.warn(value.replace(WARN_PREFIX, '')); + } else if (ERROR_PREFIX.test(value)) { + this.languageServer.error(value.replace(ERROR_PREFIX, '')); + } else { + this.languageServer.appendLine(value); + } + } + + clear(): void { + this.languageServer.clear(); + } + + debug(message: string, ...args: any[]): void { + this.languageServer.debug(message, ...args); + } + + dispose(): void { + this.languageServer.dispose(); + } + + error(error: string | Error, ...args: any[]): void { + this.languageServer.error(error, ...args); + } + + hide(): void { + this.languageServer.hide(); + } + + info(message: string, ...args: any[]): void { + this.languageServer.info(message, ...args); + } + + replace(value: string): void { + this.languageServer.replace(value); + } + + show(preserveFocus?: boolean): void; + show(column?: ViewColumn, preserveFocus?: boolean): void; + show(columnOrPreserveFocus?: ViewColumn | boolean, preserveFocus?: boolean): void { + if (typeof columnOrPreserveFocus === 'boolean') { + this.languageServer.show(columnOrPreserveFocus); + } else { + this.languageServer.show(columnOrPreserveFocus, preserveFocus); + } + } + + trace(message: string, ...args: any[]): void { + this.languageServer.trace(message, ...args); + } + + warn(message: string, ...args: any[]): void { + this.languageServer.warn(message, ...args); + } + + /** + * Create a logger that prepends all messages with the given tag. + */ + createTaggedLogger(tag: string) { + return { + trace: (message: string, verbose?: string) => this.trace(formatLogMessage(tag, message), verbose), + debug: (message: string) => this.debug(formatLogMessage(tag, message)), + info: (message: string) => this.info(formatLogMessage(tag, message)), + warn: (message: string) => this.warn(formatLogMessage(tag, message)), + error: (message: string) => this.error(formatLogMessage(tag, message)), + }; + } +} + +export const ttslLogger = new TTSLLogger(); + +const formatLogMessage = (tag: string, message: string): string => { + return tag ? `[${tag}] ${message}` : message; +}; diff --git a/packages/ttsl-vscode/src/extension/mainClient.ts b/packages/ttsl-vscode/src/extension/mainClient.ts index e4f7c53b..fe869d9f 100644 --- a/packages/ttsl-vscode/src/extension/mainClient.ts +++ b/packages/ttsl-vscode/src/extension/mainClient.ts @@ -5,8 +5,12 @@ import { LanguageClient, TransportKind } from 'vscode-languageclient/node.js'; import { getTTSLOutputChannel, initializeLog } from './output.js'; import { dumpDiagnostics } from './commands/dumpDiagnostics.js'; import { openDiagnosticsDumps } from './commands/openDiagnosticsDumps.js'; +import { installRunner } from './actions/installRunner.ts'; +import { updateRunner } from './actions/updateRunner.ts'; +import { TTSLServices } from '../../../ttsl-lang/src/language/ttsl-module.ts'; let client: LanguageClient; +let services: TTSLServices; // This function is called when the extension is activated. export const activate = async function (context: vscode.ExtensionContext) { @@ -72,5 +76,7 @@ const registerVSCodeCommands = function (context: vscode.ExtensionContext) { context.subscriptions.push(vscode.commands.registerCommand('ttsl.dumpDiagnostics', dumpDiagnostics(context))); context.subscriptions.push( vscode.commands.registerCommand('ttsl.openDiagnosticsDumps', openDiagnosticsDumps(context)), + vscode.commands.registerCommand('ttsl.installRunne', installRunner(client)), + vscode.commands.registerCommand('ttsl.updateRunner', updateRunner(context, client)), ); -}; +}; \ No newline at end of file