Move the stream-compression package from wp-playground to php-wasm (#895

)

Stream Compression introduced in #851 has no dependencies on WordPress
and can be used in any JavaScript project. It also makes sense as a
dependency for some `@php-wasm` packages. This commit, therefore, moves
it from the `wp-playground` to the `php-wasm` npm namespace, making it
reusable across the entire project.

In addition, this adds a new `iterateFiles` function to the
`@php-wasm/universal` package, which allows iterating over the files in
the PHP filesystem. It uses the `stream-compression` package, which was
some of the motivation for the move.

This PR also ships eslint rules to keep the `stream-compression` package
independent from the heavy `@php-wasm/web` and `@php-wasm/node`
packages. This should enable using it in other project with a minimal
dependency overhead of just `@php-wasm/util` and
`@php-wasm/node-polyfills`.

## Testing instructions

Since the package isn't used anywhere yet, only confirm if the CI checks
pass.

.eslintrc.json

@@ -15,6 +15,12 @@
 							{
 								"sourceTag": "*",
 								"notDependOnLibsWithTags": ["scope:web-client"]
+							},
+							{
+								"sourceTag": "scope:independent-from-php-binaries",
+								"notDependOnLibsWithTags": [
+									"scope:php-binaries"
+								]
 							}
 						]
 					}

packages/php-wasm/node/project.json

@@ -161,5 +161,5 @@
 			}
 		}
 	},
-	"tags": []
+	"tags": ["scope:php-binaries"]
 }

packages/php-wasm/stream-compression/README.md

@@ -0,0 +1,11 @@
+# php-wasm-stream-compression
+
+This library was generated with [Nx](https://nx.dev).
+
+## Building
+
+Run `nx build php-wasm-stream-compression` to build the library.
+
+## Running unit tests
+
+Run `nx test php-wasm-stream-compression` to execute the unit tests via [Jest](https://jestjs.io).

packages/playground/stream-compression/package.json → packages/php-wasm/stream-compression/package.json

@@ -1,5 +1,5 @@
 {
-	"name": "@wp-playground/stream-compression",
+	"name": "@php-wasm/stream-compression",
 	"version": "0.0.1",
 	"description": "Stream-based compression bindings.",
 	"repository": {
@@ -24,7 +24,7 @@
 	},
 	"publishConfig": {
 		"access": "public",
-		"directory": "../../../dist/packages/playground/stream-compression"
+		"directory": "../../../dist/packages/php-wasm/stream-compression"
 	},
 	"license": "GPL-2.0-or-later",
 	"type": "module",

packages/playground/stream-compression/project.json → packages/php-wasm/stream-compression/project.json

@@ -1,34 +1,34 @@
 {
-	"name": "playground-stream-compression",
+	"name": "php-wasm-stream-compression",
 	"$schema": "../../../node_modules/nx/schemas/project-schema.json",
-	"sourceRoot": "packages/playground/stream-compression/src",
+	"sourceRoot": "packages/php-wasm/stream-compression/src",
 	"projectType": "library",
 	"targets": {
 		"build": {
 			"executor": "@nx/vite:build",
 			"outputs": ["{options.outputPath}"],
 			"options": {
-				"outputPath": "dist/packages/playground/stream-compression"
+				"outputPath": "dist/packages/php-wasm/stream-compression"
 			}
 		},
 		"test": {
 			"executor": "@nx/vite:test",
 			"outputs": ["{options.reportsDirectory}"],
 			"options": {
 				"passWithNoTests": true,
-				"reportsDirectory": "../../../coverage/packages/playground/stream-compression"
+				"reportsDirectory": "../../../coverage/packages/php-wasm/stream-compression"
 			}
 		},
 		"lint": {
 			"executor": "@nx/linter:eslint",
 			"outputs": ["{options.outputFile}"],
 			"options": {
 				"lintFilePatterns": [
-					"packages/playground/stream-compression/**/*.ts",
-					"packages/playground/stream-compression/package.json"
+					"packages/php-wasm/stream-compression/**/*.ts",
+					"packages/php-wasm/stream-compression/package.json"
 				]
 			}
 		}
 	},
-	"tags": []
+	"tags": ["scope:independent-from-php-binaries"]
 }

packages/playground/stream-compression/src/index.ts → packages/php-wasm/stream-compression/src/index.ts

@@ -3,5 +3,5 @@ import '@php-wasm/node-polyfills';
 export { collectBytes } from './utils/collect-bytes';
 export { collectFile } from './utils/collect-file';
 export { iteratorToStream } from './utils/iterator-to-stream';
-export { streamWriteToPhp } from './utils/stream-write-to-php';
+export { StreamedFile } from './utils/streamed-file';
 export { encodeZip, decodeZip, decodeRemoteZip } from './zip';

packages/php-wasm/stream-compression/src/utils/streamed-file.ts

@@ -0,0 +1,58 @@
+import { collectBytes } from './collect-bytes';
+
+/**
+ * Represents a file that is streamed and not fully
+ * loaded into memory.
+ */
+export class StreamedFile extends File {
+	/**
+	 * Creates a new StreamedFile instance.
+	 *
+	 * @param readableStream The readable stream containing the file data.
+	 * @param name The name of the file.
+	 * @param type The MIME type of the file.
+	 */
+	constructor(
+		private readableStream: ReadableStream<Uint8Array>,
+		name: string,
+		type?: string
+	) {
+		super([], name, { type });
+	}
+
+	/**
+	 * Overrides the slice() method of the File class.
+	 *
+	 * @returns A Blob representing a portion of the file.
+	 */
+	override slice(): Blob {
+		throw new Error('slice() is not possible on a StreamedFile');
+	}
+
+	/**
+	 * Returns the readable stream associated with the file.
+	 *
+	 * @returns The readable stream.
+	 */
+	override stream() {
+		return this.readableStream;
+	}
+
+	/**
+	 * Loads the file data into memory and then returns it as a string.
+	 *
+	 * @returns File data as text.
+	 */
+	override async text() {
+		return new TextDecoder().decode(await this.arrayBuffer());
+	}
+
+	/**
+	 * Loads the file data into memory and then returns it as an ArrayBuffer.
+	 *
+	 * @returns File data as an ArrayBuffer.
+	 */
+	override async arrayBuffer() {
+		return await collectBytes(this.stream());
+	}
+}

packages/playground/stream-compression/tsconfig.lib.json → packages/php-wasm/stream-compression/tsconfig.lib.json

@@ -5,6 +5,10 @@
 		"declaration": true,
 		"types": ["node", "vite/client"]
 	},
-	"include": ["src/**/*.ts", "src/test/vitest-setup-file.ts"],
+	"include": [
+		"src/**/*.ts",
+		"src/test/vitest-setup-file.ts",
+		"src/utils/stream-write-to-php.ts"
+	],
 	"exclude": ["jest.config.ts", "src/**/*.spec.ts", "src/**/*.test.ts"]
 }

packages/playground/stream-compression/vite.config.ts → packages/php-wasm/stream-compression/vite.config.ts

@@ -6,7 +6,7 @@ import * as path from 'path';
 import { viteTsConfigPaths } from '../../vite-ts-config-paths';
 
 export default defineConfig({
-	cacheDir: '../../../node_modules/.vite/playground-stream-compression',
+	cacheDir: '../../../node_modules/.vite/php-wasm-stream-compression',
 
 	plugins: [
 		dts({
@@ -29,7 +29,7 @@ export default defineConfig({
 		lib: {
 			// Could also be a dictionary or array of multiple entry points.
 			entry: 'src/index.ts',
-			name: 'playground-stream-compression',
+			name: 'php-wasm-stream-compression',
 			fileName: 'index',
 			// Change this to the formats you want to support.
 			// Don't forget to update your package.json as well.

packages/php-wasm/universal/src/lib/index.ts

@@ -19,6 +19,10 @@ export type {
 
 export { UnhandledRejectionsTarget } from './wasm-error-reporting';
 
+export type { IteratePhpFilesOptions as IterateFilesOptions } from './iterate-files';
+export { iteratePhpFiles as iterateFiles } from './iterate-files';
+export { writeFilesStreamToPhp } from './write-files-stream-to-php';
+
 export { PHPResponse } from './php-response';
 export type { PHPResponseData } from './php-response';
 export type { ErrnoError } from './rethrow-file-system-error';

packages/php-wasm/universal/src/lib/iterate-files.ts

@@ -0,0 +1,71 @@
+import { joinPaths, normalizePath } from '@php-wasm/util';
+import { StreamedFile } from '@php-wasm/stream-compression';
+import { UniversalPHP } from './universal-php';
+import { streamReadFileFromPHP } from './stream-read-file-from-php';
+
+export type IteratePhpFilesOptions = {
+	/**
+	 * Should yield paths relative to the root directory?
+	 * If false, all paths will be absolute.
+	 */
+	relativePaths?: boolean;
+
+	/**
+	 * A prefix to add to all paths.
+	 * Only used if `relativePaths` is true.
+	 */
+	pathPrefix?: string;
+
+	/**
+	 * A list of paths to exclude from the results.
+	 */
+	exceptPaths?: string[];
+};
+
+/**
+ * Iterates over all files in a php directory and its subdirectories.
+ *
+ * @param php - The PHP instance.
+ * @param root - The root directory to start iterating from.
+ * @param options - Optional configuration.
+ * @returns All files found in the tree.
+ */
+export async function* iteratePhpFiles(
+	php: UniversalPHP,
+	root: string,
+	{
+		relativePaths = true,
+		pathPrefix,
+		exceptPaths = [],
+	}: IteratePhpFilesOptions = {}
+): AsyncGenerator<File> {
+	root = normalizePath(root);
+	const stack: string[] = [root];
+	while (stack.length) {
+		const currentParent = stack.pop();
+		if (!currentParent) {
+			return;
+		}
+		const files = await php.listFiles(currentParent);
+		for (const file of files) {
+			const absPath = `${currentParent}/${file}`;
+			if (exceptPaths.includes(absPath.substring(root.length + 1))) {
+				continue;
+			}
+			const isDir = await php.isDir(absPath);
+			if (isDir) {
+				stack.push(absPath);
+			} else {
+				yield new StreamedFile(
+					streamReadFileFromPHP(php, absPath),
+					relativePaths
+						? joinPaths(
+								pathPrefix || '',
+								absPath.substring(root.length + 1)
+						  )
+						: absPath
+				);
+			}
+		}
+	}
+}

packages/php-wasm/universal/src/lib/stream-read-file-from-php.ts

@@ -0,0 +1,14 @@
+import { UniversalPHP } from './universal-php';
+
+/**
+ * Reads a file from PHP filesystem using a stream.
+ */
+export function streamReadFileFromPHP(php: UniversalPHP, path: string) {
+	return new ReadableStream({
+		async pull(controller) {
+			const buffer = await php.readFileAsBuffer(path);
+			controller.enqueue(buffer);
+			controller.close();
+		},
+	});
+}

packages/playground/stream-compression/src/utils/stream-write-to-php.ts → packages/php-wasm/universal/src/lib/write-files-stream-to-php.ts

@@ -1,10 +1,10 @@
-import { UniversalPHP } from '@php-wasm/universal';
 import { dirname, joinPaths } from '@php-wasm/util';
+import { UniversalPHP } from './universal-php';
 
 /**
  * Writes streamed files to PHP filesystem.
  */
-export function streamWriteToPhp(php: UniversalPHP, root: string) {
+export function writeFilesStreamToPhp(php: UniversalPHP, root: string) {
 	return new WritableStream({
 		async write(file: File) {
 			const filePath = joinPaths(root, file.name);

packages/php-wasm/util/project.json

@@ -38,5 +38,5 @@
 			}
 		}
 	},
-	"tags": []
+	"tags": ["scope:independent-from-php-binaries"]
 }

packages/php-wasm/util/tsconfig.lib.json

@@ -5,6 +5,6 @@
 		"declaration": true,
 		"types": ["node"]
 	},
-	"include": ["src/**/*.ts"],
+	"include": ["src/**/*.ts", "../universal/src/lib/stream-write-to-php.ts"],
 	"exclude": ["jest.config.ts", "src/**/*.spec.ts", "src/**/*.test.ts"]
 }

packages/php-wasm/web/project.json

@@ -132,5 +132,5 @@
 			}
 		}
 	},
-	"tags": []
+	"tags": ["scope:php-binaries"]
 }