diff --git a/src/base/contentTypes/FileExtensions.ts b/src/base/contentTypes/FileExtensions.ts new file mode 100644 index 00000000..a1d5b00e --- /dev/null +++ b/src/base/contentTypes/FileExtensions.ts @@ -0,0 +1,63 @@ +import { ContentDataTypeRegistry } from "./ContentDataTypeRegistry"; +import { ContentDataTypes } from "./ContentDataTypes"; + +/** + * Methods related to file extensions for the content data types + * that are defined in `ContentDataTypes` + */ +export class FileExtensions { + /** + * A dictionary of the content data types to the default + * file extensions (without dot) + */ + private static readonly knownFileExtensionsWithoutDot = { + [ContentDataTypes.CONTENT_TYPE_GLB]: "glb", + [ContentDataTypes.CONTENT_TYPE_B3DM]: "b3dm", + [ContentDataTypes.CONTENT_TYPE_I3DM]: "i3dm", + [ContentDataTypes.CONTENT_TYPE_CMPT]: "cmpt", + [ContentDataTypes.CONTENT_TYPE_PNTS]: "pnts", + [ContentDataTypes.CONTENT_TYPE_GEOM]: "geom", + [ContentDataTypes.CONTENT_TYPE_VCTR]: "vctr", + [ContentDataTypes.CONTENT_TYPE_SUBT]: "subt", + [ContentDataTypes.CONTENT_TYPE_PNG]: "png", + [ContentDataTypes.CONTENT_TYPE_JPEG]: "jpg", + [ContentDataTypes.CONTENT_TYPE_GIF]: "gif", + [ContentDataTypes.CONTENT_TYPE_GEOJSON]: "geojson", + [ContentDataTypes.CONTENT_TYPE_3TZ]: "3tz", + [ContentDataTypes.CONTENT_TYPE_GLTF]: "gltf", + [ContentDataTypes.CONTENT_TYPE_TILESET]: "json", + }; + + /** + * Returns the default extension (without dot) for files with the given + * content data type. If the given content data type is undefined or + * not known, then an empty string will be returned. + * + * @param type - The `ContentDataType` + * @returns The file extension + */ + static getDefaultFileExtension(type: string | undefined): string { + if (type === undefined) { + return ""; + } + const result = FileExtensions.knownFileExtensionsWithoutDot[type]; + if (result === undefined) { + return ""; + } + return result; + } + + /** + * Returns the default extension (without dot) for the given file + * data, depending on its content type. If the content data type + * cannot be determined or is not known, then an empty string will + * be returned. + * + * @param data - The file data + * @returns A promise to the file extension + */ + static async determineFileExtension(data: Buffer): Promise { + const type = await ContentDataTypeRegistry.findType("", data); + return FileExtensions.getDefaultFileExtension(type); + } +} diff --git a/src/base/index.ts b/src/base/index.ts index c6a2f9d0..640ab1fc 100644 --- a/src/base/index.ts +++ b/src/base/index.ts @@ -20,6 +20,7 @@ export * from "./contentTypes/ContentDataTypeEntry"; export * from "./contentTypes/ContentDataTypeRegistry"; export * from "./contentTypes/ContentDataTypes"; export * from "./contentTypes/LazyContentData"; +export * from "./contentTypes/FileExtensions"; export * from "./io/FileResourceResolver"; export * from "./io/ResourceResolver"; diff --git a/src/cli/Analyzer.ts b/src/cli/Analyzer.ts new file mode 100644 index 00000000..cf4ba9fe --- /dev/null +++ b/src/cli/Analyzer.ts @@ -0,0 +1,638 @@ +import fs from "fs"; +import path from "path"; + +import { Paths } from "../base"; +import { ResourceResolvers } from "../base"; +import { ContentDataTypes } from "../base"; +import { ContentDataTypeRegistry } from "../base"; + +import { Subtree } from "../structure"; +import { Availability } from "../structure"; +import { TileImplicitTiling } from "../structure"; + +import { AvailabilityInfo } from "../tilesets"; +import { BinarySubtreeDataResolver } from "../tilesets"; +import { ExplicitTraversedTile } from "../tilesets"; +import { SubtreeInfo } from "../tilesets"; +import { SubtreeInfos } from "../tilesets"; +import { TileFormats } from "../tilesets"; +import { TilesetTraverser } from "../tilesets"; +import { TraversedTile } from "../tilesets"; +import { TileDataLayouts } from "../tilesets"; + +import { GltfUtilities } from "../tools"; + +import { ToolsUtilities } from "./ToolsUtilities"; + +import { Loggers } from "../base"; +const logger = Loggers.get("CLI"); + +/** + * Implementation of the functionality of the `analyze` command + * of the 3D Tiles Tools. + * + * The `analyze` command is primarily intended for debugging, and + * many details of its behavior are intentionally not specified. + * This class is only used internally, and may change arbitrarily + * in the future. + * + * @internal + */ +export class Analyzer { + /** + * Whether the contents of JSON files in the output should be + * formatted (pretty-printed). This might be controlled with + * a command line option in the future. + */ + private static readonly doFormatJson = true; + + /** + * Analyze the data in the given buffer, which is associated with + * a file that has the given base name. + * + * Note that the given `inputBaseName` is not necessarily the name + * of a file that actually exists: This function may be called, + * for example, with an input base name like + * `exampleCmpt.inner[2]` to analyze the content of an inner + * tile of a composite tile. + * + * (The input base name will only be used as a prefix for the + * output file names that are written) + * + * @param inputDirectoryName - The directory name of the input + * @param inputBaseName - The base name (file name) of the input + * @param inputBuffer - The buffer containing the input data + * @param outputDirectoryName - The name of the output directory + * @param force - Whether existing files should be overwritten + */ + static async analyze( + inputDirectoryName: string, + inputBaseName: string, + inputBuffer: Buffer, + outputDirectoryName: string, + force: boolean + ) { + const type = await ContentDataTypeRegistry.findType("", inputBuffer); + if ( + type === ContentDataTypes.CONTENT_TYPE_B3DM || + type === ContentDataTypes.CONTENT_TYPE_I3DM || + type === ContentDataTypes.CONTENT_TYPE_PNTS + ) { + Analyzer.analyzeBasicTileContent( + inputBaseName, + inputBuffer, + outputDirectoryName, + force + ); + } else if (type === ContentDataTypes.CONTENT_TYPE_CMPT) { + await Analyzer.analyzeCompositeTileContent( + inputDirectoryName, + inputBaseName, + inputBuffer, + outputDirectoryName, + force + ); + } else if (type === ContentDataTypes.CONTENT_TYPE_GLB) { + Analyzer.analyzeGlb( + inputBaseName, + inputBuffer, + outputDirectoryName, + force + ); + } else if (type === ContentDataTypes.CONTENT_TYPE_TILESET) { + await Analyzer.analyzeTileset( + inputDirectoryName, + inputBaseName, + inputBuffer, + outputDirectoryName, + force + ); + } else { + logger.warn(`Could not determine content type of ${inputBaseName}`); + } + } + + /** + * Implementation for `analyze`: Analyze B3DM, I3DM, or PNTS tile content + * + * @param inputBaseName - The base name (file name) of the input + * @param inputBuffer - The buffer containing the input data + * @param outputDirectoryName - The name of the output directory + * @param force - Whether existing files should be overwritten + */ + private static analyzeBasicTileContent( + inputBaseName: string, + inputBuffer: Buffer, + outputDirectoryName: string, + force: boolean + ) { + const tileDataLayout = TileDataLayouts.create(inputBuffer); + const tileData = TileFormats.extractTileData(inputBuffer, tileDataLayout); + + // Create the JSON strings for the layout information, + // feature table, batch table, and the GLB JSON + const layoutJsonString = Analyzer.stringify(tileDataLayout); + const featureTableJsonString = Analyzer.stringify( + tileData.featureTable.json + ); + const batchTableJsonString = Analyzer.stringify(tileData.batchTable.json); + let glbJsonString = "{}"; + if (tileData.payload.length !== 0) { + const glbJsonBuffer = GltfUtilities.extractJsonFromGlb(tileData.payload); + glbJsonString = glbJsonBuffer.toString(); + } + if (Analyzer.doFormatJson) { + const glbJson = JSON.parse(glbJsonString); + glbJsonString = Analyzer.stringify(glbJson); + } + + // Determine the output file names. They are files in the + // output directory, prefixed with the name of the input + // file, and with suffixes that indicate the actual contents + const outputBaseName = Paths.resolve(outputDirectoryName, inputBaseName); + const layoutFileName = outputBaseName + ".layout.json"; + const featureTableJsonFileName = outputBaseName + ".featureTable.json"; + const batchTableJsonFileName = outputBaseName + ".batchTable.json"; + const glbFileName = outputBaseName + ".glb"; + const glbJsonFileName = outputBaseName + ".glb.json"; + + // Write all output files + Paths.ensureDirectoryExists(outputDirectoryName); + Analyzer.writeJsonFileOptional(layoutJsonString, layoutFileName, force); + Analyzer.writeFileOptional(tileData.payload, glbFileName, force); + Analyzer.writeJsonFileOptional( + featureTableJsonString, + featureTableJsonFileName, + force + ); + Analyzer.writeJsonFileOptional( + batchTableJsonString, + batchTableJsonFileName, + force + ); + Analyzer.writeJsonFileOptional(glbJsonString, glbJsonFileName, force); + } + + /** + * Implementation for `analyze`: Analyze CMPT tile content + * + * @param inputDirectoryName - The directory name of the input + * @param inputBaseName - The base name (file name) of the input + * @param inputBuffer - The buffer containing the input data + * @param outputDirectoryName - The name of the output directory + * @param force - Whether existing files should be overwritten + */ + private static async analyzeCompositeTileContent( + inputDirectoryName: string, + inputBaseName: string, + inputBuffer: Buffer, + outputDirectoryName: string, + force: boolean + ) { + const compositeTileData = TileFormats.readCompositeTileData(inputBuffer); + const n = compositeTileData.innerTileBuffers.length; + for (let i = 0; i < n; i++) { + const innerTileDataBuffer = compositeTileData.innerTileBuffers[i]; + const innerTileBaseName = `${inputBaseName}.inner[${i}]`; + await Analyzer.analyze( + inputDirectoryName, + innerTileBaseName, + innerTileDataBuffer, + outputDirectoryName, + force + ); + } + } + + /** + * Implementation for `analyze`: Analyze GLB data + * + * @param inputBaseName - The base name (file name) of the input + * @param inputBuffer - The buffer containing the input data + * @param outputDirectoryName - The name of the output directory + * @param force - Whether existing files should be overwritten + */ + private static analyzeGlb( + inputBaseName: string, + inputBuffer: Buffer, + outputDirectoryName: string, + force: boolean + ) { + let glbJsonString = "{}"; + const glbJsonBuffer = GltfUtilities.extractJsonFromGlb(inputBuffer); + glbJsonString = glbJsonBuffer.toString(); + if (Analyzer.doFormatJson) { + const glbJson = JSON.parse(glbJsonString); + glbJsonString = Analyzer.stringify(glbJson); + } + const outputBaseName = Paths.resolve(outputDirectoryName, inputBaseName); + const glbJsonFileName = outputBaseName + ".glb.json"; + Paths.ensureDirectoryExists(outputDirectoryName); + Analyzer.writeJsonFileOptional(glbJsonString, glbJsonFileName, force); + } + + /** + * Implementation for `analyze`: Analyze a tileset (subtrees only) + * + * @param inputDirectoryName - The directory name of the input + * @param inputBaseName - The base name (file name) of the input + * @param inputBuffer - The buffer containing the input data + * @param outputDirectoryName - The name of the output directory + * @param force - Whether existing files should be overwritten + */ + private static async analyzeTileset( + inputDirectoryName: string, + inputBaseName: string, + inputBuffer: Buffer, + outputDirectoryName: string, + force: boolean + ) { + // Assmemble the information about all subtrees + // in a single markdown string + let subtreeInfosMarkdown = ""; + + // Traverse the tile hierarchy of the given tileset JSON + const resourceResolver = + ResourceResolvers.createFileResourceResolver(inputDirectoryName); + const tileset = JSON.parse(inputBuffer.toString()); + const tilesetTraverser = new TilesetTraverser( + inputDirectoryName, + resourceResolver, + { + depthFirst: false, + traverseExternalTilesets: false, + } + ); + await tilesetTraverser.traverse( + tileset, + async (traversedTile: TraversedTile) => { + // Check if the traversed tile is the root of + // a subtree within an implicit tile hierarchy + if (traversedTile instanceof ExplicitTraversedTile) { + return true; + } + const subtreeUri = traversedTile.getSubtreeUri(); + if (subtreeUri === undefined) { + return true; + } + const implicitTiling = Analyzer.findImplicitTiling(traversedTile); + if (implicitTiling === undefined) { + // Should never happen + logger.error( + `Could not obtain implicit tiling for traversed tile: ${traversedTile}` + ); + return false; + } + + // Create the markdown string for the single subtree file + const fullSubtreeFileName = Paths.resolve( + inputDirectoryName, + subtreeUri + ); + const subtreeDirectoryName = path.dirname(fullSubtreeFileName); + const subtreeData = fs.readFileSync(fullSubtreeFileName); + const subtreeInfoMarkdown = await Analyzer.createSubtreeInfoMarkdown( + subtreeDirectoryName, + subtreeData, + implicitTiling + ); + + // Append the markdown for the single subtree file + // to the global one, with a header containing the + // current subtree URI + subtreeInfosMarkdown += "## Subtree " + subtreeUri + ":"; + subtreeInfosMarkdown += "\n"; + subtreeInfosMarkdown += subtreeInfoMarkdown; + return true; + } + ); + + // Write the resuling subtree information as a markdown file + const outputBaseName = Paths.resolve(outputDirectoryName, inputBaseName); + const subtreeInfosMarkdownFileName = outputBaseName + ".subtrees.md"; + Analyzer.writeFileOptional( + Buffer.from(subtreeInfosMarkdown), + subtreeInfosMarkdownFileName, + force + ); + } + + /** + * Implementation for `analyze`: Analyze subtree data + * + * @param inputDirectoryName - The directory name of the input + * @param inputBuffer - The buffer containing the input data + * @param implicitTiling - The `TileImplicitTiling` that describes + * the structure of the subtree data in the input buffer + */ + private static async createSubtreeInfoMarkdown( + inputDirectoryName: string, + inputBuffer: Buffer, + implicitTiling: TileImplicitTiling + ) { + // Obtain the "raw" data for the given subtree data, including + // the parsed 'Subtree' object, and create a 'SubtreeInfo' + const resourceResolver = + ResourceResolvers.createFileResourceResolver(inputDirectoryName); + const binarySubtreeData = await BinarySubtreeDataResolver.resolveFromBuffer( + inputBuffer, + resourceResolver + ); + const subtree = binarySubtreeData.subtree; + const subtreeInfo = SubtreeInfos.create(binarySubtreeData, implicitTiling); + + // Create the JSON string for the 'Subtree', and the markdown + // string that contains availability information + const subtreeJsonString = JSON.stringify(subtree, null, 2); + const availabilityMarkdownString = Analyzer.createAvailabilityInfosMarkdown( + subtree, + subtreeInfo + ); + + // Assemble the resulting string: The subtree JSON + // as a code block, and the availability info string: + let s = ""; + s += "Subtree JSON:"; + s += "\n"; + s += "```"; + s += subtreeJsonString; + s += "\n"; + s += "```"; + s += "\n"; + s += availabilityMarkdownString; + s += "\n"; + return s; + } + + /** + * Returns a markdown string containing information about the + * tile-, content-, and child subtree availability that is + * stored in the given subtree data. + * + * Details about the structure of the returned string are + * totally unspecified. + * + * @param subtree - The `Subtree` + * @param subtreeInfo - The `SubtreeInfo` + * @returns The markdown string + */ + private static createAvailabilityInfosMarkdown( + subtree: Subtree, + subtreeInfo: SubtreeInfo + ) { + const tileAvailability = subtree.tileAvailability; + const contentAvailability = subtree.contentAvailability; + const childSubtreeAvailability = subtree.childSubtreeAvailability; + + const tileAvailabilityInfo = subtreeInfo.tileAvailabilityInfo; + const contentAvailabilityInfos = subtreeInfo.contentAvailabilityInfos; + const childSubtreeAvailabilityInfo = + subtreeInfo.childSubtreeAvailabilityInfo; + + let s = ""; + s += "#### Tile Availability:"; + s += "\n"; + s += Analyzer.createAvailabilityInfoMarkdown( + tileAvailability, + tileAvailabilityInfo + ); + s += "\n"; + + if (contentAvailability != undefined) { + const n = contentAvailability.length; + for (let i = 0; i < n; i++) { + s += "#### Content Availability (" + i + " of " + n + "):"; + s += "\n"; + s += Analyzer.createAvailabilityInfoMarkdown( + contentAvailability[i], + contentAvailabilityInfos[i] + ); + s += "\n"; + } + } + + s += "#### Child Subtree Availability:"; + s += "\n"; + s += Analyzer.createAvailabilityInfoMarkdown( + childSubtreeAvailability, + childSubtreeAvailabilityInfo + ); + s += "\n"; + + return s; + } + + /** + * Returns a markdown string containing information about the + * given availabiltiy data. + * + * Details about the structure of the returned string are + * totally unspecified. + * + * @param availability - The `Availability` + * @param availabilityInfo - The `AvailabilityInfo` + * @returns The markdown string + */ + private static createAvailabilityInfoMarkdown( + availability: Availability, + availabilityInfo: AvailabilityInfo + ) { + if (availability.constant !== undefined) { + return "Constant: " + availability.constant + "\n"; + } + return Analyzer.createAvailabilityInfoMarkdownTable(availabilityInfo); + } + + /** + * Returns a markdown string containing information about the + * given availabiltiy info. + * + * Details about the structure of the returned string are + * totally unspecified. + * + * @param availabilityInfo - The `AvailabilityInfo` + * @returns The markdown string + */ + private static createAvailabilityInfoMarkdownTable( + availabilityInfo: AvailabilityInfo + ) { + // Create a markdown table with the following structure: + // + //| Byte index: | 0|1|2| + //| --- | --- | --- | --- | + //| Bytes: | 0x0d | 0x32 | 0x01 | + //| Bits [0...7] : | 10110000|01001100|10000000| + // + const length = availabilityInfo.length; + const numBytes = Math.ceil(length / 8); + + let s = ""; + + // Header + s += "| Byte index: |"; + for (let i = 0; i < numBytes; i++) { + if (i > 0) { + s += "|"; + } + s += i.toString(); + } + s += "|"; + s += "\n"; + + // Separator + s += "| --- |"; + for (let i = 0; i < numBytes; i++) { + if (i > 0) { + s += "|"; + } + s += " --- "; + } + s += "|"; + s += "\n"; + + // Bytes + s += "| Byte: |"; + for (let i = 0; i < numBytes; i++) { + if (i > 0) { + s += "|"; + } + let byte = 0; + for (let j = 0; j < 8; j++) { + const index = i * 8 + j; + if (index < length) { + const a = availabilityInfo.isAvailable(index); + if (a) { + byte |= 1 << j; + } + } + } + const bs = byte.toString(16); + s += "0x"; + if (bs.length < 2) { + s += "0"; + } + s += bs; + } + s += "|"; + s += "\n"; + + // Bits [0...7] + s += "| Bits [0...7]: |"; + for (let i = 0; i < numBytes; i++) { + if (i > 0) { + s += "|"; + } + for (let j = 0; j < 8; j++) { + const index = i * 8 + j; + if (index < length) { + const a = availabilityInfo.isAvailable(index); + if (a) { + s += "1"; + } else { + s += "0"; + } + } else { + s += "0"; + } + } + } + s += "|"; + s += "\n"; + + return s; + } + + /** + * Returns the first `TileImplicitTiling` that is found for the + * given traversed tile or any of its ancestors, or `undefined` + * if no implicit tiling information can be found. + * + * @param traversedTile - The `TraversedTile` instance + * @returns The `TileImplicitTiling`, or `undefined` + */ + private static findImplicitTiling( + traversedTile: TraversedTile + ): TileImplicitTiling | undefined { + if (traversedTile instanceof ExplicitTraversedTile) { + const explicitTraversedTile = traversedTile as ExplicitTraversedTile; + const implicitTiling = explicitTraversedTile.getImplicitTiling(); + if (implicitTiling !== undefined) { + return implicitTiling; + } + } + const parent = traversedTile.getParent(); + if (parent === undefined) { + return undefined; + } + return Analyzer.findImplicitTiling(parent); + } + + /** + * Returns a JSON string representation of the given object, + * possibly formatted/indented. + * + * (This may be controlled with a command line flag in the future). + * + * @param input - The input object + * @returns The stringified object + */ + private static stringify(input: any): string { + if (Analyzer.doFormatJson) { + return JSON.stringify(input, null, 2); + } + return JSON.stringify(input); + } + + /** + * Writes the given JSON string to the specifified file. + * + * If the given JSON string represents the empty object `"{}"`, + * then nothing will be written. + * + * If the file exists and `force===false`, then an error message + * will be printed. + * + * @param jsonString - The JSON string + * @param fileName - The file name + * @param force - Whether files should be overwritten + */ + private static writeJsonFileOptional( + jsonString: string, + fileName: string, + force: boolean + ) { + if (jsonString === "{}") { + return; + } + if (!ToolsUtilities.canWrite(fileName, force)) { + logger.error(`Cannot write ${fileName}`); + return; + } + logger.info(`Writing ${fileName}`); + fs.writeFileSync(fileName, Buffer.from(jsonString)); + } + + /** + * Writes the given data to the specifified file. + * + * If the buffer is empty, then nothing will be written. + * + * If the file exists and `force===false`, then an error message + * will be printed. + * + * @param buffer - The file data + * @param fileName - The file name + * @param force - Whether files should be overwritten + */ + private static writeFileOptional(buffer: Buffer, fileName: string, force) { + if (buffer.length === 0) { + return; + } + if (!ToolsUtilities.canWrite(fileName, force)) { + logger.error(`Cannot write ${fileName}`); + return; + } + logger.info(`Writing ${fileName}`); + fs.writeFileSync(fileName, buffer); + } +} diff --git a/src/cli/ToolsMain.ts b/src/cli/ToolsMain.ts index bc51c3cc..56e2a6c6 100644 --- a/src/cli/ToolsMain.ts +++ b/src/cli/ToolsMain.ts @@ -3,16 +3,13 @@ import path from "path"; import { Paths } from "../base"; import { DeveloperError } from "../base"; -import { Buffers } from "../base"; import { Iterables } from "../base"; import { ContentDataTypes } from "../base"; import { TileFormats } from "../tilesets"; -import { TileDataLayouts } from "../tilesets"; import { TileFormatError } from "../tilesets"; import { ContentOps } from "../tools"; -import { GltfUtilities } from "../tools"; import { PipelineExecutor } from "../tools"; import { Pipelines } from "../tools"; @@ -22,7 +19,10 @@ import { TileFormatsMigration } from "../tools"; import { TilesetConverter } from "../tools"; import { TilesetJsonCreator } from "../tools"; -import { ContentDataTypeRegistry } from "../base"; +import { FileExtensions } from "../base"; + +import { ToolsUtilities } from "./ToolsUtilities"; +import { Analyzer } from "./Analyzer"; import { Loggers } from "../base"; const logger = Loggers.get("CLI"); @@ -48,7 +48,7 @@ export class ToolsMain { logger.debug(` output: ${output}`); logger.debug(` force: ${force}`); - ToolsMain.ensureCanWrite(output, force); + ToolsUtilities.ensureCanWrite(output, force); const inputBuffer = fs.readFileSync(input); const inputTileData = TileFormats.readTileData(inputBuffer); const outputBuffer = TileFormats.extractGlbPayload(inputTileData); @@ -62,7 +62,7 @@ export class ToolsMain { logger.debug(` output: ${output}`); logger.debug(` force: ${force}`); - ToolsMain.ensureCanWrite(output, force); + ToolsUtilities.ensureCanWrite(output, force); const inputBuffer = fs.readFileSync(input); const outputBuffer = await TileFormatsMigration.convertB3dmToGlb( inputBuffer @@ -77,7 +77,7 @@ export class ToolsMain { logger.debug(` output: ${output}`); logger.debug(` force: ${force}`); - ToolsMain.ensureCanWrite(output, force); + ToolsUtilities.ensureCanWrite(output, force); const inputBuffer = fs.readFileSync(input); const outputBuffer = await TileFormatsMigration.convertPntsToGlb( inputBuffer @@ -93,11 +93,11 @@ export class ToolsMain { logger.debug(` output: ${output}`); logger.debug(` force: ${force}`); - ToolsMain.ensureCanWrite(output, force); + ToolsUtilities.ensureCanWrite(output, force); const inputBuffer = fs.readFileSync(input); // Prepare the resolver for external GLBs in I3DM - const externalGlbResolver = ToolsMain.createResolver(input); + const externalGlbResolver = ToolsUtilities.createResolver(input); const outputBuffer = await TileFormatsMigration.convertI3dmToGlb( inputBuffer, externalGlbResolver @@ -113,11 +113,11 @@ export class ToolsMain { logger.debug(` output: ${output}`); logger.debug(` force: ${force}`); - ToolsMain.ensureCanWrite(output, force); + ToolsUtilities.ensureCanWrite(output, force); const inputBuffer = fs.readFileSync(input); const inputTileData = TileFormats.readTileData(inputBuffer); // Prepare the resolver for external GLBs in I3DM - const externalGlbResolver = ToolsMain.createResolver(input); + const externalGlbResolver = ToolsUtilities.createResolver(input); const outputBuffer = await TileFormats.obtainGlbPayload( inputTileData, externalGlbResolver @@ -138,7 +138,7 @@ export class ToolsMain { logger.debug(` force: ${force}`); const inputBuffer = fs.readFileSync(input); - const externalGlbResolver = ToolsMain.createResolver(input); + const externalGlbResolver = ToolsUtilities.createResolver(input); const glbBuffers = await TileFormats.extractGlbBuffers( inputBuffer, externalGlbResolver @@ -157,7 +157,7 @@ export class ToolsMain { } for (let i = 0; i < glbsLength; i++) { const glbPath = glbPaths[i]; - ToolsMain.ensureCanWrite(glbPath, force); + ToolsUtilities.ensureCanWrite(glbPath, force); const glbBuffer = glbBuffers[i]; fs.writeFileSync(glbPath, glbBuffer); } @@ -182,38 +182,27 @@ export class ToolsMain { for (let i = 0; i < outputBuffers.length; i++) { const outputBuffer = outputBuffers[i]; const prefix = Paths.replaceExtension(output, ""); - const extension = await ToolsMain.determineFileExtension(outputBuffer); + const extension = await FileExtensions.determineFileExtension( + outputBuffer + ); + if (extension === "") { + logger.warn("Could not determine type of inner tile"); + } const outputPath = `${prefix}_${i}.${extension}`; - ToolsMain.ensureCanWrite(outputPath, force); + ToolsUtilities.ensureCanWrite(outputPath, force); fs.writeFileSync(outputPath, outputBuffer); } logger.debug(`Executing splitCmpt DONE`); } - private static async determineFileExtension(data: Buffer): Promise { - const type = await ContentDataTypeRegistry.findType("", data); - switch (type) { - case ContentDataTypes.CONTENT_TYPE_B3DM: - return "b3dm"; - case ContentDataTypes.CONTENT_TYPE_I3DM: - return "i3dm"; - case ContentDataTypes.CONTENT_TYPE_PNTS: - return "pnts"; - case ContentDataTypes.CONTENT_TYPE_CMPT: - return "cmpt"; - } - logger.warn("Could not determine type of inner tile"); - return "UNKNOWN"; - } - static async glbToB3dm(input: string, output: string, force: boolean) { logger.debug(`Executing glbToB3dm`); logger.debug(` input: ${input}`); logger.debug(` output: ${output}`); logger.debug(` force: ${force}`); - ToolsMain.ensureCanWrite(output, force); + ToolsUtilities.ensureCanWrite(output, force); const inputBuffer = fs.readFileSync(input); const outputTileData = TileFormats.createDefaultB3dmTileDataFromGlb(inputBuffer); @@ -228,7 +217,7 @@ export class ToolsMain { logger.debug(` output: ${output}`); logger.debug(` force: ${force}`); - ToolsMain.ensureCanWrite(output, force); + ToolsUtilities.ensureCanWrite(output, force); const inputBuffer = fs.readFileSync(input); const outputTileData = TileFormats.createDefaultI3dmTileDataFromGlb(inputBuffer); @@ -249,7 +238,7 @@ export class ToolsMain { logger.debug(` force: ${force}`); logger.debug(` options: ${JSON.stringify(options)}`); - ToolsMain.ensureCanWrite(output, force); + ToolsUtilities.ensureCanWrite(output, force); const inputBuffer = fs.readFileSync(input); const outputBuffer = await ContentOps.optimizeB3dmBuffer( inputBuffer, @@ -271,7 +260,7 @@ export class ToolsMain { logger.debug(` force: ${force}`); logger.debug(` options: ${JSON.stringify(options)}`); - ToolsMain.ensureCanWrite(output, force); + ToolsUtilities.ensureCanWrite(output, force); const inputBuffer = fs.readFileSync(input); const outputBuffer = await ContentOps.optimizeI3dmBuffer( inputBuffer, @@ -282,7 +271,7 @@ export class ToolsMain { logger.debug(`Executing optimizeI3dm DONE`); } - static analyze( + static async analyze( inputFileName: string, outputDirectoryName: string, force: boolean @@ -290,9 +279,11 @@ export class ToolsMain { logger.info(`Analyzing ${inputFileName}`); logger.info(`writing results to ${outputDirectoryName}`); + const inputDirectoryName = path.dirname(inputFileName); const inputBaseName = path.basename(inputFileName); const inputBuffer = fs.readFileSync(inputFileName); - ToolsMain.analyzeInternal( + await Analyzer.analyze( + inputDirectoryName, inputBaseName, inputBuffer, outputDirectoryName, @@ -300,124 +291,6 @@ export class ToolsMain { ); logger.info(`Analyzing ${inputFileName} DONE`); } - static analyzeInternal( - inputBaseName: string, - inputBuffer: Buffer, - outputDirectoryName: string, - force: boolean - ) { - // A function to create a JSON string from an - // object. The formatting will be controlled - // via a command line flag in the future. - const doFormatJson = true; - const stringify = (input: any) => { - if (doFormatJson) { - return JSON.stringify(input, null, 2); - } - return JSON.stringify(input); - }; - - // A function to write a JSON string to a file, if - // the JSON string does not represent an empty - // object, and if the file can be written. - const writeJsonFileOptional = (jsonString: string, fileName: string) => { - if (jsonString === "{}") { - return; - } - if (!ToolsMain.canWrite(fileName, force)) { - logger.error(`Cannot write ${fileName}`); - return; - } - logger.info(`Writing ${fileName}`); - fs.writeFileSync(fileName, Buffer.from(jsonString)); - }; - - // A function to write a buffer to a file, if - // the buffer is not empty, and if the file - // can be written. - const writeFileOptional = (buffer: Buffer, fileName: string) => { - if (buffer.length === 0) { - return; - } - if (!ToolsMain.canWrite(fileName, force)) { - logger.error(`Cannot write ${fileName}`); - return; - } - logger.info(`Writing ${fileName}`); - fs.writeFileSync(fileName, buffer); - }; - - // Read the buffer and its magic header - const magic = Buffers.getMagicString(inputBuffer, 0); - - if (magic === "b3dm" || magic === "i3dm" || magic === "pnts") { - // Handle the basic legacy tile formats - const tileDataLayout = TileDataLayouts.create(inputBuffer); - const tileData = TileFormats.extractTileData(inputBuffer, tileDataLayout); - - // Create the JSON strings for the layout information, - // feature table, batch table, and the GLB JSON - const layoutJsonString = stringify(tileDataLayout); - const featureTableJsonString = stringify(tileData.featureTable.json); - const batchTableJsonString = stringify(tileData.batchTable.json); - let glbJsonString = "{}"; - if (tileData.payload.length !== 0) { - const glbJsonBuffer = GltfUtilities.extractJsonFromGlb( - tileData.payload - ); - glbJsonString = glbJsonBuffer.toString(); - } - if (doFormatJson) { - const glbJson = JSON.parse(glbJsonString); - glbJsonString = stringify(glbJson); - } - - // Determine the output file names. They are files in the - // output directory, prefixed with the name of the input - // file, and with suffixes that indicate the actual contents - const outputBaseName = Paths.resolve(outputDirectoryName, inputBaseName); - const layoutFileName = outputBaseName + ".layout.json"; - const featureTableJsonFileName = outputBaseName + ".featureTable.json"; - const batchTableJsonFileName = outputBaseName + ".batchTable.json"; - const glbFileName = outputBaseName + ".glb"; - const glbJsonFileName = outputBaseName + ".glb.json"; - - // Write all output files - Paths.ensureDirectoryExists(outputDirectoryName); - writeJsonFileOptional(layoutJsonString, layoutFileName); - writeFileOptional(tileData.payload, glbFileName); - writeJsonFileOptional(featureTableJsonString, featureTableJsonFileName); - writeJsonFileOptional(batchTableJsonString, batchTableJsonFileName); - writeJsonFileOptional(glbJsonString, glbJsonFileName); - } else if (magic === "cmpt") { - // Handle composite tiles - const compositeTileData = TileFormats.readCompositeTileData(inputBuffer); - const n = compositeTileData.innerTileBuffers.length; - for (let i = 0; i < n; i++) { - const innerTileDataBuffer = compositeTileData.innerTileBuffers[i]; - const innerTileBaseName = `${inputBaseName}.inner[${i}]`; - ToolsMain.analyzeInternal( - innerTileBaseName, - innerTileDataBuffer, - outputDirectoryName, - force - ); - } - } else if (magic === "glTF") { - // Handle GLB files - let glbJsonString = "{}"; - const glbJsonBuffer = GltfUtilities.extractJsonFromGlb(inputBuffer); - glbJsonString = glbJsonBuffer.toString(); - if (doFormatJson) { - const glbJson = JSON.parse(glbJsonString); - glbJsonString = stringify(glbJson); - } - const outputBaseName = Paths.resolve(outputDirectoryName, inputBaseName); - const glbJsonFileName = outputBaseName + ".glb.json"; - Paths.ensureDirectoryExists(outputDirectoryName); - writeJsonFileOptional(glbJsonString, glbJsonFileName); - } - } private static createGzipPipelineJson( input: string, @@ -457,7 +330,7 @@ export class ToolsMain { force: boolean, tilesOnly: boolean ) { - ToolsMain.ensureCanWrite(output, force); + ToolsUtilities.ensureCanWrite(output, force); const pipelineJson = ToolsMain.createGzipPipelineJson( input, @@ -486,7 +359,7 @@ export class ToolsMain { } static async ungzip(input: string, output: string, force: boolean) { - ToolsMain.ensureCanWrite(output, force); + ToolsUtilities.ensureCanWrite(output, force); const pipelineJson = ToolsMain.createUngzipPipelineJson(input, output); const pipeline = Pipelines.createPipeline(pipelineJson); @@ -499,7 +372,7 @@ export class ToolsMain { output: string, force: boolean ) { - ToolsMain.ensureCanWrite(output, force); + ToolsUtilities.ensureCanWrite(output, force); await TilesetConverter.convert( input, inputTilesetJsonFileName, @@ -509,7 +382,7 @@ export class ToolsMain { } static async combine(input: string, output: string, force: boolean) { - ToolsMain.ensureCanWrite(output, force); + ToolsUtilities.ensureCanWrite(output, force); await TilesetOperations.combine(input, output, force); } @@ -527,7 +400,7 @@ export class ToolsMain { logger.debug(` targetVersion: ${targetVersion}`); logger.debug(` gltfUpgradeOptions: ${JSON.stringify(gltfUpgradeOptions)}`); - ToolsMain.ensureCanWrite(output, force); + ToolsUtilities.ensureCanWrite(output, force); await TilesetOperations.upgrade( input, output, @@ -545,7 +418,7 @@ export class ToolsMain { logger.debug(` output: ${output}`); logger.debug(` force: ${force}`); - ToolsMain.ensureCanWrite(output, force); + ToolsUtilities.ensureCanWrite(output, force); await TilesetOperations.merge(inputs, output, force); logger.debug(`Executing merge DONE`); @@ -575,7 +448,7 @@ export class ToolsMain { logger.debug(` output: ${output}`); logger.debug(` force: ${force}`); - ToolsMain.ensureCanWrite(output, force); + ToolsUtilities.ensureCanWrite(output, force); let baseDir = inputName; let contentUris: string[] = []; if (!Paths.isDirectory(inputName)) { @@ -614,70 +487,4 @@ export class ToolsMain { logger.debug(`Executing createTilesetJson DONE`); } - - /** - * Creates a function that can resolve URIs relative to - * the given input file. - * - * The function will resolve relative URIs against the - * base directory of the given input file name, and - * return the corresponding file data. If the data - * cannot be read, then the function will print an - * error message and return `undefined`. - * - * @param input - The input file name - * @returns The resolver function - */ - private static createResolver( - input: string - ): (uri: string) => Promise { - const baseDir = path.dirname(input); - const resolver = async (uri: string): Promise => { - const externalGlbUri = path.resolve(baseDir, uri); - try { - return fs.readFileSync(externalGlbUri); - } catch (error) { - logger.error(`Could not resolve ${uri} against ${baseDir}`); - } - }; - return resolver; - } - - /** - * Returns whether the specified file can be written. - * - * This is the case when `force` is `true`, or when it does not - * exist yet. - * - * @param fileName - The file name - * @param force The 'force' flag state from the command line - * @returns Whether the file can be written - */ - static canWrite(fileName: string, force: boolean): boolean { - if (force) { - return true; - } - if (!fs.existsSync(fileName)) { - return true; - } - return false; - } - - /** - * Ensures that the specified file can be written, and throws - * a `DeveloperError` otherwise. - * - * @param fileName - The file name - * @param force The 'force' flag state from the command line - * @returns Whether the file can be written - * @throws DeveloperError When the file exists and `force` was `false`. - */ - static ensureCanWrite(fileName: string, force: boolean): true { - if (ToolsMain.canWrite(fileName, force)) { - return true; - } - throw new DeveloperError( - `File ${fileName} already exists. Specify -f or --force to overwrite existing files.` - ); - } } diff --git a/src/cli/ToolsUtilities.ts b/src/cli/ToolsUtilities.ts new file mode 100644 index 00000000..8304d1f8 --- /dev/null +++ b/src/cli/ToolsUtilities.ts @@ -0,0 +1,81 @@ +import fs from "fs"; +import path from "path"; + +import { DeveloperError } from "../base"; + +import { Loggers } from "../base"; +const logger = Loggers.get("CLI"); + +/** + * Utilities for the command line interface functionality of + * the tools that are implemented in ToolsMain. + * + * @internal + */ +export class ToolsUtilities { + /** + * Returns whether the specified file can be written. + * + * This is the case when `force` is `true`, or when it does not + * exist yet. + * + * @param fileName - The file name + * @param force The 'force' flag state from the command line + * @returns Whether the file can be written + */ + static canWrite(fileName: string, force: boolean): boolean { + if (force) { + return true; + } + if (!fs.existsSync(fileName)) { + return true; + } + return false; + } + + /** + * Ensures that the specified file can be written, and throws + * a `DeveloperError` otherwise. + * + * @param fileName - The file name + * @param force The 'force' flag state from the command line + * @returns Whether the file can be written + * @throws DeveloperError When the file exists and `force` was `false`. + */ + static ensureCanWrite(fileName: string, force: boolean): true { + if (ToolsUtilities.canWrite(fileName, force)) { + return true; + } + throw new DeveloperError( + `File ${fileName} already exists. Specify -f or --force to overwrite existing files.` + ); + } + + /** + * Creates a function that can resolve URIs relative to + * the given input file. + * + * The function will resolve relative URIs against the + * base directory of the given input file name, and + * return the corresponding file data. If the data + * cannot be read, then the function will print an + * error message and return `undefined`. + * + * @param input - The input file name + * @returns The resolver function + */ + static createResolver( + input: string + ): (uri: string) => Promise { + const baseDir = path.dirname(input); + const resolver = async (uri: string): Promise => { + const externalGlbUri = path.resolve(baseDir, uri); + try { + return fs.readFileSync(externalGlbUri); + } catch (error) { + logger.error(`Could not resolve ${uri} against ${baseDir}`); + } + }; + return resolver; + } +} diff --git a/src/cli/main.ts b/src/cli/main.ts index a5137ac5..033d2d8c 100644 --- a/src/cli/main.ts +++ b/src/cli/main.ts @@ -11,7 +11,7 @@ let logger = Loggers.get("CLI"); // the `--options` will be passed to downstream // calls (e.g. calls to `gltf-pipeline`) const optionsIndex = process.argv.indexOf("--options"); -let toolArgs; +let toolArgs: string[]; let optionArgs: string[]; if (optionsIndex < 0) { toolArgs = process.argv.slice(2); @@ -526,7 +526,7 @@ async function runCommand(command: string, toolArgs: any, optionArgs: any) { } else if (command === "pipeline") { await ToolsMain.pipeline(input, force); } else if (command === "analyze") { - ToolsMain.analyze(input, output, force); + await ToolsMain.analyze(input, output, force); } else if (command === "createTilesetJson") { const cartographicPositionDegrees = validateOptionalNumberArray( toolArgs.cartographicPositionDegrees,