feat(cli): fetch all tiff tags with --fetch-tags (#1155)

* feat(core): use more typed arrays for bulk reading data types

* feat(cli): force fetching of tags with --fetch-tags

* refactor: remove typed as they appear slower than switch case for reading of bytes

* fix: set loaded when offsets are all loaded

* refactor: fixup test

* perf: no point slicing buffer with typed array constructor with offset

* refactor: correct typing on tiff tags

packages/cli/src/commands/info.ts

@@ -1,5 +1,5 @@
 import { fsa } from '@chunkd/fs';
-import { CogTiff, Tag, TiffTagGeo, TiffTag, TiffTagValueType, TiffVersion, toHex } from '@cogeotiff/core';
+import { CogTiff, Tag, TiffTagGeo, TiffTag, TiffTagValueType, TiffVersion, toHex, TagOffset } from '@cogeotiff/core';
 import { CogTiffImage } from '@cogeotiff/core/src/cog.tiff.image.js';
 import c from 'ansi-colors';
 import { command, flag, option, optional, restPositionals } from 'cmd-ts';
@@ -20,8 +20,8 @@ export const commandInfo = command({
   args: {
     ...DefaultArgs,
     path: option({ short: 'f', long: 'file', type: optional(Url) }),
-    tags: flag({ short: 't', long: 'tags', description: 'Dump tiff Tags' }),
-    offsets: flag({ short: 'o', long: 'offsets', description: 'Load byte offsets too' }),
+    tags: flag({ short: 't', long: 'tags', description: 'Dump tiff tags' }),
+    fetchTags: flag({ long: 'fetch-tags', description: 'Fetch extra tiff tag information' }),
     paths: restPositionals({ type: Url, description: 'Files to process' }),
   },
   async handler(args) {
@@ -86,13 +86,8 @@ export const commandInfo = command({
       if (args.tags) {
         for (const img of tiff.images) {
           const tiffTags = [...img.tags.values()];
-          // Load the first 25 values of offset arrays
-          if (args.offsets) {
-            for (const tag of tiffTags) {
-              if (tag.type !== 'offset') continue;
-              // for (let i = 0; i < Math.min(tag.count, 100); i++) tag.getValueAt(i);
-            }
-          }
+
+          if (args.fetchTags) await Promise.all(tiffTags.map((t) => img.fetch(t.id)));
 
           result.push({
             title: `Image: ${img.id} - Tiff tags`,
@@ -180,7 +175,14 @@ function formatTag(tag: Tag): { key: string; value: string } {
   const tagDebug = `(${TiffTagValueType[tag.dataType]}${tag.count > 1 ? ' x' + tag.count : ''}`;
   const key = `${c.dim(toHex(tag.id)).padEnd(7, ' ')} ${String(tagName)} ${c.dim(tagDebug)})`.padEnd(50, ' ');
 
-  if (Array.isArray(tag.value)) return { key, value: JSON.stringify(tag.value.slice(0, 250)) };
+  // Array of values that is not a string!
+  if (tag.count > 1 && tag.dataType !== TiffTagValueType.Ascii) {
+    if (tag.value == null || (tag as TagOffset).isLoaded === false) {
+      return { key, value: c.dim('Tag not Loaded, use --fetch-tags to force load') };
+    }
+    const val = [...(tag.value as number[])]; // Ensure the value is not a TypedArray
+    return { key, value: val.length > 25 ? val.slice(0, 25).join(', ') + '...' : val.join(', ') };
+  }
 
   let tagString = JSON.stringify(tag.value) ?? c.dim('null');
   if (tagString.length > 256) tagString = tagString.slice(0, 250) + '...';
@@ -191,8 +193,6 @@ function formatGeoTag(tagId: TiffTagGeo, value: string | number): { key: string;
   const tagName = TiffTagGeo[tagId];
   const key = `${c.dim(toHex(tagId)).padEnd(7, ' ')} ${String(tagName).padEnd(30)}`;
 
-  if (Array.isArray(value)) return { key, value: JSON.stringify(value.slice(0, 250)) };
-
   let tagString = JSON.stringify(value) ?? c.dim('null');
   if (tagString.length > 256) tagString = tagString.slice(0, 250) + '...';
   return { key, value: tagString };

packages/core/src/__benchmark__/cog.read.benchmark.ts

@@ -1,6 +1,7 @@
 import { readFile } from 'fs/promises';
 import { CogTiff } from '../cog.tiff.js';
 import { SourceMemory } from './source.memory.js';
+import { TiffTag } from '../index.js';
 
 // console.log = console.trace;
 /** Read a tile from every image inside of a tiff 300 tiles read */
@@ -15,6 +16,10 @@ async function main(): Promise<void> {
 
     // 6 images
     for (const img of tiff.images) await img.getTile(0, 0);
+
+    // Force loading all the byte arrays in which benchmarks the bulk array loading
+    await tiff.images[0].fetch(TiffTag.TileByteCounts);
+    await tiff.images[0].fetch(TiffTag.TileOffsets);
   }
 }
 

packages/core/src/__test__/cog.read.test.ts

@@ -4,7 +4,7 @@ import { TestFileSource } from '../__benchmark__/source.file.js';
 import { CogTiff } from '../cog.tiff.js';
 import { TiffMimeType } from '../const/tiff.mime.js';
 import { TiffVersion } from '../const/tiff.version.js';
-import { TiffTag } from '../index.js';
+import { TiffTag, TiffTagGeo } from '../index.js';
 
 function validate(tif: CogTiff): void {
   assert.equal(tif.images.length, 5);
@@ -78,9 +78,8 @@ describe('CogRead', () => {
     assert.equal(im.epsg, 2193);
     assert.equal(im.compression, TiffMimeType.None);
     assert.equal(im.isTiled(), false);
-    assert.deepEqual(
-      await im.fetch(TiffTag.StripByteCounts),
-      new Uint16Array([8064, 8064, 8064, 8064, 8064, 8064, 8064, 5040]),
-    );
+    assert.equal(im.tagsGeo.get(TiffTagGeo.GTCitationGeoKey), 'NZGD2000 / New Zealand Transverse Mercator 2000');
+    assert.equal(im.tagsGeo.get(TiffTagGeo.GeogCitationGeoKey), 'NZGD2000');
+    assert.deepEqual(await im.fetch(TiffTag.StripByteCounts), [8064, 8064, 8064, 8064, 8064, 8064, 8064, 5040]);
   });
 });

packages/core/src/cog.tiff.image.ts

@@ -6,7 +6,7 @@ import { Tag, TagInline, TagOffset } from './read/tiff.tag.js';
 import { BoundingBox, Size } from './vector.js';
 import { fetchAllOffsets, fetchLazy, getValueAt } from './read/tiff.tag.factory.js';
 
-// /** Invalid EPSG code */
+/** Invalid EPSG code */
 export const InvalidProjectionCode = 32767;
 
 /**
@@ -48,12 +48,15 @@ export interface CogTiffImageTileSize {
 export class CogTiffImage {
   /** All IFD tags that have been read for the image */
   tags: Map<TiffTag, Tag>;
-
-  /** Id of the tif image, generally the image index inside the tif */
+  /**
+   * Id of the tif image, generally the image index inside the tif
+   * where 0 is the root image, and every sub image is +1
+   *
+   * @example 0, 1, 2
+   */
   id: number;
-
+  /** Reference to the TIFF that owns this image */
   tiff: CogTiff;
-
   /** Has loadGeoTiffTags been called */
   isGeoTagsLoaded = false;
   /** Sub tags stored in TiffTag.GeoKeyDirectory */
@@ -93,9 +96,10 @@ export class CogTiffImage {
   }
 
   /**
-   * Get the value of a TiffTag if it exists null otherwise
+   * Get the value of a TiffTag if it has been loaded, null otherwise
    *
    * if the value is not loaded @see {CogTiffImage.fetch}
+   * @returns value if loaded, null otherwise
    */
   value<T>(tag: TiffTag): T | null {
     const sourceTag = this.tags.get(tag);
@@ -106,6 +110,8 @@ export class CogTiffImage {
 
   /**
    * Load and unpack the GeoKeyDirectory
+   *
+   * @see {TiffTag.GeoKeyDirectory}
    */
   async loadGeoTiffTags(): Promise<void> {
     // Already loaded
@@ -129,21 +135,22 @@ export class CogTiffImage {
     if (typeof geoTags === 'number') throw new Error('Invalid geo tags found');
     for (let i = 4; i <= geoTags[3] * 4; i += 4) {
       const key = geoTags[i] as TiffTagGeo;
-      const location = geoTags[i + 1];
+      const locationTagId = geoTags[i + 1];
 
       const offset = geoTags[i + 3];
 
-      if (location === 0) {
+      if (locationTagId === 0) {
         this.tagsGeo.set(key, offset);
         continue;
       }
-      const tag = this.tags.get(location);
+
+      const tag = this.tags.get(locationTagId);
       if (tag == null || tag.value == null) continue;
       const count = geoTags[i + 2];
-      if (Array.isArray(tag.value)) {
-        this.tagsGeo.set(key, tag.value[offset + count - 1]);
-      } else if (typeof tag.value === 'string') {
+      if (typeof tag.value === 'string') {
         this.tagsGeo.set(key, tag.value.slice(offset, offset + count - 1).trim());
+      } else {
+        throw new Error('Failed to extract GeoTiffTags');
       }
     }
   }
@@ -497,7 +504,7 @@ function getOffset(
 ): number | Promise<number> {
   if (index > x.count || index < 0) throw new Error('TagIndex: out of bounds ' + x.id + ' @ ' + index);
   if (x.type === 'inline') {
-    if (Array.isArray(x.value)) return x.value[index] as number;
+    if (x.count > 1) return (x.value as number[])[index] as number;
     return x.value as number;
   }
   return getValueAt(tiff, x, index);

packages/core/src/read/tiff.tag.factory.ts

@@ -44,14 +44,17 @@ function readTagValue(
     case TiffTagValueType.Float64:
       return bytes.getFloat64(offset, isLittleEndian);
 
+    case TiffTagValueType.Float32:
+      return bytes.getFloat32(offset, isLittleEndian);
+
     case TiffTagValueType.Uint64:
       return getUint64(bytes, offset, isLittleEndian);
     default:
       throw new Error(`Unknown read type "${fieldType}" "${TiffTagValueType[fieldType]}"`);
   }
 }
 
-function readValue<T>(tiff: CogTiff, bytes: DataView, offset: number, type: number, count: number): T {
+function readValue<T>(tiff: CogTiff, bytes: DataView, offset: number, type: TiffTagValueType, count: number): T {
   const typeSize = getTiffTagSize(type);
   const dataLength = count * typeSize;
 
@@ -61,17 +64,7 @@ function readValue<T>(tiff: CogTiff, bytes: DataView, offset: number, type: numb
     case TiffTagValueType.Ascii:
       return String.fromCharCode.apply(
         null,
-        new Uint8Array(
-          bytes.buffer.slice(bytes.byteOffset + offset, bytes.byteOffset + offset + dataLength - 1),
-        ) as unknown as number[],
-      ) as unknown as T;
-    case TiffTagValueType.Uint32:
-      return new Uint32Array(
-        bytes.buffer.slice(bytes.byteOffset + offset, bytes.byteOffset + offset + dataLength),
-      ) as unknown as T;
-    case TiffTagValueType.Uint16:
-      return new Uint16Array(
-        bytes.buffer.slice(bytes.byteOffset + offset, bytes.byteOffset + offset + dataLength),
+        new Uint8Array(bytes.buffer, offset, dataLength - 1) as unknown as number[],
       ) as unknown as T;
   }
 
@@ -118,6 +111,7 @@ export function createTag(tiff: CogTiff, view: DataViewOffset, offset: number):
         count: dataCount,
         dataType,
         dataOffset,
+        isLoaded: false,
         value: [],
         tagOffset: offset,
       };
@@ -155,6 +149,7 @@ export async function fetchAllOffsets(tiff: CogTiff, tag: TagOffset): Promise<nu
   }
 
   tag.value = readValue(tiff, tag.view, 0, tag.dataType, tag.count) as number[];
+  tag.isLoaded = true;
   return tag.value;
 }
 

packages/core/src/read/tiff.tag.ts

@@ -34,11 +34,11 @@ export interface TagInline<T> extends TagBase {
 /** Tiff tag that is a list of offsets this can be partially read */
 export interface TagOffset extends TagBase {
   type: 'offset';
-  /** Values of the offest's this is a sparse array unless @see {isLoaded} is true */
-  value: number[] | Uint32Array | Uint16Array;
+  /** Values of the offsets this is a sparse array unless @see {TagOffset.isLoaded} is true */
+  value: number[];
   /** has all the values been read */
-  isLoaded?: boolean;
-  /** Raw buffer of the values for lazy decoding, Reading 1000s of uint64s can take quite a while */
+  isLoaded: boolean;
+  /** Raw buffer of the values for lazy decoding, as reading 100,000s of uint64s can take quite a long time */
   view?: DataViewOffset;
   /** Where in the file the value is read from */
   dataOffset: number;