updates for local dev with sfs

.gitignore

@@ -7,3 +7,4 @@ dist
 logs
 gcp-private-key.json
 image_cache
+_local

Makefile

@@ -0,0 +1,6 @@
+
+include .env
+
+# simple file dev server
+sfs-dev:
+	python3 -m http.server -d ./_local/static ${SFS_PORT}

src/config.ts

@@ -4,6 +4,8 @@ import path from 'path';
 import dotenv from 'dotenv';
 dotenv.config();
 
+import { isString } from './lib/modules/type-validation/validate-primitives';
+
 const DEFAULT_PORT = 8080;
 
 let PORT: number;
@@ -16,6 +18,11 @@ const APP_ENV = process.env.APP_ENV;
 const JCD_GCP_BUCKET = 'jcd-image-1';
 const JCD_V3_GCP_BUCKET = 'jcd-image-v3';
 
+const JCD_WEB_ORIGIN = getEnvVarOrErr('JCD_WEB_ORIGIN');
+const IMG_MEM_CACHE = getBoolEnvVar('IMG_MEM_CACHE');
+
+const SFS_PORT = getNumberEnvVar('SFS_PORT');
+
 (() => {
   try {
     init();
@@ -25,6 +32,22 @@ const JCD_V3_GCP_BUCKET = 'jcd-image-v3';
   }
 })();
 
+export type EzdConfig = {
+  PORT: number;
+  APP_ROOT: string;
+  RUNTIME_ENV: string;
+  GCP_PROJECT_ID: string;
+  aws_access_key_id: string;
+  APP_ENV: string;
+  JCD_GCP_BUCKET: string;
+  JCD_V3_GCP_BUCKET: string;
+
+  JCD_WEB_ORIGIN: string;
+  IMG_MEM_CACHE: boolean;
+
+  SFS_PORT: number;
+}
+
 export const config = {
   PORT,
   APP_ROOT,
@@ -34,7 +57,11 @@ export const config = {
   APP_ENV,
   JCD_GCP_BUCKET,
   JCD_V3_GCP_BUCKET,
-} as const;
+
+  JCD_WEB_ORIGIN,
+  IMG_MEM_CACHE,
+  SFS_PORT,
+} as const satisfies EzdConfig;
 
 function init() {
   let isEnvPortStrValid: boolean, isEnvPortValid: boolean;
@@ -52,3 +79,31 @@ function init() {
 export function isDevEnv(): boolean {
   return config.APP_ENV === 'dev';
 }
+
+function getEnvVarOrErr(envKey: string): string {
+  let rawEnvVar: string | undefined;
+  rawEnvVar = process.env[envKey];
+  if(!isString(rawEnvVar)) {
+    throw new Error(`Invalid ${envKey}`);
+  }
+  return rawEnvVar;
+}
+
+function getBoolEnvVar(envKey: string): boolean {
+  let envVal = process.env[envKey];
+  if(envVal?.toLowerCase() === 'true') {
+    return true;
+  }
+  return false;
+}
+
+function getNumberEnvVar(envKey: string): number {
+  let rawPort: string;
+  let portNum: number;
+  rawPort = getEnvVarOrErr(envKey);
+  portNum = +rawPort;
+  if(isNaN(portNum)) {
+    throw new Error(`invalid env var ${envKey}, expected 'number'`);
+  }
+  return portNum;
+}

src/lib/constants.ts

@@ -0,0 +1,21 @@
+
+import path from 'path';
+
+export const BASE_DIR = path.resolve(__dirname, `..${path.sep}..`);
+
+const LOG_DIR_NAME = 'logs';
+const LOG_DIR_PATH = [
+  BASE_DIR,
+  LOG_DIR_NAME,
+].join(path.sep);
+
+const MEM_LOG_NAME = 'mem.log';
+const MEM_LOG_PATH = [
+  LOG_DIR_PATH,
+  MEM_LOG_NAME,
+].join(path.sep);
+
+export {
+  LOG_DIR_PATH,
+  MEM_LOG_PATH,
+};

src/lib/modules/mem-logger.ts

@@ -1,12 +1,25 @@
-import { MemCache } from './mem-cache/mem-cache';
 
+import fs, { WriteStream } from 'fs';
+
+import { MemCache } from './mem-cache/mem-cache';
+import { DateTimeUtil } from './util/datetime-util';
+import { MEM_LOG_PATH } from '../constants';
 const INTERVAL = 1e4;
 
+export type MemLoggerStream = {
+  write: WriteStream['write'];
+};
+
 export class MemLogger {
   memCache?: MemCache;
-  constructor(memCache?: MemCache) {
+  logStream: MemLoggerStream;
+  constructor(memCache?: MemCache, logFilePath?: string) {
     // this.logHeap();
     this.memCache = memCache;
+    this.logStream = fs.createWriteStream(MEM_LOG_PATH, {
+      flags: 'a',
+    });
+    // this.logStream = logStream ?? process.stdout;
   }
 
   run() {
@@ -24,17 +37,18 @@ export class MemLogger {
       return [ memKey, byteVal ];
     });
 
-    process.stdout.write('\n');
+    this.logStream.write('\n');
+    this.logStream.write(`[${DateTimeUtil.getDebugDateTimeStr(new Date)}]\n`);
     memVals.forEach(memTuple => {
       let memKey: string, byteVal: number, megaBytes: number;
       [ memKey, byteVal ] = memTuple;
       megaBytes = byteVal / 1024 / 1024;
-      console.log(`${memKey}: ${(+megaBytes.toFixed(3)).toLocaleString()} MB`);
+      this.logStream.write(`${memKey}: ${(+megaBytes.toFixed(3)).toLocaleString()} MB\n`);
     });
 
     if(this.memCache !== undefined) {
       cacheSize = this.memCache.cacheSize() / 1024 / 1024;
-      console.log(`Cache Size: ${cacheSize.toFixed(3)} MB`);
+      this.logStream.write(`mem_cache: ${cacheSize.toFixed(3)} MB\n`);
     }
 
     setTimeout(() => {

src/lib/modules/util/datetime-util.ts

@@ -0,0 +1,10 @@
+
+import { format } from 'date-fns';
+
+export const DateTimeUtil = {
+  getDebugDateTimeStr,
+} as const;
+
+function getDebugDateTimeStr(date: Date) {
+  return format(date, 'MM-dd-y HH:mm:ss.SSS');
+}

src/lib/services/image-service-dev.ts

@@ -1,19 +1,26 @@
 
 import { Readable } from 'stream';
 import type { ReadableStream } from 'node:stream/web';
+import http from 'http';
 
 import { GetGcpImageStreamOpts } from './gcp-storage-service';
 
 import { ImageStream } from '../../models/image-stream';
 import { isNull } from '../modules/type-validation/validate-primitives';
+import { config } from '../../config';
+
+export type ImageStreamDevOpts = GetGcpImageStreamOpts & {
+  versionFolder: 'img-v0' | 'img-v3';
+};
 
 export class ImageServiceDev {
-  static async getImageStream(opts: GetGcpImageStreamOpts): Promise<ImageStream> {
+  static async getImageStream(opts: ImageStreamDevOpts): Promise<ImageStream> {
     let imageStream: ImageStream;
     let imageReadStream: Readable;
     let headers: Record<string, string>;
     let imgPath: string;
     let imgUrl: string;
+    // let resp: nodeFetch.Response;
     let resp: Response;
     let contentType: string | null;
     // let uri = `http://127.0.0.1:4445/`;
@@ -22,7 +29,9 @@ export class ImageServiceDev {
     } else {
       imgPath = `${opts.folderKey}/${opts.imageKey}`;
     }
-    imgUrl = `http://127.0.0.1:4445/${imgPath}`;
+    imgUrl = `http://127.0.0.1:${config.SFS_PORT}/${opts.versionFolder}/${imgPath}`;
+    console.log(imgUrl);
+    // resp = await nodeFetch(imgUrl);
     resp = await fetch(imgUrl);
     if(isNull(resp.body)) {
       throw new Error('null response body when fetching stream');
@@ -31,15 +40,39 @@ export class ImageServiceDev {
       see: https://github.com/DefinitelyTyped/DefinitelyTyped/discussions/65542#discussioncomment-6071004
     */
     imageReadStream = Readable.fromWeb(resp.body as ReadableStream<Uint8Array>);
+    // imageReadStream = Readable.from(resp.body);
     contentType = resp.headers.get('content-type');
     headers = {};
     if(!isNull(contentType)) {
       headers['content-type'] = contentType;
     }
+    // if(!isNull(resp.headers.get('content-length'))) {
+    //   headers['content-type'] = resp.headers.get('content-length');
+    // }
     imageStream = {
       stream: imageReadStream,
       headers,
     };
     return imageStream;
   }
 }
+
+type FetchImageRes = {
+  headers: http.IncomingHttpHeaders;
+  stream: Readable;
+};
+
+function fetchImage(imgUrl: string): Promise<FetchImageRes> {
+  return new Promise<FetchImageRes>((resolve, reject) => {
+    let clientReq: http.ClientRequest;
+    clientReq = http.request(imgUrl, (res) => {
+      resolve({
+        stream: res,
+        headers: res.headers,
+      });
+    });
+    clientReq.once('error', (err) => {
+      reject(err);
+    });
+  });
+}

src/lib/services/image-service-v0.ts

@@ -1,6 +1,7 @@
 
 import { PassThrough, Readable } from 'stream';
 
+import { ImageServiceDev } from './image-service-dev';
 import {
   getS3ImageStream,
 } from './aws-s3-service';
@@ -35,7 +36,15 @@ export async function getImageTransformStream(opts: GetImageTransformStreamOpts)
   ) {
     imageStream = await getResizeImageStream(opts);
   } else {
-    imageStream = await getImageStream(imageKey, folderKey);
+    if(config.APP_ENV === 'dev') {
+      imageStream = await ImageServiceDev.getImageStream({
+        imageKey,
+        folderKey,
+        versionFolder: 'img-v0',
+      });
+    } else {
+      imageStream = await getImageStream(imageKey, folderKey);
+    }
   }
   return {
     stream: imageStream.stream,

src/lib/services/image-service-v1.ts

@@ -9,6 +9,8 @@ import { imageTransform } from '../modules/image-transform';
 import {
   getGcpImageStream,
 } from './gcp-storage-service';
+import { config } from '../../config';
+import { ImageServiceDev } from './image-service-dev';
 
 export async function getImageTransformStreamV1(opts: GetImageTransformStreamOpts): Promise<ImageStream> {
   let gcpImageStream: ImageStream, transformStream: Readable, imageStream: ImageStream;
@@ -18,10 +20,18 @@ export async function getImageTransformStreamV1(opts: GetImageTransformStreamOpt
     width,
     height,
   } = opts;
-  gcpImageStream = await getGcpImageStream({
-    imageKey,
-    folderKey,
-  });
+  if(config.APP_ENV === 'dev') {
+    gcpImageStream = await ImageServiceDev.getImageStream({
+      imageKey,
+      folderKey,
+      versionFolder: 'img-v3',
+    });
+  } else {
+    gcpImageStream = await getGcpImageStream({
+      imageKey,
+      folderKey,
+    });
+  }
 
   transformStream = await imageTransform({
     imageStream: gcpImageStream.stream,

src/lib/services/image-service-v2.ts

@@ -36,15 +36,15 @@ async function getImageStreamFromSource(opts: GetImageTransformStreamOpts): Prom
   } = opts;
 
   if(DO_CACHE) {
-    // gcpImageStream = await getImageStreamDev({
-    //   folderKey,
-    //   imageKey,
-    // });
-    gcpImageStream = await getGcpImageStreamCached({
+    gcpImageStream = await getImageStreamDev({
       folderKey,
       imageKey,
-      jcdBucketVersion: JCD_VERSION_ENUM.JCD_V3,
     });
+    // gcpImageStream = await getGcpImageStreamCached({
+    //   folderKey,
+    //   imageKey,
+    //   jcdBucketVersion: JCD_VERSION_ENUM.JCD_V3,
+    // });
   } else {
     gcpImageStream = await getGcpImageStream({
       imageKey,
@@ -66,7 +66,10 @@ async function getImageStreamFromSource(opts: GetImageTransformStreamOpts): Prom
 }
 
 function getImageStreamDev(opts: GetGcpImageStreamOpts): Promise<ImageStream> {
-  return ImageServiceDev.getImageStream(opts);
+  return ImageServiceDev.getImageStream({
+    ...opts,
+    versionFolder: 'img-v3',
+  });
 }
 
 async function getGcpImageStreamCached(opts: GetGcpImageStreamOpts): Promise<ImageStream> {

src/main.ts

@@ -14,5 +14,10 @@ import { initServer } from './server';
 })();
 
 async function main() {
+  setProcName();
   await initServer();
 }
+
+function setProcName() {
+  process.title = 'eaglelizard-api';
+}