feat(server): calculate sha1 checksum

server/apps/immich/src/api-v1/asset/asset.controller.ts

@@ -72,7 +72,7 @@ export class AssetController {
     @Body(ValidationPipe) assetInfo: CreateAssetDto,
   ): Promise<AssetFileUploadResponseDto> {
     try {
-      const savedAsset = await this.assetService.createUserAsset(authUser, assetInfo, file.path, file.mimetype);
+      const savedAsset = await this.assetService.createUserAsset(authUser, assetInfo, file.path, file.mimetype, file.checksum);
       if (!savedAsset) {
         throw new BadRequestException('Asset not created');
       }

server/apps/immich/src/api-v1/asset/asset.service.ts

@@ -52,6 +52,7 @@ export class AssetService {
     assetInfo: CreateAssetDto,
     path: string,
     mimeType: string,
+    checksum?: string,
   ): Promise<AssetEntity | undefined> {
     const asset = new AssetEntity();
     asset.deviceAssetId = assetInfo.deviceAssetId;
@@ -64,6 +65,7 @@ export class AssetService {
     asset.isFavorite = assetInfo.isFavorite;
     asset.mimeType = mimeType;
     asset.duration = assetInfo.duration || null;
+    asset.checksum = checksum || null;
 
     const createdAsset = await this.assetRepository.save(asset);
     if (!createdAsset) {

server/apps/immich/src/config/asset-upload.config.ts

@@ -1,11 +1,11 @@
 import { HttpException, HttpStatus } from '@nestjs/common';
 import { MulterOptions } from '@nestjs/platform-express/multer/interfaces/multer-options.interface';
 import { existsSync, mkdirSync } from 'fs';
-import { diskStorage } from 'multer';
 import { extname } from 'path';
 import { Request } from 'express';
 import { APP_UPLOAD_LOCATION } from '../constants/upload_location.constant';
 import { randomUUID } from 'crypto';
+import { diskStorage } from '../utils/disk-storage';
 
 export const assetUploadOption: MulterOptions = {
   fileFilter: (req: Request, file: any, cb: any) => {

server/apps/immich/src/global.d.ts

@@ -3,6 +3,12 @@ import { UserResponseDto } from './api-v1/user/response-dto/user-response.dto';
 declare global {
   namespace Express {
     // eslint-disable-next-line @typescript-eslint/no-empty-interface
-    interface User extends UserResponseDto {}
+    interface User extends UserResponseDto { }
+
+    namespace Multer {
+      interface File {
+        checksum?: string;
+      }
+    }
   }
 }

server/apps/immich/src/utils/disk-storage.ts

@@ -0,0 +1,98 @@
+import { Request } from 'express';
+import { ParamsDictionary } from 'express-serve-static-core';
+import { DiskStorageOptions, StorageEngine } from 'multer';
+import * as crypto from 'node:crypto';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import type { ParsedQs } from 'qs';
+import * as mkdirp from 'mkdirp';
+
+type GetFileNameFn = (req: Request, file: Express.Multer.File, cb: (error: Error | null, filename: string) => void) => void;
+type GetDestFn = (req: Request, file: Express.Multer.File, cb: (error: Error | null, destination: string) => void) => void;
+
+export interface ExtendedDiskStorageOptions extends DiskStorageOptions {
+}
+
+function getFilename(
+  req: Request,
+  file: Express.Multer.File,
+  cb: (error: Error | null, filename: string) => void
+) {
+  crypto.randomBytes(16, function (err, raw) {
+    cb(err, err ? '' : raw.toString('hex'));
+  });
+}
+
+function getDestination(
+  req: Request,
+  file: Express.Multer.File,
+  cb: (error: Error | null, destination: string) => void
+) {
+  cb(null, os.tmpdir())
+}
+
+class ExtendedDiskStorage implements StorageEngine {
+  private _getFilenameFn: GetFileNameFn;
+  private _getDestinationFn: GetDestFn;
+
+  constructor(opts: ExtendedDiskStorageOptions) {
+    this._getFilenameFn = (opts.filename || getFilename);
+
+    if (typeof opts.destination === 'string') {
+      mkdirp.sync(opts.destination);
+      this._getDestinationFn = ($0, $1, cb) => {
+        cb(null, opts.destination as string);
+      };
+    } else {
+      this._getDestinationFn = (opts.destination || getDestination);
+    }
+  }
+
+  public _handleFile(req: Request<ParamsDictionary, any, any, ParsedQs, Record<string, any>>, file: Express.Multer.File, cb: (error?: any, info?: Partial<Express.Multer.File> | undefined) => void): void {
+    const that = this;
+
+    that._getDestinationFn(req, file, (err, destination) => {
+      if (err) return cb(err);
+
+      that._getFilenameFn(req, file, (err, filename) => {
+        if (err) return cb(err);
+
+        const finalPath = path.join(destination, filename);
+        const outStream = fs.createWriteStream(finalPath);
+        const sha1Hash = crypto.createHash('sha1');
+
+        file.stream.on('data', (chunk) => {
+          sha1Hash.update(chunk);
+        });
+        file.stream.pipe(outStream);
+        outStream.on('error', cb);
+        outStream.on('finish', () => {
+          const sha1Hex = sha1Hash.digest('hex');
+
+          sha1Hash.destroy();
+          cb(null, {
+            destination: destination,
+            filename: filename,
+            path: finalPath,
+            size: outStream.bytesWritten,
+            checksum: sha1Hex,
+          });
+        });
+      })
+    })
+  }
+
+  public _removeFile(req: Request<ParamsDictionary, any, any, ParsedQs, Record<string, any>>, file: Express.Multer.File, cb: (error: Error | null) => void): void {
+    const path = file.path;
+    const refFile = file as any; // make intellisense happy
+
+    delete refFile.destination;
+    delete refFile.filename;
+    delete refFile.path;
+
+    fs.unlink(path, cb);
+  }
+}
+
+export const diskStorage = (opts: ExtendedDiskStorageOptions) => new ExtendedDiskStorage(opts);

server/libs/database/src/entities/asset.entity.ts

@@ -44,6 +44,9 @@ export class AssetEntity {
   @Column({ type: 'varchar', nullable: true })
   mimeType!: string | null;
 
+  @Column({ type: 'varchar', length: 40, nullable: true })
+  checksum?: string | null; // sha1 checksum
+
   @Column({ type: 'varchar', nullable: true })
   duration!: string | null;
 

server/libs/database/src/migrations/1661270339373-AddAssetChecksum.ts

@@ -0,0 +1,16 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddAssetChecksum1661270339373 implements MigrationInterface {
+    name = 'AddAssetChecksum1661270339373'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "assets" ADD "checksum" character varying(40)`);
+        // await queryRunner.query(`ALTER TABLE "exif" ALTER COLUMN "exifTextSearchableColumn" SET NOT NULL`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        // await queryRunner.query(`ALTER TABLE "exif" ALTER COLUMN "exifTextSearchableColumn" DROP NOT NULL`);
+        await queryRunner.query(`ALTER TABLE "assets" DROP COLUMN "checksum"`);
+    }
+
+}

server/package.json

@@ -11,7 +11,8 @@
     "format": "prettier --write \"apps/**/*.ts\" \"libs/**/*.ts\"",
     "start": "nest start",
     "start:dev": "nest start --watch",
-    "start:debug": "nest start --debug --watch",
+    "start:debug-api": "nest start immich --debug --watch",
+    "start:debug-worker": "nest start microservices --debug 9230 --watch",
     "start:prod": "node dist/main",
     "lint": "eslint \"{apps,libs}/**/*.ts\" --max-warnings 0",
     "lint:fix": "npm run lint -- --fix",
@@ -84,6 +85,7 @@
     "@types/jest": "27.0.2",
     "@types/lodash": "^4.14.178",
     "@types/mapbox__mapbox-sdk": "^0.13.4",
+    "@types/mkdirp": "^1.0.2",
     "@types/multer": "^1.4.7",
     "@types/node": "^16.0.0",
     "@types/passport-jwt": "^3.0.6",