Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: mount checks on a folder level #13801

Merged
merged 1 commit into from
Oct 29, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions server/src/entities/system-metadata.entity.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { SystemConfig } from 'src/config';
import { SystemMetadataKey } from 'src/enum';
import { StorageFolder, SystemMetadataKey } from 'src/enum';
import { Column, DeepPartial, Entity, PrimaryColumn } from 'typeorm';

@Entity('system_metadata')
Expand All @@ -12,14 +12,14 @@ export class SystemMetadataEntity<T extends keyof SystemMetadata = SystemMetadat
}

export type VersionCheckMetadata = { checkedAt: string; releaseVersion: string };
export type SystemFlags = { mountFiles: boolean };
export type SystemFlags = { mountChecks: Record<StorageFolder, boolean> };

export interface SystemMetadata extends Record<SystemMetadataKey, Record<string, any>> {
[SystemMetadataKey.ADMIN_ONBOARDING]: { isOnboarded: boolean };
[SystemMetadataKey.FACIAL_RECOGNITION_STATE]: { lastRun?: string };
[SystemMetadataKey.LICENSE]: { licenseKey: string; activationKey: string; activatedAt: Date };
[SystemMetadataKey.REVERSE_GEOCODING_STATE]: { lastUpdate?: string; lastImportFileName?: string };
[SystemMetadataKey.SYSTEM_CONFIG]: DeepPartial<SystemConfig>;
[SystemMetadataKey.SYSTEM_FLAGS]: SystemFlags;
[SystemMetadataKey.SYSTEM_FLAGS]: DeepPartial<SystemFlags>;
[SystemMetadataKey.VERSION_CHECK_STATE]: VersionCheckMetadata;
}
49 changes: 43 additions & 6 deletions server/src/services/storage.service.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,20 +30,57 @@ describe(StorageService.name, () => {

await expect(sut.onBootstrap()).resolves.toBeUndefined();

expect(systemMock.set).toHaveBeenCalledWith(SystemMetadataKey.SYSTEM_FLAGS, { mountFiles: true });
expect(systemMock.set).toHaveBeenCalledWith(SystemMetadataKey.SYSTEM_FLAGS, {
mountChecks: {
'encoded-video': true,
library: true,
profile: true,
thumbs: true,
upload: true,
},
});
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/encoded-video');
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/library');
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/profile');
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs');
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/upload');
expect(storageMock.createFile).toHaveBeenCalledWith('upload/encoded-video/.immich', expect.any(Buffer));
expect(storageMock.createFile).toHaveBeenCalledWith('upload/library/.immich', expect.any(Buffer));
expect(storageMock.createFile).toHaveBeenCalledWith('upload/profile/.immich', expect.any(Buffer));
expect(storageMock.createFile).toHaveBeenCalledWith('upload/thumbs/.immich', expect.any(Buffer));
expect(storageMock.createFile).toHaveBeenCalledWith('upload/upload/.immich', expect.any(Buffer));
});

it('should enable mount folder checking for a new folder type', async () => {
systemMock.get.mockResolvedValue({
mountChecks: {
'encoded-video': true,
library: false,
profile: true,
thumbs: true,
upload: true,
},
});

await expect(sut.onBootstrap()).resolves.toBeUndefined();

expect(systemMock.set).toHaveBeenCalledWith(SystemMetadataKey.SYSTEM_FLAGS, {
mountChecks: {
'encoded-video': true,
library: true,
profile: true,
thumbs: true,
upload: true,
},
});
expect(storageMock.mkdirSync).toHaveBeenCalledTimes(1);
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/library');
expect(storageMock.createFile).toHaveBeenCalledTimes(1);
expect(storageMock.createFile).toHaveBeenCalledWith('upload/library/.immich', expect.any(Buffer));
});

it('should throw an error if .immich is missing', async () => {
systemMock.get.mockResolvedValue({ mountFiles: true });
systemMock.get.mockResolvedValue({ mountChecks: { upload: true } });
storageMock.readFile.mockRejectedValue(new Error("ENOENT: no such file or directory, open '/app/.immich'"));

await expect(sut.onBootstrap()).rejects.toThrow('Failed to read');
Expand All @@ -53,7 +90,7 @@ describe(StorageService.name, () => {
});

it('should throw an error if .immich is present but read-only', async () => {
systemMock.get.mockResolvedValue({ mountFiles: true });
systemMock.get.mockResolvedValue({ mountChecks: { upload: true } });
storageMock.overwriteFile.mockRejectedValue(new Error("ENOENT: no such file or directory, open '/app/.immich'"));

await expect(sut.onBootstrap()).rejects.toThrow('Failed to write');
Expand All @@ -64,7 +101,7 @@ describe(StorageService.name, () => {
it('should skip mount file creation if file already exists', async () => {
const error = new Error('Error creating file') as any;
error.code = 'EEXIST';
systemMock.get.mockResolvedValue({ mountFiles: false });
systemMock.get.mockResolvedValue({ mountChecks: {} });
storageMock.createFile.mockRejectedValue(error);

await expect(sut.onBootstrap()).resolves.toBeUndefined();
Expand All @@ -73,15 +110,15 @@ describe(StorageService.name, () => {
});

it('should throw an error if mount file could not be created', async () => {
systemMock.get.mockResolvedValue({ mountFiles: false });
systemMock.get.mockResolvedValue({ mountChecks: {} });
storageMock.createFile.mockRejectedValue(new Error('Error creating file'));

await expect(sut.onBootstrap()).rejects.toBeInstanceOf(ImmichStartupError);
expect(systemMock.set).not.toHaveBeenCalled();
});

it('should startup if checks are disabled', async () => {
systemMock.get.mockResolvedValue({ mountFiles: true });
systemMock.get.mockResolvedValue({ mountChecks: { upload: true } });
configMock.getEnv.mockReturnValue(
mockEnvData({
storage: { ignoreMountCheckErrors: true },
Expand Down
24 changes: 18 additions & 6 deletions server/src/services/storage.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { Injectable } from '@nestjs/common';
import { join } from 'node:path';
import { StorageCore } from 'src/cores/storage.core';
import { OnEvent } from 'src/decorators';
import { SystemFlags } from 'src/entities/system-metadata.entity';
import { StorageFolder, SystemMetadataKey } from 'src/enum';
import { DatabaseLock } from 'src/interfaces/database.interface';
import { IDeleteFilesJob, JobStatus } from 'src/interfaces/job.interface';
Expand All @@ -19,25 +20,36 @@ export class StorageService extends BaseService {
const envData = this.configRepository.getEnv();

await this.databaseRepository.withLock(DatabaseLock.SystemFileMounts, async () => {
const flags = (await this.systemMetadataRepository.get(SystemMetadataKey.SYSTEM_FLAGS)) || { mountFiles: false };
const enabled = flags.mountFiles ?? false;
const flags =
(await this.systemMetadataRepository.get(SystemMetadataKey.SYSTEM_FLAGS)) ||
({ mountChecks: {} } as SystemFlags);

this.logger.log(`Verifying system mount folder checks (enabled=${enabled})`);
if (!flags.mountChecks) {
flags.mountChecks = {};
}

let updated = false;

this.logger.log(`Verifying system mount folder checks, current state: ${JSON.stringify(flags)}`);

try {
// check each folder exists and is writable
for (const folder of Object.values(StorageFolder)) {
if (!enabled) {
if (!flags.mountChecks[folder]) {
this.logger.log(`Writing initial mount file for the ${folder} folder`);
await this.createMountFile(folder);
}

await this.verifyReadAccess(folder);
await this.verifyWriteAccess(folder);

if (!flags.mountChecks[folder]) {
flags.mountChecks[folder] = true;
updated = true;
}
}

if (!flags.mountFiles) {
flags.mountFiles = true;
if (updated) {
await this.systemMetadataRepository.set(SystemMetadataKey.SYSTEM_FLAGS, flags);
this.logger.log('Successfully enabled system mount folders checks');
}
Expand Down
Loading