Skip to content

Commit

Permalink
Integrate new ocr service
Browse files Browse the repository at this point in the history
Add ocr service

Add custom logger

Add ocr development service

Handle ocr error response

Use SKIP_PROCESSING on ocr service in development
  • Loading branch information
daniel-va committed Oct 16, 2024
1 parent b60efd5 commit 71dce22
Show file tree
Hide file tree
Showing 15 changed files with 399 additions and 194 deletions.
1 change: 1 addition & 0 deletions .github/workflows/code-quality.yml
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,7 @@ jobs:
DB_PASSWORD: ${{ env.DB_PASSWORD }}
run: |
cd development
touch .env.ocr
dos2unix ./init/elasticsearch/init.sh
chmod +x ./init/elasticsearch/init.sh
chmod +r ./init/elasticsearch/mappings/swissgeol_asset_asset.json
Expand Down
10 changes: 7 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ Be aware that you need to manually insert the `{DB_*}` values beforehand.

```bash
cd development
docker compose exec db sh -c 'pg_dump --dbname=postgresql://{DB_USERNAME}:{DB_PASSWORD}@{DB_HOST}:5432/{DB_DATABASE} --data-only --exclude-table asset_user --exclude-table _prisma_migrations -n public > /dump.sql'
docker compose exec db sh -c 'pg_dump --dbname=postgresql://{DB_USERNAME}:{DB_PASSWORD}@{DB_HOST}:5432/{DB_DATABASE} --data-only --exclude-table asset_user --exclude-table workgroups_on_users --exclude-table _prisma_migrations -n public > /dump.sql'
```

> The export will output warnings related to circular foreign-key constraints.
Expand All @@ -105,10 +105,14 @@ Ensure to start your database service beforehand.
```bash
# Reset the database:
npm run prisma -- migrate reset -f
npm run prisma -- migrate deploy

# Import example data:
# Switch to the directory containing the database's `docker-compose.yml`:
cd development

# Remove the initial workgroup as it will collide with the import:
docker compose exec db sh -c 'psql --dbname=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:5432/${POSTGRES_DB} -c "DELETE FROM workgroup"'

# Import example data:
docker compose exec db sh -c 'psql --dbname=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:5432/${POSTGRES_DB} -v ON_ERROR_STOP=1 -f /dump.sql'
```

Expand Down
4 changes: 1 addition & 3 deletions apps/server-asset-sg/.env
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,5 @@ OAUTH_SHOW_DEBUG_INFO=true
OAUTH_TOKEN_ENDPOINT=http://localhost:4011/connect/token
OAUTH_AUTHORIZED_GROUPS=assets.swissgeol
ANONYMOUS_MODE=false
OCR_URL=
OCR_CALLBACK_URL=


OCR_SERVICE_URL=http://localhost:5052
149 changes: 149 additions & 0 deletions apps/server-asset-sg/src/app.logger.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
import { Injectable, LoggerService, LogLevel } from '@nestjs/common';
import colors from 'colors/safe';

@Injectable()
export class AppLogger implements LoggerService {
levels: Set<LogLevel> | null = null;

log(message: unknown, ...optionalParams: unknown[]) {
this.write(levels.log, message, optionalParams);
}

error(message: unknown, ...optionalParams: unknown[]) {
this.write(levels.error, message, optionalParams);
}

warn(message: unknown, ...optionalParams: unknown[]) {
this.write(levels.warn, message, optionalParams);
}

debug?(message: unknown, ...optionalParams: unknown[]) {
this.write(levels.debug, message, optionalParams);
}

verbose?(message: unknown, ...optionalParams: unknown[]) {
this.write(levels.verbose, message, optionalParams);
}

fatal?(message: unknown, ...optionalParams: unknown[]) {
this.write(levels.fatal, message, optionalParams);
}

setLogLevels?(levels: LogLevel[]) {
this.levels = new Set(levels);
}

private hasLevel(level: LogLevel): boolean {
return this.levels == null || this.levels.has(level);
}

private write(level: Level, message: unknown, params: unknown[]) {
if (!this.hasLevel(level.key)) {
return;
}
const lastParam = params[params.length - 1];
let source = 'main';
if (typeof lastParam === 'string') {
source = lastParam;
params = params.slice(0, params.length - 1);
}

const now = new Date();
const nameSpacer = ' '.repeat(MAX_NAME_LENGTH - level.name.length);
const prefix =
colors.reset(` ${now.toISOString()} `) + nameSpacer + level.bgColor(` ${level.name} `) + ' ' + source;
let output = ' ' + level.color(`${message}`);
if (params.length !== 0) {
output += ' ' + stringify(params, level);
}
console.log(`${prefix} ${output}`);
}
}

const stringify = (value: unknown, level: Level, options: { isNested?: boolean } = {}): string => {
if (Array.isArray(value)) {
return stringifyArray(value, level, options);
}
if (value instanceof Error) {
return JSON.stringify(value.message);
}
if (value != null && typeof value === 'object') {
return stringifyObject(value, level, options);
}
return level.color(JSON.stringify(value));
};

const stringifyArray = (value: unknown[], level: Level, options: { isNested?: boolean } = {}): string => {
let output = '';
for (const element of value) {
if (output.length > 0) {
output += ', ';
}
output += stringify(element, level, { isNested: true });
}
return options.isNested ? `[${output}]` : output;
};

const stringifyObject = (value: object, level: Level, options: { isNested?: boolean } = {}): string => {
const valueString = value.toString();
if (valueString !== '[object Object]') {
return level.color(valueString);
}

let output = '';
for (const [k, v] of Object.entries(value)) {
if (output.length > 0) {
output += ', ';
}
output += `${k}: ${stringify(v, level, { isNested: true })}`;
}
return options.isNested ? `[${output}]` : output;
};

interface Level {
key: LogLevel;
name: string;
color: typeof colors.white;
bgColor: typeof colors.bgWhite;
}

const levels: Record<LogLevel, Level> = {
log: {
key: 'log',
name: 'INFO',
color: colors.green,
bgColor: colors.bgGreen,
},
error: {
key: 'error',
name: 'ERROR',
color: colors.red,
bgColor: colors.bgRed,
},
warn: {
key: 'warn',
name: 'WARN',
color: colors.yellow,
bgColor: colors.bgYellow,
},
debug: {
key: 'debug',
name: 'DEBUG',
color: colors.blue,
bgColor: colors.bgBlue,
},
verbose: {
key: 'verbose',
name: 'VERBOSE',
color: colors.magenta,
bgColor: colors.bgMagenta,
},
fatal: {
key: 'fatal',
name: 'FATAL',
color: colors.red,
bgColor: colors.bgRed,
},
};

const MAX_NAME_LENGTH = Math.max(...Object.values(levels).map((it) => it.name.length));
4 changes: 2 additions & 2 deletions apps/server-asset-sg/src/app.module.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@ import { ContactRepo } from '@/features/contacts/contact.repo';
import { ContactsController } from '@/features/contacts/contacts.controller';
import { FavoriteRepo } from '@/features/favorites/favorite.repo';
import { FavoritesController } from '@/features/favorites/favorites.controller';
import { FileOcrService } from '@/features/files/file-ocr.service';
import { FileRepo } from '@/features/files/file.repo';
import { FilesController } from '@/features/files/files.controller';
import { OcrController } from '@/features/ocr/ocr.controller';
import { StudiesController } from '@/features/studies/studies.controller';
import { StudyRepo } from '@/features/studies/study.repo';
import { UserRepo } from '@/features/users/user.repo';
Expand All @@ -43,7 +43,6 @@ import { WorkgroupsController } from '@/features/workgroups/workgroups.controlle
ContactsController,
FavoritesController,
FilesController,
OcrController,
StudiesController,
UsersController,
WorkgroupsController,
Expand All @@ -60,6 +59,7 @@ import { WorkgroupsController } from '@/features/workgroups/workgroups.controlle
ContactRepo,
FavoriteRepo,
FileRepo,
FileOcrService,
PrismaService,
StudyRepo,
UserRepo,
Expand Down
Loading

0 comments on commit 71dce22

Please sign in to comment.