Initial commit with Email Import Wizard and Task Processor updates
This commit is contained in:
@@ -0,0 +1,218 @@
|
||||
import { Injectable, Logger, OnApplicationBootstrap } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
import * as fs from 'fs/promises';
|
||||
import { PdfService } from '../preprocessing/pdf.service';
|
||||
import { QrCodeService } from '../preprocessing/qr-code.service';
|
||||
import {
|
||||
BarcodeTemplate,
|
||||
type BarcodeActionType,
|
||||
} from '../database/entities/barcode-template.entity';
|
||||
import { InboxDocument, type StoredQrCode } from '../database/entities/inbox-document.entity';
|
||||
import { PageCacheService } from './page-cache.service';
|
||||
|
||||
export interface MatchedBarcode {
|
||||
page: number;
|
||||
value: string;
|
||||
templateId: number | null;
|
||||
templateName: string | null;
|
||||
splitBefore: boolean;
|
||||
actions: BarcodeActionType[];
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class BarcodeScannerService implements OnApplicationBootstrap {
|
||||
private readonly logger = new Logger(BarcodeScannerService.name);
|
||||
private templatesCache: BarcodeTemplate[] | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly pdfService: PdfService,
|
||||
private readonly qrCodeService: QrCodeService,
|
||||
private readonly pageCache: PageCacheService,
|
||||
@InjectRepository(BarcodeTemplate)
|
||||
private readonly templateRepo: Repository<BarcodeTemplate>,
|
||||
@InjectRepository(InboxDocument)
|
||||
private readonly documentRepo: Repository<InboxDocument>,
|
||||
) {}
|
||||
|
||||
async onApplicationBootstrap(): Promise<void> {
|
||||
await this.migrateLegacySplitBefore();
|
||||
}
|
||||
|
||||
invalidateTemplates(): void {
|
||||
this.templatesCache = null;
|
||||
}
|
||||
|
||||
private async migrateLegacySplitBefore(): Promise<void> {
|
||||
let rows: BarcodeTemplate[];
|
||||
try {
|
||||
rows = await this.templateRepo.find();
|
||||
} catch (err: any) {
|
||||
this.logger.warn(`Template-Migration: Query fehlgeschlagen: ${err.message}`);
|
||||
return;
|
||||
}
|
||||
let migrated = 0;
|
||||
for (const tpl of rows) {
|
||||
const actions = (tpl.Actions ?? []) as string[];
|
||||
if (actions.includes('SPLIT_BEFORE')) {
|
||||
tpl.SplitBefore = true;
|
||||
tpl.Actions = actions.filter((a) => a !== 'SPLIT_BEFORE') as BarcodeActionType[];
|
||||
await this.templateRepo.save(tpl);
|
||||
migrated += 1;
|
||||
}
|
||||
}
|
||||
if (migrated > 0) {
|
||||
this.logger.log(`Template-Migration: ${migrated} Vorlage(n) auf SplitBefore-Flag umgestellt`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rendert alle Seiten, extrahiert QR-Codes, persistiert Page-Cache + DB-Row.
|
||||
* Wird nach dem Move aus dem Watcher und beim Backfill aufgerufen.
|
||||
*/
|
||||
async scanAndMatch(doc: InboxDocument): Promise<MatchedBarcode[]> {
|
||||
const pdfPath = this.pageCache.documentPdfPath(doc.Id);
|
||||
const { qrCodes, pageCount } = await this.performScan(doc.Id, pdfPath);
|
||||
|
||||
doc.QrCodes = qrCodes;
|
||||
doc.PageCount = pageCount;
|
||||
try {
|
||||
await this.documentRepo.save(doc);
|
||||
} catch (err: any) {
|
||||
this.logger.warn(`Scan-Ergebnis konnte nicht gespeichert werden (${doc.Id}): ${err.message}`);
|
||||
}
|
||||
|
||||
return this.matchTemplates(qrCodes);
|
||||
}
|
||||
|
||||
/**
|
||||
* Scannt nur, wenn die Row noch keine Seitenanzahl hat (= noch nie gescannt).
|
||||
*/
|
||||
async ensureScanned(doc: InboxDocument): Promise<boolean> {
|
||||
if (doc.PageCount > 0) return false;
|
||||
await this.scanAndMatch(doc);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read-only: mapped die persistierten QR-Codes auf MatchedBarcodes.
|
||||
*/
|
||||
async getMatched(doc: InboxDocument): Promise<MatchedBarcode[]> {
|
||||
return this.matchTemplates(doc.QrCodes ?? []);
|
||||
}
|
||||
|
||||
private async matchTemplates(qrCodes: StoredQrCode[]): Promise<MatchedBarcode[]> {
|
||||
if (qrCodes.length === 0) return [];
|
||||
const templates = await this.getTemplates();
|
||||
return qrCodes.map((qr) => {
|
||||
const tpl = this.firstMatch(qr.value, templates);
|
||||
return {
|
||||
page: qr.page,
|
||||
value: qr.value,
|
||||
templateId: tpl?.Id ?? null,
|
||||
templateName: tpl?.Name ?? null,
|
||||
splitBefore: tpl?.SplitBefore ?? false,
|
||||
actions: tpl?.Actions ?? [],
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
private firstMatch(value: string, templates: BarcodeTemplate[]): BarcodeTemplate | null {
|
||||
for (const tpl of templates) {
|
||||
try {
|
||||
const re = new RegExp(tpl.Regex);
|
||||
if (re.test(value)) return tpl;
|
||||
} catch {
|
||||
// ignore invalid regex
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private async getTemplates(): Promise<BarcodeTemplate[]> {
|
||||
if (!this.templatesCache) {
|
||||
this.templatesCache = await this.templateRepo.find({ order: { Id: 'ASC' } });
|
||||
}
|
||||
return this.templatesCache;
|
||||
}
|
||||
|
||||
private async performScan(
|
||||
documentId: string,
|
||||
pdfPath: string,
|
||||
): Promise<{ qrCodes: StoredQrCode[]; pageCount: number }> {
|
||||
let images: string[] = [];
|
||||
try {
|
||||
images = await this.pdfService.pdfToImages(pdfPath, 400);
|
||||
const qrCodes: StoredQrCode[] = [];
|
||||
const templates = await this.getTemplates();
|
||||
|
||||
for (let i = 0; i < images.length; i++) {
|
||||
try {
|
||||
const buffer = await fs.readFile(images[i]);
|
||||
const qrs = await this.qrCodeService.extractFromImage(buffer);
|
||||
// Nur QR-Codes speichern, die zu einer Eingangsdokumentart passen.
|
||||
// Mehrere passende QRs pro Seite werden alle übernommen.
|
||||
for (const qr of qrs) {
|
||||
if (this.firstMatch(qr.data, templates)) {
|
||||
qrCodes.push({ page: i + 1, value: qr.data });
|
||||
}
|
||||
}
|
||||
} catch (err: any) {
|
||||
this.logger.warn(`QR-Scan fehlgeschlagen (${pdfPath}, Seite ${i + 1}): ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
await this.pageCache.clear(documentId);
|
||||
await this.pageCache.generate(documentId, images);
|
||||
|
||||
return { qrCodes, pageCount: images.length };
|
||||
} catch (err: any) {
|
||||
this.logger.warn(`Kein QR-Scan möglich für ${pdfPath}: ${err.message}`);
|
||||
return { qrCodes: [], pageCount: 0 };
|
||||
} finally {
|
||||
await this.pdfService.cleanup(images);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rescannt alle Inbox-Dokumente — wird nach Änderungen an Eingangsdokumentarten aufgerufen.
|
||||
* Läuft sequenziell, um PDF-Rendering nicht zu überlasten. Fire-and-forget vom Caller.
|
||||
*/
|
||||
async rescanAll(): Promise<{ scanned: number; failed: number }> {
|
||||
this.invalidateTemplates();
|
||||
let docs: InboxDocument[];
|
||||
try {
|
||||
docs = await this.documentRepo.find();
|
||||
} catch (err: any) {
|
||||
this.logger.warn(`Rescan: DB-Query fehlgeschlagen: ${err.message}`);
|
||||
return { scanned: 0, failed: 0 };
|
||||
}
|
||||
if (docs.length === 0) return { scanned: 0, failed: 0 };
|
||||
|
||||
this.logger.log(`Rescan: starte Neuerfassung für ${docs.length} Inbox-Dokument(e)`);
|
||||
let scanned = 0;
|
||||
let failed = 0;
|
||||
for (const doc of docs) {
|
||||
try {
|
||||
const pdfPath = this.pageCache.documentPdfPath(doc.Id);
|
||||
try {
|
||||
await fs.access(pdfPath);
|
||||
} catch {
|
||||
this.logger.warn(`Rescan: PDF fehlt für ${doc.Id} (${pdfPath})`);
|
||||
failed++;
|
||||
continue;
|
||||
}
|
||||
const { qrCodes, pageCount } = await this.performScan(doc.Id, pdfPath);
|
||||
doc.QrCodes = qrCodes;
|
||||
doc.PageCount = pageCount;
|
||||
await this.documentRepo.save(doc);
|
||||
scanned++;
|
||||
} catch (err: any) {
|
||||
this.logger.warn(`Rescan fehlgeschlagen für ${doc.Id}: ${err.message}`);
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
this.logger.log(`Rescan abgeschlossen: ${scanned} ok, ${failed} fehlgeschlagen`);
|
||||
return { scanned, failed };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
import {
|
||||
Body,
|
||||
Controller,
|
||||
Delete,
|
||||
Get,
|
||||
Logger,
|
||||
Param,
|
||||
ParseIntPipe,
|
||||
Post,
|
||||
Put,
|
||||
BadRequestException,
|
||||
NotFoundException,
|
||||
} from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
import { BarcodeTemplate, type BarcodeActionType } from '../database/entities/barcode-template.entity';
|
||||
import { RequirePermissions } from '../auth/permissions.decorator';
|
||||
import { Permission } from '../auth/permissions.enum';
|
||||
import { BarcodeScannerService } from './barcode-scanner.service';
|
||||
|
||||
const VALID_ACTIONS: BarcodeActionType[] = ['SEND_TO_PAPERLESS', 'SEND_BY_EMAIL'];
|
||||
|
||||
interface UpsertDto {
|
||||
Name?: string;
|
||||
Regex?: string;
|
||||
SplitBefore?: boolean;
|
||||
DateinameTemplate?: string | null;
|
||||
Actions?: BarcodeActionType[];
|
||||
}
|
||||
|
||||
function validate(dto: UpsertDto, partial = false): void {
|
||||
if (!partial || dto.Name !== undefined) {
|
||||
if (typeof dto.Name !== 'string' || !dto.Name.trim()) {
|
||||
throw new BadRequestException('Name ist erforderlich');
|
||||
}
|
||||
}
|
||||
if (!partial || dto.Regex !== undefined) {
|
||||
if (typeof dto.Regex !== 'string' || !dto.Regex.trim()) {
|
||||
throw new BadRequestException('Regex ist erforderlich');
|
||||
}
|
||||
try {
|
||||
new RegExp(dto.Regex);
|
||||
} catch {
|
||||
throw new BadRequestException('Regex ist ungültig');
|
||||
}
|
||||
}
|
||||
if (dto.SplitBefore !== undefined && typeof dto.SplitBefore !== 'boolean') {
|
||||
throw new BadRequestException('SplitBefore muss ein Boolean sein');
|
||||
}
|
||||
if (dto.DateinameTemplate !== undefined && dto.DateinameTemplate !== null &&
|
||||
typeof dto.DateinameTemplate !== 'string') {
|
||||
throw new BadRequestException('DateinameTemplate muss ein String sein');
|
||||
}
|
||||
if (!partial || dto.Actions !== undefined) {
|
||||
if (!Array.isArray(dto.Actions)) {
|
||||
throw new BadRequestException('Actions muss eine Liste sein');
|
||||
}
|
||||
for (const a of dto.Actions) {
|
||||
if (!VALID_ACTIONS.includes(a)) {
|
||||
throw new BadRequestException(`Unbekannte Aktion: ${a}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Controller('api/barcode-templates')
|
||||
@RequirePermissions(Permission.MANAGE_SETTINGS)
|
||||
export class BarcodeTemplatesController {
|
||||
private readonly logger = new Logger(BarcodeTemplatesController.name);
|
||||
|
||||
constructor(
|
||||
@InjectRepository(BarcodeTemplate)
|
||||
private readonly repo: Repository<BarcodeTemplate>,
|
||||
private readonly scanner: BarcodeScannerService,
|
||||
) {}
|
||||
|
||||
private triggerRescan(): void {
|
||||
this.scanner.rescanAll().catch((err) => {
|
||||
this.logger.error(`Rescan nach Template-Änderung fehlgeschlagen: ${err.message}`);
|
||||
});
|
||||
}
|
||||
|
||||
@Get()
|
||||
async list() {
|
||||
return this.repo.find({ order: { Id: 'ASC' } });
|
||||
}
|
||||
|
||||
@Post()
|
||||
async create(@Body() dto: UpsertDto) {
|
||||
validate(dto);
|
||||
const entity = this.repo.create({
|
||||
Name: dto.Name!.trim(),
|
||||
Regex: dto.Regex!,
|
||||
SplitBefore: dto.SplitBefore ?? false,
|
||||
DateinameTemplate: dto.DateinameTemplate ?? null,
|
||||
Actions: dto.Actions!,
|
||||
});
|
||||
const saved = await this.repo.save(entity);
|
||||
this.scanner.invalidateTemplates();
|
||||
this.triggerRescan();
|
||||
return saved;
|
||||
}
|
||||
|
||||
@Put(':id')
|
||||
async update(@Param('id', ParseIntPipe) id: number, @Body() dto: UpsertDto) {
|
||||
validate(dto, true);
|
||||
const existing = await this.repo.findOneBy({ Id: id });
|
||||
if (!existing) throw new NotFoundException('Vorlage nicht gefunden');
|
||||
|
||||
const regexChanged = dto.Regex !== undefined && dto.Regex !== existing.Regex;
|
||||
|
||||
if (dto.Name !== undefined) existing.Name = dto.Name.trim();
|
||||
if (dto.Regex !== undefined) existing.Regex = dto.Regex;
|
||||
if (dto.SplitBefore !== undefined) existing.SplitBefore = dto.SplitBefore;
|
||||
if (dto.DateinameTemplate !== undefined) existing.DateinameTemplate = dto.DateinameTemplate ?? null;
|
||||
if (dto.Actions !== undefined) existing.Actions = dto.Actions;
|
||||
|
||||
const saved = await this.repo.save(existing);
|
||||
this.scanner.invalidateTemplates();
|
||||
if (regexChanged) {
|
||||
// Nur bei Regex-Änderungen rescannen — Name/Aktionen ändern nichts am Match-Set.
|
||||
this.triggerRescan();
|
||||
}
|
||||
return saved;
|
||||
}
|
||||
|
||||
@Delete(':id')
|
||||
async remove(@Param('id', ParseIntPipe) id: number) {
|
||||
const res = await this.repo.delete(id);
|
||||
if (!res.affected) throw new NotFoundException('Vorlage nicht gefunden');
|
||||
this.scanner.invalidateTemplates();
|
||||
this.triggerRescan();
|
||||
return { ok: true };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { BarcodeTemplate } from '../database/entities/barcode-template.entity';
|
||||
import { InboxDocument } from '../database/entities/inbox-document.entity';
|
||||
import { BarcodeTemplatesController } from './barcode-templates.controller';
|
||||
import { BarcodeScannerService } from './barcode-scanner.service';
|
||||
import { PageCacheService } from './page-cache.service';
|
||||
import { PreprocessingModule } from '../preprocessing/preprocessing.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
TypeOrmModule.forFeature([BarcodeTemplate, InboxDocument]),
|
||||
PreprocessingModule,
|
||||
],
|
||||
controllers: [BarcodeTemplatesController],
|
||||
providers: [BarcodeScannerService, PageCacheService],
|
||||
exports: [BarcodeScannerService, PageCacheService],
|
||||
})
|
||||
export class BarcodeModule {}
|
||||
@@ -0,0 +1,117 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import * as path from 'path';
|
||||
import * as fs from 'fs/promises';
|
||||
import sharp from 'sharp';
|
||||
|
||||
const THUMBNAIL_WIDTH = 180;
|
||||
|
||||
@Injectable()
|
||||
export class PageCacheService {
|
||||
private readonly logger = new Logger(PageCacheService.name);
|
||||
private readonly inboxRoot: string;
|
||||
|
||||
constructor(configService: ConfigService) {
|
||||
this.inboxRoot = configService.get<string>('INBOX_DATA_DIR', '/mnt/data/inbox');
|
||||
}
|
||||
|
||||
documentDir(documentId: string): string {
|
||||
return path.join(this.inboxRoot, documentId);
|
||||
}
|
||||
|
||||
documentPdfPath(documentId: string): string {
|
||||
return path.join(this.documentDir(documentId), 'document.pdf');
|
||||
}
|
||||
|
||||
previewPath(documentId: string, page: number): string {
|
||||
return path.join(this.documentDir(documentId), `page-${page}.preview.png`);
|
||||
}
|
||||
|
||||
thumbnailPath(documentId: string, page: number): string {
|
||||
return path.join(this.documentDir(documentId), `page-${page}.thumb.png`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Übernimmt die bereits gerenderten 200-dpi-PNGs als preview.png und
|
||||
* erzeugt parallel eine kleinere thumb.png pro Seite.
|
||||
*/
|
||||
async generate(documentId: string, renderedImages: string[]): Promise<void> {
|
||||
const dir = this.documentDir(documentId);
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
|
||||
for (let i = 0; i < renderedImages.length; i++) {
|
||||
const page = i + 1;
|
||||
const src = renderedImages[i];
|
||||
const previewDest = this.previewPath(documentId, page);
|
||||
const thumbDest = this.thumbnailPath(documentId, page);
|
||||
|
||||
try {
|
||||
await fs.copyFile(src, previewDest);
|
||||
await sharp(src).resize({ width: THUMBNAIL_WIDTH }).png().toFile(thumbDest);
|
||||
} catch (err: any) {
|
||||
this.logger.warn(
|
||||
`Seiten-Cache fehlgeschlagen (${documentId} Seite ${page}): ${err.message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Entfernt alle page-*.png-Dateien eines Dokuments (Original-PDF bleibt).
|
||||
*/
|
||||
async clear(documentId: string): Promise<void> {
|
||||
const dir = this.documentDir(documentId);
|
||||
let entries: string[];
|
||||
try {
|
||||
entries = await fs.readdir(dir);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const name of entries) {
|
||||
if (!/^page-\d+\.(preview|thumb)\.png$/.test(name)) continue;
|
||||
await fs.unlink(path.join(dir, name)).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Verschiebt Page-Cache-Dateien nach dem Löschen einer Seite:
|
||||
* page-N.*.png weg, page-(N+1..oldPageCount) rutschen um 1 nach vorne.
|
||||
*/
|
||||
async shiftAfterPageDelete(
|
||||
documentId: string,
|
||||
deletedPage: number,
|
||||
oldPageCount: number,
|
||||
): Promise<void> {
|
||||
const dir = this.documentDir(documentId);
|
||||
await fs
|
||||
.unlink(path.join(dir, `page-${deletedPage}.thumb.png`))
|
||||
.catch(() => undefined);
|
||||
await fs
|
||||
.unlink(path.join(dir, `page-${deletedPage}.preview.png`))
|
||||
.catch(() => undefined);
|
||||
|
||||
for (let n = deletedPage + 1; n <= oldPageCount; n++) {
|
||||
for (const variant of ['thumb', 'preview'] as const) {
|
||||
const from = path.join(dir, `page-${n}.${variant}.png`);
|
||||
const to = path.join(dir, `page-${n - 1}.${variant}.png`);
|
||||
try {
|
||||
await fs.rename(from, to);
|
||||
} catch (err: any) {
|
||||
this.logger.warn(
|
||||
`Cache-Shift fehlgeschlagen (${documentId} Seite ${n} ${variant}): ${err.message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async hasPreview(documentId: string, page: number): Promise<boolean> {
|
||||
try {
|
||||
await fs.access(this.previewPath(documentId, page));
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user