bjoernpoettker
aa4c181b0c
Build and Push Multi-Platform Images / build-and-push (push) Successful in 48s
491 lines
20 KiB
TypeScript
491 lines
20 KiB
TypeScript
import { Injectable, Logger, HttpException, HttpStatus } from '@nestjs/common';
|
|
import { ConfigService } from '@nestjs/config';
|
|
import { InjectRepository } from '@nestjs/typeorm';
|
|
import { Repository } from 'typeorm';
|
|
import axios from 'axios';
|
|
import { PDFDocument, rgb, StandardFonts } from 'pdf-lib';
|
|
import { Attachment } from '../database/entities/attachment.entity';
|
|
import { Email } from '../database/entities/email.entity';
|
|
import { Content } from '../database/entities/content.entity';
|
|
import { CorrespondentEmailMapping } from '../database/entities/correspondent-email-mapping.entity';
|
|
import { Task } from '../database/entities/task.entity';
|
|
import { PaperlessService } from '../paperless/paperless.service';
|
|
import * as QRCode from 'qrcode';
|
|
import { EmailPageCacheService } from './email-page-cache.service';
|
|
import { PdfService } from '../preprocessing/pdf.service';
|
|
import * as path from 'path';
|
|
import * as os from 'os';
|
|
import * as fs from 'fs/promises';
|
|
import * as crypto from 'crypto';
|
|
|
|
@Injectable()
|
|
export class EmailImportService {
|
|
private readonly logger = new Logger(EmailImportService.name);
|
|
|
|
constructor(
|
|
private readonly configService: ConfigService,
|
|
@InjectRepository(Email) private readonly emailRepo: Repository<Email>,
|
|
@InjectRepository(Attachment) private readonly attachmentRepo: Repository<Attachment>,
|
|
@InjectRepository(Content) private readonly contentRepo: Repository<Content>,
|
|
@InjectRepository(CorrespondentEmailMapping) private readonly mappingRepo: Repository<CorrespondentEmailMapping>,
|
|
@InjectRepository(Task) private readonly taskRepo: Repository<Task>,
|
|
private readonly paperlessService: PaperlessService,
|
|
private readonly pdfService: PdfService,
|
|
private readonly pageCache: EmailPageCacheService,
|
|
) {}
|
|
|
|
async ensurePreviews(emailId: number): Promise<void> {
|
|
const attachments = await this.attachmentRepo.find({
|
|
where: { EmailMessageId: emailId, ContentType: 'application/pdf' },
|
|
relations: ['Content'],
|
|
});
|
|
|
|
for (const attachment of attachments) {
|
|
const hasPreview = await this.pageCache.hasPreview(attachment.Id, 1);
|
|
if (!hasPreview && attachment.Content?.Content1) {
|
|
this.logger.log(`Generiere fehlende Vorschaubilder für Anhang ${attachment.Id} (Email ${emailId})`);
|
|
const tempPdfPath = path.join(os.tmpdir(), `email-att-gen-${attachment.Id}.pdf`);
|
|
try {
|
|
await fs.writeFile(tempPdfPath, attachment.Content.Content1);
|
|
|
|
const images = await this.pdfService.pdfToImages(tempPdfPath, 400);
|
|
await this.pageCache.generate(attachment.Id, images);
|
|
|
|
attachment.PageCount = images.length;
|
|
await this.attachmentRepo.save(attachment);
|
|
|
|
await this.pdfService.cleanup(images);
|
|
} catch (err: any) {
|
|
this.logger.warn(`Fehler bei on-demand Vorschau-Generierung für Anhang ${attachment.Id}: ${err.message}`);
|
|
} finally {
|
|
await fs.unlink(tempPdfPath).catch(() => {});
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// --- Korrespondenten Mapping ---
|
|
async getMappings() {
|
|
return this.mappingRepo.find();
|
|
}
|
|
|
|
async addMapping(emailAddress: string, paperlessCorrespondentId: number) {
|
|
let mapping = await this.mappingRepo.findOne({ where: { EmailAddress: emailAddress } });
|
|
if (!mapping) {
|
|
mapping = this.mappingRepo.create({ EmailAddress: emailAddress, PaperlessCorrespondentId: paperlessCorrespondentId });
|
|
} else {
|
|
mapping.PaperlessCorrespondentId = paperlessCorrespondentId;
|
|
}
|
|
return this.mappingRepo.save(mapping);
|
|
}
|
|
|
|
async deleteMapping(id: number) {
|
|
return this.mappingRepo.delete(id);
|
|
}
|
|
|
|
async getCorrespondentByEmail(emailAddress: string): Promise<number | null> {
|
|
const mapping = await this.mappingRepo.findOne({ where: { EmailAddress: emailAddress } });
|
|
return mapping ? mapping.PaperlessCorrespondentId : null;
|
|
}
|
|
|
|
// --- Belegnummern API ---
|
|
private buildUrl(urlTemplate: string, dateStr: string): string {
|
|
const dateObj = new Date(dateStr);
|
|
const year = (isNaN(dateObj.getTime()) ? new Date() : dateObj).getFullYear().toString();
|
|
return urlTemplate.replace('{Jahr}', year);
|
|
}
|
|
|
|
async getBelegnummer(emailDate: string): Promise<string> {
|
|
const urlTemplate = this.configService.get<string>('BELEGNUMMER_GET_URL');
|
|
if (!urlTemplate) throw new HttpException('BELEGNUMMER_GET_URL not configured', HttpStatus.INTERNAL_SERVER_ERROR);
|
|
|
|
const url = this.buildUrl(urlTemplate, emailDate);
|
|
try {
|
|
this.logger.debug(`Fetching Belegnummer from ${url}`);
|
|
const response = await axios.get(url);
|
|
|
|
// If the response is an object, try to extract 'nummer' or 'number'
|
|
let result = response.data;
|
|
if (result && typeof result === 'object') {
|
|
result = result.nummer || result.number || result.data?.nummer || JSON.stringify(result);
|
|
}
|
|
|
|
this.logger.debug(`Received Belegnummer: ${result}`);
|
|
return String(result);
|
|
} catch (error: any) {
|
|
const status = error.response?.status || 'UNKNOWN';
|
|
const detail = error.response?.data ? JSON.stringify(error.response.data) : error.message;
|
|
this.logger.error(`Failed to fetch Belegnummer from ${url}. Status: ${status}, Detail: ${detail}`);
|
|
throw new HttpException(`Fehler beim Abrufen der Belegnummer: ${detail}`, HttpStatus.BAD_GATEWAY);
|
|
}
|
|
}
|
|
|
|
async releaseBelegnummer(emailDate: string, number: string): Promise<void> {
|
|
const urlTemplate = this.configService.get<string>('BELEGNUMMER_RELEASE_URL');
|
|
if (!urlTemplate) {
|
|
this.logger.warn('BELEGNUMMER_RELEASE_URL not configured, skipping release.');
|
|
return;
|
|
}
|
|
|
|
const cleanNumber = number.replace(/^0+/, '') || '0';
|
|
let url = this.buildUrl(urlTemplate, emailDate);
|
|
url = url.replace('{Nummer}', cleanNumber);
|
|
|
|
try {
|
|
this.logger.log(`Releasing Belegnummer: ${cleanNumber} (original: ${number}) via ${url}`);
|
|
await axios.get(url);
|
|
} catch (error: any) {
|
|
this.logger.error(`Failed to release Belegnummer at ${url}: ${error.message}`);
|
|
}
|
|
}
|
|
|
|
async setBelegnummer(emailDate: string, number: string): Promise<void> {
|
|
const urlTemplate = this.configService.get<string>('BELEGNUMMER_SET_URL');
|
|
if (!urlTemplate) throw new HttpException('BELEGNUMMER_SET_URL not configured', HttpStatus.INTERNAL_SERVER_ERROR);
|
|
|
|
const cleanNumber = number.replace(/^0+/, '') || '0';
|
|
let url = this.buildUrl(urlTemplate, emailDate);
|
|
url = url.replace('{Nummer}', cleanNumber);
|
|
|
|
try {
|
|
this.logger.log(`Setting Belegnummer: ${cleanNumber} (original: ${number}) via ${url}`);
|
|
await axios.get(url);
|
|
} catch (error: any) {
|
|
this.logger.error(`Failed to set Belegnummer at ${url}: ${error.message}`);
|
|
throw new HttpException('Fehler beim Setzen der Belegnummer', HttpStatus.BAD_GATEWAY);
|
|
}
|
|
}
|
|
|
|
// --- Checksum Check for Split Documents ---
|
|
async checkSplitChecksum(attachmentId: number, pages: { start: number; end: number }): Promise<boolean> {
|
|
const content = await this.contentRepo.findOne({ where: { AttachmentEntityId: attachmentId } });
|
|
if (!content) return false;
|
|
|
|
const pdfDoc = await PDFDocument.load(content.Content1, { ignoreEncryption: true });
|
|
const total = pdfDoc.getPageCount();
|
|
const startIdx = Math.max(1, pages.start) - 1;
|
|
const endIdx = Math.min(pages.end === 999 ? total : pages.end, total) - 1;
|
|
|
|
const sliced = await PDFDocument.create();
|
|
const indices = Array.from({ length: endIdx - startIdx + 1 }, (_, i) => startIdx + i);
|
|
const copied = await sliced.copyPages(pdfDoc, indices);
|
|
copied.forEach(p => sliced.addPage(p));
|
|
|
|
const checksum = crypto.createHash('md5').update(Buffer.from(await sliced.save())).digest('hex');
|
|
return this.paperlessService.checksumExists(checksum);
|
|
}
|
|
|
|
// --- Print Preview ---
|
|
async generatePrintPdf(attachmentId: number, barcodeData: any): Promise<Buffer> {
|
|
const content = await this.contentRepo.findOne({ where: { AttachmentEntityId: attachmentId } });
|
|
if (!content) throw new HttpException('Inhalt nicht gefunden', HttpStatus.NOT_FOUND);
|
|
|
|
let pdfBytes: Buffer = content.Content1;
|
|
|
|
const pages: { start: number; end: number } | undefined = barcodeData._pages;
|
|
if (pages) {
|
|
const pdfDoc = await PDFDocument.load(pdfBytes, { ignoreEncryption: true });
|
|
const total = pdfDoc.getPageCount();
|
|
const startIdx = Math.max(1, pages.start) - 1;
|
|
const endIdx = Math.min(pages.end === 999 ? total : pages.end, total) - 1;
|
|
const sliced = await PDFDocument.create();
|
|
const indices = Array.from({ length: endIdx - startIdx + 1 }, (_, i) => startIdx + i);
|
|
const copied = await sliced.copyPages(pdfDoc, indices);
|
|
copied.forEach(p => sliced.addPage(p));
|
|
pdfBytes = Buffer.from(await sliced.save());
|
|
}
|
|
|
|
const { _pages, ...barcode } = barcodeData;
|
|
return this.applyBarcodeToPdf(pdfBytes, barcode);
|
|
}
|
|
|
|
async applyBarcodeToPdf(pdfBytes: Buffer, barcodeData: any): Promise<Buffer> {
|
|
this.logger.debug(`applyBarcodeToPdf: Input size = ${pdfBytes.length} bytes`);
|
|
|
|
let currentPdfBytes = pdfBytes;
|
|
const tempInputPath = path.join(os.tmpdir(), `input-${Date.now()}.pdf`);
|
|
await fs.writeFile(tempInputPath, pdfBytes);
|
|
|
|
try {
|
|
// First try to load to check encryption
|
|
let pdfDoc = await PDFDocument.load(currentPdfBytes, { ignoreEncryption: true });
|
|
|
|
if (pdfDoc.isEncrypted) {
|
|
this.logger.log('PDF ist verschlüsselt, versuche Bereinigung via Ghostscript...');
|
|
const sanitizedPath = await this.pdfService.sanitizePdf(tempInputPath);
|
|
currentPdfBytes = await fs.readFile(sanitizedPath);
|
|
await fs.unlink(sanitizedPath).catch(() => {});
|
|
// Reload sanitized PDF
|
|
pdfDoc = await PDFDocument.load(currentPdfBytes);
|
|
}
|
|
|
|
const pages = pdfDoc.getPages();
|
|
this.logger.debug(`applyBarcodeToPdf: Pages = ${pages.length}, Encrypted = ${pdfDoc.isEncrypted}`);
|
|
|
|
if (pages.length === 0) {
|
|
this.logger.warn('applyBarcodeToPdf: Keine Seiten gefunden');
|
|
return Buffer.from(await pdfDoc.save());
|
|
}
|
|
|
|
const firstPage = pages[0];
|
|
|
|
const { x, y, nummer, datum, jahr } = barcodeData;
|
|
|
|
// Parse date
|
|
const d = new Date(datum);
|
|
const yyyy = (isNaN(d.getTime()) ? new Date() : d).getFullYear().toString();
|
|
const mm = String((isNaN(d.getTime()) ? new Date() : d).getMonth() + 1).padStart(2, '0');
|
|
const dd = String((isNaN(d.getTime()) ? new Date() : d).getDate()).padStart(2, '0');
|
|
|
|
const qrDateStr = `${yyyy}${mm}${dd}`; // yyyyMMdd
|
|
const qrContent = `${String(jahr).padStart(4, '0')}-${String(nummer).padStart(6, '0')}-${qrDateStr}`;
|
|
const printDateStr = `${dd}.${mm}.${yyyy}`;
|
|
|
|
// Dimensions: 57x32 mm
|
|
const PT_PER_MM = 2.83465;
|
|
const boxW = 57 * PT_PER_MM;
|
|
const boxH = 32 * PT_PER_MM;
|
|
|
|
// A4 dimensions: 210x297 mm
|
|
const PAGE_H_PT = 297 * PT_PER_MM;
|
|
|
|
// Convert mm to points (Y is from bottom in pdf-lib)
|
|
const startX = Number(x) * PT_PER_MM;
|
|
const startY = PAGE_H_PT - (Number(y) * PT_PER_MM) - boxH;
|
|
|
|
// 1. Draw Background Box (White with Black border)
|
|
firstPage.drawRectangle({
|
|
x: startX,
|
|
y: startY,
|
|
width: boxW,
|
|
height: boxH,
|
|
color: rgb(1, 1, 1),
|
|
borderColor: rgb(0, 0, 0),
|
|
borderWidth: 1,
|
|
});
|
|
|
|
// 2. Draw QR Code
|
|
const qrBuffer = await QRCode.toBuffer(qrContent, {
|
|
errorCorrectionLevel: 'H',
|
|
margin: 0,
|
|
width: 300,
|
|
color: { dark: '#000000', light: '#FFFFFF' }
|
|
});
|
|
const qrImage = await pdfDoc.embedPng(qrBuffer);
|
|
|
|
// QR Code size: 27x27 mm (10% smaller than 30x30)
|
|
const qrSize = 27 * PT_PER_MM;
|
|
const padding = (32 - 27) / 2; // Center vertically in 32mm box
|
|
const qrX = startX + (padding * PT_PER_MM);
|
|
const qrY = startY + (padding * PT_PER_MM);
|
|
|
|
firstPage.drawImage(qrImage, {
|
|
x: qrX,
|
|
y: qrY,
|
|
width: qrSize,
|
|
height: qrSize,
|
|
});
|
|
|
|
// 3. Draw Texts
|
|
const helveticaBold = await pdfDoc.embedFont(StandardFonts.HelveticaBold);
|
|
|
|
// Helper to draw centered text in a specific area
|
|
const drawCenteredInArea = (text: string, relX: number, relY: number, areaW: number, areaH: number, fontSize: number) => {
|
|
const textWidth = helveticaBold.widthOfTextAtSize(text, fontSize);
|
|
const absX = startX + (relX * PT_PER_MM) + (areaW * PT_PER_MM / 2) - (textWidth / 2);
|
|
const absY = (startY + boxH) - (relY * PT_PER_MM) - (areaH * PT_PER_MM / 2) - (fontSize / 2.5);
|
|
|
|
firstPage.drawText(text, {
|
|
x: absX,
|
|
y: absY,
|
|
size: fontSize,
|
|
font: helveticaBold,
|
|
color: rgb(0, 0, 0),
|
|
});
|
|
};
|
|
|
|
const isNeu = barcodeData.isNeu === true;
|
|
|
|
// Text Area X: +33.3mm, Width: 21mm
|
|
// Year: Y + 3mm, Height: 7.5mm
|
|
drawCenteredInArea(String(jahr).padStart(4, '0'), 33.3, 3, 21, 7.5, 12);
|
|
|
|
// Number: Y + 10.5mm, Height: 7.5mm
|
|
const numberText = isNeu ? '<- neu ->' : String(nummer).padStart(6, '0');
|
|
drawCenteredInArea(numberText, 33.3, 10.5, 21, 7.5, 12);
|
|
|
|
// "Eingegangen": Y + 19mm, Height: 4mm, Size 8
|
|
drawCenteredInArea('Eingegangen', 33.3, 19, 21, 4, 8);
|
|
|
|
// Date: Y + 19 + 3.5 = 22.5mm, Height: 4mm, Size 8
|
|
drawCenteredInArea(printDateStr, 33.3, 22.5, 21, 4, 8);
|
|
|
|
return Buffer.from(await pdfDoc.save());
|
|
} finally {
|
|
await fs.unlink(tempInputPath).catch(() => {});
|
|
}
|
|
}
|
|
|
|
// --- Import Logic ---
|
|
async executeImport(data: {
|
|
attachments: {
|
|
attachmentId: number;
|
|
type: 'MAIN' | 'ATTACHMENT' | 'IGNORE';
|
|
paperlessCorrespondentId?: number | null;
|
|
parentDocumentId?: number | null; // Used if type is ATTACHMENT (should map to a Custom Field theoretically, or just tags. For now, CF if configured, but we pass it)
|
|
splitRanges?: { start: number; end: number }[]; // 1-based pages, e.g. [{start: 1, end: 3}, {start: 4, end: 5}]
|
|
barcode?: { x: number; y: number; nummer: string; datum: string; jahr: string };
|
|
belegnummer?: string;
|
|
}[];
|
|
emailDate: string;
|
|
}): Promise<{ success: boolean; results: any[] }> {
|
|
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'paperless-mail-import-'));
|
|
const results = [];
|
|
|
|
try {
|
|
for (const att of data.attachments) {
|
|
if (att.type === 'IGNORE') continue;
|
|
|
|
const attachmentEntity = await this.attachmentRepo.findOne({ where: { Id: att.attachmentId } });
|
|
if (!attachmentEntity) continue;
|
|
|
|
const content = await this.contentRepo.findOne({ where: { AttachmentEntityId: att.attachmentId } });
|
|
if (!content) continue;
|
|
|
|
const originalPdfBytes = content.Content1;
|
|
const baseFilename = attachmentEntity.FileName.replace(/\.pdf$/i, '');
|
|
const paperlessIds: any = {};
|
|
|
|
const uploadPromises = [];
|
|
|
|
// Formatting the date for Paperless (ISO format)
|
|
const createdDate = new Date(data.emailDate).toISOString();
|
|
|
|
if (att.splitRanges && att.splitRanges.length > 0) {
|
|
// SPLIT PDF
|
|
const pdfDoc = await PDFDocument.load(originalPdfBytes, { ignoreEncryption: true });
|
|
const totalPages = pdfDoc.getPageCount();
|
|
|
|
for (const range of att.splitRanges) {
|
|
const start = Math.max(1, range.start);
|
|
const end = Math.min(range.end, totalPages);
|
|
|
|
if (start > end) {
|
|
this.logger.warn(`Ungültiger Bereich für Splitting: ${start}-${end} (Seiten gesamt: ${totalPages})`);
|
|
continue;
|
|
}
|
|
|
|
const newPdf = await PDFDocument.create();
|
|
// Pages are 0-indexed in pdf-lib
|
|
const pageIndices = Array.from({ length: end - start + 1 }, (_, i) => start - 1 + i);
|
|
const copiedPages = await newPdf.copyPages(pdfDoc, pageIndices);
|
|
copiedPages.forEach((p) => newPdf.addPage(p));
|
|
|
|
const splitPdfBytes = await newPdf.save();
|
|
const tempFilePath = path.join(tempDir, `${baseFilename}_${start}-${end}.pdf`);
|
|
await fs.writeFile(tempFilePath, Buffer.from(splitPdfBytes));
|
|
|
|
uploadPromises.push({
|
|
path: tempFilePath,
|
|
filename: `${baseFilename}_${start}-${end}`,
|
|
rangeKey: `${start}-${end}`,
|
|
});
|
|
}
|
|
} else {
|
|
// Process Full Attachment
|
|
const tempFilePath = path.join(tempDir, `${baseFilename}.pdf`);
|
|
await fs.writeFile(tempFilePath, originalPdfBytes);
|
|
uploadPromises.push({
|
|
path: tempFilePath,
|
|
filename: baseFilename,
|
|
rangeKey: 'full',
|
|
});
|
|
}
|
|
|
|
// 0. Check if ASN already exists
|
|
if (att.belegnummer) {
|
|
await this.paperlessService.validateAsnNotExists(att.belegnummer);
|
|
}
|
|
|
|
// Upload all generated PDFs
|
|
for (const uploadItem of uploadPromises) {
|
|
const options: any = {
|
|
filename: uploadItem.filename,
|
|
title: att.belegnummer ? `Beleg ${att.belegnummer}` : uploadItem.filename,
|
|
created: createdDate,
|
|
owner: null,
|
|
};
|
|
if (att.paperlessCorrespondentId) options.correspondent = att.paperlessCorrespondentId;
|
|
|
|
const paperlessTaskId = await this.paperlessService.uploadDocument(uploadItem.path, options);
|
|
|
|
// Create background task for enrichment (same logic as Inbox)
|
|
const backgroundTask = this.taskRepo.create({
|
|
TaskId: paperlessTaskId,
|
|
InterneBelegnummer: att.belegnummer || '',
|
|
Eingangsdatum: att.barcode?.datum ? new Date(att.barcode.datum) : createdDate,
|
|
Belegdatum: createdDate,
|
|
BarcodeJson: att.barcode ? JSON.stringify(att.barcode) : null,
|
|
BetriebID: null, // Owner
|
|
Fertig: 0,
|
|
DocumentType: att.type === 'MAIN' ? null : 5, // 5 = Anlage
|
|
SourceAttachmentID: att.attachmentId,
|
|
SourceAttachmentRange: uploadItem.rangeKey,
|
|
});
|
|
await this.taskRepo.save(backgroundTask);
|
|
|
|
// Still poll for Doc ID so we can return it to the frontend for immediate preview
|
|
let docId = null;
|
|
for (let i = 0; i < 30; i++) {
|
|
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
try {
|
|
const taskStatus = await this.paperlessService.getTask(paperlessTaskId);
|
|
// Paperless returns { results: [ ... ] } for filtered tasks
|
|
const statusObj = taskStatus.results ? taskStatus.results[0] : (Array.isArray(taskStatus) ? taskStatus[0] : taskStatus);
|
|
if (statusObj && statusObj.related_document) {
|
|
docId = statusObj.related_document;
|
|
break;
|
|
}
|
|
} catch(e) {}
|
|
}
|
|
|
|
if (docId) {
|
|
paperlessIds[uploadItem.rangeKey] = docId;
|
|
}
|
|
}
|
|
|
|
// Update Database
|
|
attachmentEntity.PaperlessDocumentIds = paperlessIds;
|
|
attachmentEntity.ImportStatus = 1;
|
|
if (att.belegnummer) {
|
|
attachmentEntity.InterneBelegnummer = att.belegnummer;
|
|
}
|
|
await this.attachmentRepo.save(attachmentEntity);
|
|
|
|
// Confirm Belegnummer if used
|
|
if (att.belegnummer && att.barcode?.nummer) {
|
|
await this.setBelegnummer(data.emailDate, att.barcode.nummer).catch(e => this.logger.warn(`Failed to set Belegnummer: ${e.message}`));
|
|
}
|
|
|
|
results.push({ attachmentId: att.attachmentId, paperlessIds });
|
|
}
|
|
|
|
// Mark Email as processed (Status = 1)
|
|
if (data.attachments.length > 0) {
|
|
const firstAtt = await this.attachmentRepo.findOne({
|
|
where: { Id: data.attachments[0].attachmentId }
|
|
});
|
|
if (firstAtt) {
|
|
await this.emailRepo.update(firstAtt.EmailMessageId, { Status: 1 });
|
|
this.logger.log(`Email ${firstAtt.EmailMessageId} als verarbeitet markiert.`);
|
|
}
|
|
}
|
|
|
|
return { success: true, results };
|
|
} finally {
|
|
// Clean up temp dir
|
|
await fs.rm(tempDir, { recursive: true, force: true }).catch(() => {});
|
|
}
|
|
}
|
|
}
|