fix(mail): Duplikate auch bei neu erzeugten PDFs erkennen
Build and Push Multi-Platform Images / build-and-push (push) Successful in 45s
Build and Push Multi-Platform Images / build-and-push (push) Successful in 45s
Absender erzeugen PDFs beim erneuten Versand oft neu (andere Erstellungszeit/Dokument-ID), dadurch unterscheidet sich die Checksumme und die Duplikatprüfung griff nicht (z.B. Mail 7071 vs. 7072). - Neuer Text-Fingerprint (SHA-256 über normalisierten PDF-Text via pdf-parse) in Attachments.ContentFingerprint, inkl. Migration und Index - Wird beim Mail-Abruf berechnet; Altbestand füllt ein Hintergrund- Backfill beim Start auf, fehlende Werte werden bei Bedarf nachberechnet - Duplikatprüfung und Import-Sperre berücksichtigen Anhänge mit gleichem Inhalt, die importiert sind oder gerade importiert werden - Jest braucht für pdf-parse --experimental-vm-modules (test-Skripte) Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
534b313054
commit
797cbb9b71
12 files changed
+746
-7
No files matched your search
@@ -20,6 +20,7 @@ import { deriveTaskMetadata } from '../paperless/task-metadata.util';
|
||||
import * as QRCode from 'qrcode';
|
||||
import { EmailPageCacheService } from './email-page-cache.service';
|
||||
import { ImapFolderService } from './imap-folder.service';
|
||||
import { AttachmentFingerprintService } from './attachment-fingerprint.service';
|
||||
import { PdfService } from '../preprocessing/pdf.service';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
@@ -71,6 +72,7 @@ export class EmailImportService {
|
||||
private readonly pdfService: PdfService,
|
||||
private readonly pageCache: EmailPageCacheService,
|
||||
private readonly imapFolderService: ImapFolderService,
|
||||
private readonly fingerprints: AttachmentFingerprintService,
|
||||
) {}
|
||||
|
||||
async ensurePreviews(emailId: number): Promise<void> {
|
||||
@@ -263,8 +265,14 @@ export class EmailImportService {
|
||||
if (!attachment) return false;
|
||||
|
||||
if (!pages) {
|
||||
if (
|
||||
attachment.Checksum &&
|
||||
(await this.hasPendingImport(attachment.Checksum, 'full'))
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
if (await this.hasTwinByContent(attachment, 'full')) return true;
|
||||
if (!attachment.Checksum) return false;
|
||||
if (await this.hasPendingImport(attachment.Checksum, 'full')) return true;
|
||||
return this.paperlessService.checksumExists(attachment.Checksum);
|
||||
}
|
||||
|
||||
@@ -287,6 +295,9 @@ export class EmailImportService {
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
if (await this.hasTwinByContent(attachment, `${start}-${end}`)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const sliced = await PDFDocument.create();
|
||||
const indices = Array.from(
|
||||
@@ -322,6 +333,19 @@ export class EmailImportService {
|
||||
return count > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gleicher Inhalt, andere Datei: Absender erzeugen PDFs beim erneuten
|
||||
* Versand oft neu (andere Checksumme). Der Text-Fingerprint erkennt das.
|
||||
*/
|
||||
private async hasTwinByContent(
|
||||
attachment: Attachment,
|
||||
range: string,
|
||||
): Promise<boolean> {
|
||||
const fingerprint = await this.fingerprints.ensure(attachment);
|
||||
if (!fingerprint) return false;
|
||||
return this.fingerprints.hasImportedTwin(attachment.Id, fingerprint, range);
|
||||
}
|
||||
|
||||
// --- Print Preview ---
|
||||
async generatePrintPdf(
|
||||
attachmentId: number,
|
||||
@@ -570,7 +594,9 @@ export class EmailImportService {
|
||||
const ranges = att.splitRanges?.length ? att.splitRanges : [undefined];
|
||||
for (const range of ranges) {
|
||||
const bereich = range ? `${range.start}-${range.end}` : 'full';
|
||||
const key = `${attachment.Checksum || `id:${attachment.Id}`}:${bereich}`;
|
||||
// Fingerprint vor Checksumme: fängt auch neu erzeugte PDFs ab
|
||||
const fingerprint = await this.fingerprints.ensure(attachment);
|
||||
const key = `${fingerprint || attachment.Checksum || `id:${attachment.Id}`}:${bereich}`;
|
||||
const bezeichnung = range
|
||||
? `„${attachment.FileName}" (Seiten ${bereich})`
|
||||
: `„${attachment.FileName}"`;
|
||||
|
||||
Reference in new issue
Block a user