From 797cbb9b71a1cca9bcee621e0341eb9674b6bd1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20P=C3=B6ttker?= Date: Fri, 25 Sep 2026 12:27:31 +0200 Subject: [PATCH] fix(mail): Duplikate auch bei neu erzeugten PDFs erkennen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Absender erzeugen PDFs beim erneuten Versand oft neu (andere Erstellungszeit/Dokument-ID), dadurch unterscheidet sich die Checksumme und die Duplikatprüfung griff nicht (z.B. Mail 7071 vs. 7072). - Neuer Text-Fingerprint (SHA-256 über normalisierten PDF-Text via pdf-parse) in Attachments.ContentFingerprint, inkl. Migration und Index - Wird beim Mail-Abruf berechnet; Altbestand füllt ein Hintergrund- Backfill beim Start auf, fehlende Werte werden bei Bedarf nachberechnet - Duplikatprüfung und Import-Sperre berücksichtigen Anhänge mit gleichem Inhalt, die importiert sind oder gerade importiert werden - Jest braucht für pdf-parse --experimental-vm-modules (test-Skripte) Co-Authored-By: Claude Opus 5.5 (1M context) --- paperless-backend/package-lock.json | 217 ++++++++++++++++++ paperless-backend/package.json | 9 +- .../database/entities/attachment.entity.ts | 9 + ...0000000000-AttachmentContentFingerprint.ts | 31 +++ .../email-download/email-download.service.ts | 5 + .../attachment-fingerprint.service.spec.ts | 157 +++++++++++++ .../email/attachment-fingerprint.service.ts | 124 ++++++++++ .../src/email/email-import.service.spec.ts | 68 ++++++ .../src/email/email-import.service.ts | 30 ++- paperless-backend/src/email/email.module.ts | 8 +- .../src/email/pdf-fingerprint.util.spec.ts | 58 +++++ .../src/email/pdf-fingerprint.util.ts | 37 +++ 12 files changed, 746 insertions(+), 7 deletions(-) create mode 100644 paperless-backend/src/database/migrations/1790000000000-AttachmentContentFingerprint.ts create mode 100644 paperless-backend/src/email/attachment-fingerprint.service.spec.ts create mode 100644 paperless-backend/src/email/attachment-fingerprint.service.ts create mode 100644 paperless-backend/src/email/pdf-fingerprint.util.spec.ts create mode 100644 paperless-backend/src/email/pdf-fingerprint.util.ts diff --git a/paperless-backend/package-lock.json b/paperless-backend/package-lock.json index 92ceec8..017e999 100644 --- a/paperless-backend/package-lock.json +++ b/paperless-backend/package-lock.json @@ -36,6 +36,7 @@ "passport": "^0.7.0", "passport-jwt": "^4.0.1", "pdf-lib": "^1.17.1", + "pdf-parse": "^2.4.5", "qrcode": "^1.5.4", "reflect-metadata": "^0.2.2", "rxjs": "^7.8.1", @@ -2740,6 +2741,190 @@ "node": ">=8" } }, + "node_modules/@napi-rs/canvas": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.80.tgz", + "integrity": "sha512-DxuT1ClnIPts1kQx8FBmkk4BQDTfI5kIzywAaMjQSXfNnra5UFU9PwurXrl+Je3bJ6BGsp/zmshVVFbCmyI+ww==", + "license": "MIT", + "workspaces": [ + "e2e/*" + ], + "engines": { + "node": ">= 10" + }, + "optionalDependencies": { + "@napi-rs/canvas-android-arm64": "0.1.80", + "@napi-rs/canvas-darwin-arm64": "0.1.80", + "@napi-rs/canvas-darwin-x64": "0.1.80", + "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.80", + "@napi-rs/canvas-linux-arm64-gnu": "0.1.80", + "@napi-rs/canvas-linux-arm64-musl": "0.1.80", + "@napi-rs/canvas-linux-riscv64-gnu": "0.1.80", + "@napi-rs/canvas-linux-x64-gnu": "0.1.80", + "@napi-rs/canvas-linux-x64-musl": "0.1.80", + "@napi-rs/canvas-win32-x64-msvc": "0.1.80" + } + }, + "node_modules/@napi-rs/canvas-android-arm64": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.80.tgz", + "integrity": "sha512-sk7xhN/MoXeuExlggf91pNziBxLPVUqF2CAVnB57KLG/pz7+U5TKG8eXdc3pm0d7Od0WreB6ZKLj37sX9muGOQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-darwin-arm64": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.80.tgz", + "integrity": "sha512-O64APRTXRUiAz0P8gErkfEr3lipLJgM6pjATwavZ22ebhjYl/SUbpgM0xcWPQBNMP1n29afAC/Us5PX1vg+JNQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-darwin-x64": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.80.tgz", + "integrity": "sha512-FqqSU7qFce0Cp3pwnTjVkKjjOtxMqRe6lmINxpIZYaZNnVI0H5FtsaraZJ36SiTHNjZlUB69/HhxNDT1Aaa9vA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.80.tgz", + "integrity": "sha512-eyWz0ddBDQc7/JbAtY4OtZ5SpK8tR4JsCYEZjCE3dI8pqoWUC8oMwYSBGCYfsx2w47cQgQCgMVRVTFiiO38hHQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-gnu": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.80.tgz", + "integrity": "sha512-qwA63t8A86bnxhuA/GwOkK3jvb+XTQaTiVML0vAWoHyoZYTjNs7BzoOONDgTnNtr8/yHrq64XXzUoLqDzU+Uuw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-musl": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.80.tgz", + "integrity": "sha512-1XbCOz/ymhj24lFaIXtWnwv/6eFHXDrjP0jYkc6iHQ9q8oXKzUX1Lc6bu+wuGiLhGh2GS/2JlfORC5ZcXimRcg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-riscv64-gnu": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.80.tgz", + "integrity": "sha512-XTzR125w5ZMs0lJcxRlS1K3P5RaZ9RmUsPtd1uGt+EfDyYMu4c6SEROYsxyatbbu/2+lPe7MPHOO/0a0x7L/gw==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-gnu": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.80.tgz", + "integrity": "sha512-BeXAmhKg1kX3UCrJsYbdQd3hIMDH/K6HnP/pG2LuITaXhXBiNdh//TVVVVCBbJzVQaV5gK/4ZOCMrQW9mvuTqA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-musl": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.80.tgz", + "integrity": "sha512-x0XvZWdHbkgdgucJsRxprX/4o4sEed7qo9rCQA9ugiS9qE2QvP0RIiEugtZhfLH3cyI+jIRFJHV4Fuz+1BHHMg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-win32-x64-msvc": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.80.tgz", + "integrity": "sha512-Z8jPsM6df5V8B1HrCHB05+bDiCxjE9QA//3YrkKIdVDEwn5RKaqOxCJDRJkl48cJbylcrJbW4HxZbTte8juuPg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, "node_modules/@napi-rs/wasm-runtime": { "version": "0.2.12", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.12.tgz", @@ -10261,6 +10446,38 @@ "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", "license": "0BSD" }, + "node_modules/pdf-parse": { + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/pdf-parse/-/pdf-parse-2.4.5.tgz", + "integrity": "sha512-mHU89HGh7v+4u2ubfnevJ03lmPgQ5WU4CxAVmTSh/sxVTEDYd1er/dKS/A6vg77NX47KTEoihq8jZBLr8Cxuwg==", + "license": "Apache-2.0", + "dependencies": { + "@napi-rs/canvas": "0.1.80", + "pdfjs-dist": "5.4.296" + }, + "bin": { + "pdf-parse": "bin/cli.mjs" + }, + "engines": { + "node": ">=20.16.0 <21 || >=22.3.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/mehmet-kozan" + } + }, + "node_modules/pdfjs-dist": { + "version": "5.4.296", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-5.4.296.tgz", + "integrity": "sha512-DlOzet0HO7OEnmUmB6wWGJrrdvbyJKftI1bhMitK7O2N8W2gc757yyYBbINy9IDafXAV9wmKr9t7xsTaNKRG5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=20.16.0 || >=22.3.0" + }, + "optionalDependencies": { + "@napi-rs/canvas": "^0.1.80" + } + }, "node_modules/peberminta": { "version": "0.9.0", "resolved": "https://registry.npmjs.org/peberminta/-/peberminta-0.9.0.tgz", diff --git a/paperless-backend/package.json b/paperless-backend/package.json index 6251a33..811595c 100644 --- a/paperless-backend/package.json +++ b/paperless-backend/package.json @@ -13,11 +13,11 @@ "start:debug": "nest start --debug --watch", "start:prod": "node dist/main", "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix", - "test": "jest", - "test:watch": "jest --watch", - "test:cov": "jest --coverage", + "test": "NODE_OPTIONS=--experimental-vm-modules jest", + "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch", + "test:cov": "NODE_OPTIONS=--experimental-vm-modules jest --coverage", "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", - "test:e2e": "jest --config ./test/jest-e2e.json", + "test:e2e": "NODE_OPTIONS=--experimental-vm-modules jest --config ./test/jest-e2e.json", "typeorm": "typeorm-ts-node-commonjs -d ./src/database/data-source.ts", "migration:generate": "npm run typeorm -- migration:generate", "migration:run": "npm run typeorm -- migration:run", @@ -51,6 +51,7 @@ "passport": "^0.7.0", "passport-jwt": "^4.0.1", "pdf-lib": "^1.17.1", + "pdf-parse": "^2.4.5", "qrcode": "^1.5.4", "reflect-metadata": "^0.2.2", "rxjs": "^7.8.1", diff --git a/paperless-backend/src/database/entities/attachment.entity.ts b/paperless-backend/src/database/entities/attachment.entity.ts index 6f3b084..646371c 100644 --- a/paperless-backend/src/database/entities/attachment.entity.ts +++ b/paperless-backend/src/database/entities/attachment.entity.ts @@ -31,6 +31,15 @@ export class Attachment { @Column({ type: 'varchar', length: 32, default: '' }) Checksum!: string; + /** + * SHA-256 über den normalisierten PDF-Text (siehe pdf-fingerprint.util). + * Erkennt Duplikate, deren Datei neu erzeugt wurde (andere Checksumme). + * `''` = berechnet, aber ohne Textebene; `null` = noch nicht berechnet. + */ + @Index('IX_Attachments_ContentFingerprint') + @Column({ type: 'varchar', length: 64, nullable: true }) + ContentFingerprint!: string | null; + @Column({ type: 'varchar', length: 255, nullable: true }) ContentId!: string | null; diff --git a/paperless-backend/src/database/migrations/1790000000000-AttachmentContentFingerprint.ts b/paperless-backend/src/database/migrations/1790000000000-AttachmentContentFingerprint.ts new file mode 100644 index 0000000..be6576f --- /dev/null +++ b/paperless-backend/src/database/migrations/1790000000000-AttachmentContentFingerprint.ts @@ -0,0 +1,31 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +/** + * Inhaltsbasierter Fingerprint für Mail-Anhänge: erkennt Duplikate, deren + * PDF vom Absender neu erzeugt wurde und daher eine andere Checksumme hat. + * Bestehende Anhänge werden beim Start im Hintergrund nachberechnet + * (AttachmentFingerprintService). + */ +export class AttachmentContentFingerprint1790000000000 implements MigrationInterface { + name = 'AttachmentContentFingerprint1790000000000'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + ALTER TABLE \`Attachments\` + ADD \`ContentFingerprint\` varchar(64) NULL DEFAULT NULL + `); + await queryRunner.query(` + CREATE INDEX \`IX_Attachments_ContentFingerprint\` + ON \`Attachments\` (\`ContentFingerprint\`) + `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `DROP INDEX \`IX_Attachments_ContentFingerprint\` ON \`Attachments\``, + ); + await queryRunner.query( + `ALTER TABLE \`Attachments\` DROP COLUMN \`ContentFingerprint\``, + ); + } +} diff --git a/paperless-backend/src/email-download/email-download.service.ts b/paperless-backend/src/email-download/email-download.service.ts index 561a311..ebdc010 100644 --- a/paperless-backend/src/email-download/email-download.service.ts +++ b/paperless-backend/src/email-download/email-download.service.ts @@ -19,6 +19,7 @@ import { Email } from '../database/entities/email.entity'; import { Attachment } from '../database/entities/attachment.entity'; import { Content } from '../database/entities/content.entity'; import { isERechnung } from './zugferd.util'; +import { computeContentFingerprint } from '../email/pdf-fingerprint.util'; function sanitizeFilename(name: string): string { return name @@ -304,6 +305,10 @@ export class EmailDownloadService { attachment.IsEmbedded = isEmbedded; attachment.ContentId = att.cid ? att.cid.slice(0, 255) : null; attachment.Checksum = crypto.createHash('md5').update(buffer).digest('hex'); + attachment.ContentFingerprint = + contentType.toLowerCase() === 'application/pdf' + ? await computeContentFingerprint(buffer) + : null; attachment.Erechnung = contentType.toLowerCase() === 'application/pdf' ? isERechnung(buffer) diff --git a/paperless-backend/src/email/attachment-fingerprint.service.spec.ts b/paperless-backend/src/email/attachment-fingerprint.service.spec.ts new file mode 100644 index 0000000..20bad9e --- /dev/null +++ b/paperless-backend/src/email/attachment-fingerprint.service.spec.ts @@ -0,0 +1,157 @@ +import { Test } from '@nestjs/testing'; +import { getRepositoryToken } from '@nestjs/typeorm'; +import { Brackets } from 'typeorm'; +import { PDFDocument, StandardFonts } from 'pdf-lib'; +import { AttachmentFingerprintService } from './attachment-fingerprint.service'; +import { Attachment } from '../database/entities/attachment.entity'; +import { Content } from '../database/entities/content.entity'; + +async function pdfMitText(text: string): Promise { + const doc = await PDFDocument.create(); + const font = await doc.embedFont(StandardFonts.Helvetica); + doc.addPage().drawText(text, { x: 50, y: 700, font, size: 12 }); + return Buffer.from(await doc.save()); +} + +const pdfAnhang = (felder: Partial = {}) => + ({ + Id: 7, + ContentType: 'application/pdf', + ContentFingerprint: null, + ...felder, + }) as Attachment; + +describe('AttachmentFingerprintService', () => { + let service: AttachmentFingerprintService; + let query: Record; + let attachmentRepo: { update: jest.Mock; createQueryBuilder: jest.Mock }; + let contentRepo: { findOne: jest.Mock }; + + beforeEach(async () => { + query = { + select: jest.fn().mockReturnThis(), + where: jest.fn().mockReturnThis(), + andWhere: jest.fn().mockReturnThis(), + orderBy: jest.fn().mockReturnThis(), + take: jest.fn().mockReturnThis(), + getMany: jest.fn().mockResolvedValue([]), + getCount: jest.fn().mockResolvedValue(0), + }; + attachmentRepo = { + update: jest.fn(), + createQueryBuilder: jest.fn().mockReturnValue(query), + }; + contentRepo = { + findOne: jest.fn().mockResolvedValue({ + Content1: await pdfMitText('Lastschriftankuendigung Mandat 0000844118'), + }), + }; + + const module = await Test.createTestingModule({ + providers: [ + AttachmentFingerprintService, + { provide: getRepositoryToken(Attachment), useValue: attachmentRepo }, + { provide: getRepositoryToken(Content), useValue: contentRepo }, + ], + }).compile(); + service = module.get(AttachmentFingerprintService); + }); + + describe('ensure', () => { + it('berechnet einen fehlenden Fingerprint und speichert ihn', async () => { + const anhang = pdfAnhang(); + + const fp = await service.ensure(anhang); + + expect(fp).toMatch(/^[0-9a-f]{64}$/); + expect(attachmentRepo.update).toHaveBeenCalledWith(7, { + ContentFingerprint: fp, + }); + expect(anhang.ContentFingerprint).toBe(fp); + }); + + it('nutzt einen vorhandenen Fingerprint ohne erneute Berechnung', async () => { + await expect( + service.ensure(pdfAnhang({ ContentFingerprint: 'abc' })), + ).resolves.toBe('abc'); + expect(contentRepo.findOne).not.toHaveBeenCalled(); + }); + + it('liefert null für PDFs ohne Textebene', async () => { + await expect( + service.ensure(pdfAnhang({ ContentFingerprint: '' })), + ).resolves.toBeNull(); + }); + + it('ignoriert Anhänge, die kein PDF sind', async () => { + await expect( + service.ensure(pdfAnhang({ ContentType: 'message/rfc822' })), + ).resolves.toBeNull(); + expect(contentRepo.findOne).not.toHaveBeenCalled(); + }); + }); + + describe('hasImportedTwin', () => { + const bedingungen = (range: string) => { + const brackets = query.andWhere.mock.calls + .map(([arg]) => arg as unknown) + .find((arg): arg is Brackets => arg instanceof Brackets)!; + const inner = { + where: jest.fn().mockReturnThis(), + orWhere: jest.fn().mockReturnThis(), + }; + brackets.whereFactory(inner as never); + return { inner, range }; + }; + + it('sucht andere Anhänge mit gleichem Fingerprint', async () => { + query.getCount.mockResolvedValue(1); + + await expect(service.hasImportedTwin(7, 'fp', 'full')).resolves.toBe( + true, + ); + expect(query.where).toHaveBeenCalledWith( + 'a.ContentFingerprint = :fingerprint', + { fingerprint: 'fp' }, + ); + expect(query.andWhere).toHaveBeenCalledWith('a.Id != :attachmentId', { + attachmentId: 7, + }); + }); + + it('zählt beim ganzen Anhang jeden abgeschlossenen Import', async () => { + await service.hasImportedTwin(7, 'fp', 'full'); + const { inner } = bedingungen('full'); + + expect(inner.where).toHaveBeenCalledWith(expect.any(String), { + jsonPath: '$."full"', + }); + expect(inner.orWhere).toHaveBeenCalledWith('a.ImportStatus = 1'); + }); + + it('verlangt bei Teilbereichen denselben Seitenbereich', async () => { + await service.hasImportedTwin(7, 'fp', '2-3'); + const { inner } = bedingungen('2-3'); + + expect(inner.where).toHaveBeenCalledWith(expect.any(String), { + jsonPath: '$."2-3"', + }); + expect(inner.orWhere).toHaveBeenCalledWith(expect.any(String), { + range: '2-3', + }); + expect(inner.orWhere).not.toHaveBeenCalledWith('a.ImportStatus = 1'); + }); + }); + + it('backfill berechnet fehlende Fingerprints portionsweise', async () => { + query.getMany + .mockResolvedValueOnce([pdfAnhang({ Id: 1 }), pdfAnhang({ Id: 2 })]) + .mockResolvedValueOnce([]); + + await expect(service.backfill(2)).resolves.toBe(2); + expect(attachmentRepo.update).toHaveBeenCalledTimes(2); + expect(query.andWhere).toHaveBeenLastCalledWith('a.Id > :lastId', { + lastId: 2, + }); + }); +}); diff --git a/paperless-backend/src/email/attachment-fingerprint.service.ts b/paperless-backend/src/email/attachment-fingerprint.service.ts new file mode 100644 index 0000000..45aab44 --- /dev/null +++ b/paperless-backend/src/email/attachment-fingerprint.service.ts @@ -0,0 +1,124 @@ +import { Injectable, Logger, OnApplicationBootstrap } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Brackets, Repository } from 'typeorm'; +import { Attachment } from '../database/entities/attachment.entity'; +import { Content } from '../database/entities/content.entity'; +import { computeContentFingerprint } from './pdf-fingerprint.util'; + +/** + * Verwaltet den inhaltsbasierten Fingerprint der Mail-Anhänge: berechnet ihn + * bei Bedarf nach, füllt ihn für Altbestände im Hintergrund auf und findet + * bereits importierte Anhänge mit gleichem Inhalt. + */ +@Injectable() +export class AttachmentFingerprintService implements OnApplicationBootstrap { + private readonly logger = new Logger(AttachmentFingerprintService.name); + + constructor( + @InjectRepository(Attachment) + private readonly attachmentRepo: Repository, + @InjectRepository(Content) + private readonly contentRepo: Repository, + ) {} + + onApplicationBootstrap(): void { + // Nicht auf den Backfill warten – der Start soll nicht blockieren + this.backfill().catch((err: Error) => + this.logger.error(`Fingerprint-Backfill fehlgeschlagen: ${err.message}`), + ); + } + + /** + * Liefert den Fingerprint eines Anhangs und berechnet ihn nach, falls er + * noch fehlt (z.B. Mail vor dem Backfill eingegangen). `null`, wenn kein + * vergleichbarer Text vorliegt. + */ + async ensure(attachment: Attachment): Promise { + if (attachment.ContentFingerprint != null) { + return attachment.ContentFingerprint || null; + } + if (attachment.ContentType !== 'application/pdf') return null; + + const content = await this.contentRepo.findOne({ + where: { AttachmentEntityId: attachment.Id }, + }); + if (!content) return null; + + const fingerprint = await computeContentFingerprint(content.Content1); + if (fingerprint === null) return null; + await this.attachmentRepo.update(attachment.Id, { + ContentFingerprint: fingerprint, + }); + attachment.ContentFingerprint = fingerprint; + return fingerprint || null; + } + + /** + * Gibt es einen anderen Anhang mit gleichem Inhalt, dessen Seitenbereich + * bereits importiert wurde oder gerade importiert wird? + */ + async hasImportedTwin( + attachmentId: number, + fingerprint: string, + range: string, + ): Promise { + const count = await this.attachmentRepo + .createQueryBuilder('a') + .where('a.ContentFingerprint = :fingerprint', { fingerprint }) + .andWhere('a.Id != :attachmentId', { attachmentId }) + .andWhere( + new Brackets((qb) => { + // Abgeschlossener Import mit ermittelter Paperless-ID + qb.where( + "JSON_CONTAINS_PATH(a.PaperlessDocumentIds, 'one', :jsonPath) = 1", + { jsonPath: `$."${range}"` }, + ) + // Laufender oder abgeschlossener Upload (fehlgeschlagene Tasks + // löscht der Task-Processor) + .orWhere( + 'EXISTS (SELECT 1 FROM tasks t WHERE t.SourceAttachmentID = a.Id AND t.SourceAttachmentRange = :range)', + { range }, + ); + // Ganzer Anhang: jeder Import zählt, auch wenn die ID beim Upload + // nicht ermittelt werden konnte + if (range === 'full') qb.orWhere('a.ImportStatus = 1'); + }), + ) + .getCount(); + return count > 0; + } + + /** Berechnet fehlende Fingerprints aller PDF-Anhänge in kleinen Portionen. */ + async backfill(batchSize = 25): Promise { + let lastId = 0; + let updated = 0; + for (;;) { + const batch = await this.attachmentRepo + .createQueryBuilder('a') + .select(['a.Id', 'a.ContentType', 'a.ContentFingerprint']) + .where('a.ContentFingerprint IS NULL') + .andWhere("a.ContentType = 'application/pdf'") + .andWhere('a.Id > :lastId', { lastId }) + .orderBy('a.Id', 'ASC') + .take(batchSize) + .getMany(); + if (batch.length === 0) break; + + for (const attachment of batch) { + // lastId weiterschieben, damit unlesbare PDFs nicht endlos drankommen + lastId = attachment.Id; + try { + if ((await this.ensure(attachment)) !== null) updated++; + } catch (err) { + this.logger.warn( + `Fingerprint für Anhang ${attachment.Id} nicht berechenbar: ${(err as Error).message}`, + ); + } + } + } + if (updated > 0) { + this.logger.log(`Fingerprint-Backfill: ${updated} Anhänge ergänzt.`); + } + return updated; + } +} diff --git a/paperless-backend/src/email/email-import.service.spec.ts b/paperless-backend/src/email/email-import.service.spec.ts index 8474cbf..6723247 100644 --- a/paperless-backend/src/email/email-import.service.spec.ts +++ b/paperless-backend/src/email/email-import.service.spec.ts @@ -13,6 +13,7 @@ import { PaperlessService } from '../paperless/paperless.service'; import { PdfService } from '../preprocessing/pdf.service'; import { EmailPageCacheService } from './email-page-cache.service'; import { ImapFolderService } from './imap-folder.service'; +import { AttachmentFingerprintService } from './attachment-fingerprint.service'; async function pdfMitSeiten(anzahl: number): Promise { const doc = await PDFDocument.create(); @@ -34,6 +35,7 @@ describe('EmailImportService – Duplikatprüfung', () => { checksumExists: jest.Mock; uploadDocument: jest.Mock; }; + let fingerprints: { ensure: jest.Mock; hasImportedTwin: jest.Mock }; beforeEach(async () => { taskQuery = { @@ -54,6 +56,10 @@ describe('EmailImportService – Duplikatprüfung', () => { checksumExists: jest.fn().mockResolvedValue(false), uploadDocument: jest.fn(), }; + fingerprints = { + ensure: jest.fn().mockResolvedValue(null), + hasImportedTwin: jest.fn().mockResolvedValue(false), + }; const module: TestingModule = await Test.createTestingModule({ providers: [ @@ -76,6 +82,7 @@ describe('EmailImportService – Duplikatprüfung', () => { { provide: PdfService, useValue: {} }, { provide: EmailPageCacheService, useValue: {} }, { provide: ImapFolderService, useValue: {} }, + { provide: AttachmentFingerprintService, useValue: fingerprints }, ], }).compile(); @@ -109,6 +116,20 @@ describe('EmailImportService – Duplikatprüfung', () => { await expect(service.checkDuplicate(7)).resolves.toBe(false); }); + it('erkennt neu erzeugte PDFs mit gleichem Inhalt über den Fingerprint', async () => { + fingerprints.ensure.mockResolvedValue('fp-inhalt'); + fingerprints.hasImportedTwin.mockResolvedValue(true); + + await expect(service.checkDuplicate(7)).resolves.toBe(true); + + expect(fingerprints.hasImportedTwin).toHaveBeenCalledWith( + 7, + 'fp-inhalt', + 'full', + ); + expect(paperlessService.checksumExists).not.toHaveBeenCalled(); + }); + it('meldet kein Duplikat für Anhänge ohne Checksumme', async () => { attachmentRepo.findOne.mockResolvedValue({ Id: 7, Checksum: '' }); @@ -142,6 +163,20 @@ describe('EmailImportService – Duplikatprüfung', () => { ); }); + it('vergleicht den Fingerprint mit demselben Seitenbereich', async () => { + fingerprints.ensure.mockResolvedValue('fp-inhalt'); + fingerprints.hasImportedTwin.mockResolvedValue(true); + + await expect( + service.checkDuplicate(7, { start: 2, end: 999 }), + ).resolves.toBe(true); + expect(fingerprints.hasImportedTwin).toHaveBeenCalledWith( + 7, + 'fp-inhalt', + '2-3', + ); + }); + it('prüft in Paperless die Checksumme des Teil-PDFs', async () => { await expect( service.checkDuplicate(7, { start: 1, end: 1 }), @@ -205,6 +240,39 @@ describe('EmailImportService – Duplikatprüfung', () => { expect(check).not.toHaveBeenCalled(); }); + it('sperrt auch parallele Importe verschiedener Dateien mit gleichem Inhalt', async () => { + fingerprints.ensure.mockResolvedValue('fp-inhalt'); + attachmentRepo.findOne.mockImplementation( + ({ where: { Id } }: { where: { Id: number } }) => + Promise.resolve({ + Id, + Checksum: `checksum-${Id}`, + FileName: 'a.pdf', + }), + ); + let ersterFertig!: () => void; + jest + .spyOn( + service as never as { runImport: () => Promise }, + 'runImport', + ) + .mockImplementationOnce( + () => new Promise((resolve) => (ersterFertig = () => resolve({}))), + ); + + const erster = service.executeImport(importDaten()); + await new Promise((r) => setImmediate(r)); + + const zweiter = importDaten(); + zweiter.attachments[0] = { ...zweiter.attachments[0], attachmentId: 8 }; + await expect(service.executeImport(zweiter)).rejects.toThrow( + ConflictException, + ); + + ersterFertig(); + await erster; + }); + it('lässt denselben Anhang nicht zweimal gleichzeitig importieren', async () => { let ersterFertig!: () => void; const run = jest diff --git a/paperless-backend/src/email/email-import.service.ts b/paperless-backend/src/email/email-import.service.ts index c1cc0ef..34fab64 100644 --- a/paperless-backend/src/email/email-import.service.ts +++ b/paperless-backend/src/email/email-import.service.ts @@ -20,6 +20,7 @@ import { deriveTaskMetadata } from '../paperless/task-metadata.util'; import * as QRCode from 'qrcode'; import { EmailPageCacheService } from './email-page-cache.service'; import { ImapFolderService } from './imap-folder.service'; +import { AttachmentFingerprintService } from './attachment-fingerprint.service'; import { PdfService } from '../preprocessing/pdf.service'; import * as path from 'path'; import * as os from 'os'; @@ -71,6 +72,7 @@ export class EmailImportService { private readonly pdfService: PdfService, private readonly pageCache: EmailPageCacheService, private readonly imapFolderService: ImapFolderService, + private readonly fingerprints: AttachmentFingerprintService, ) {} async ensurePreviews(emailId: number): Promise { @@ -263,8 +265,14 @@ export class EmailImportService { if (!attachment) return false; if (!pages) { + if ( + attachment.Checksum && + (await this.hasPendingImport(attachment.Checksum, 'full')) + ) { + return true; + } + if (await this.hasTwinByContent(attachment, 'full')) return true; if (!attachment.Checksum) return false; - if (await this.hasPendingImport(attachment.Checksum, 'full')) return true; return this.paperlessService.checksumExists(attachment.Checksum); } @@ -287,6 +295,9 @@ export class EmailImportService { ) { return true; } + if (await this.hasTwinByContent(attachment, `${start}-${end}`)) { + return true; + } const sliced = await PDFDocument.create(); const indices = Array.from( @@ -322,6 +333,19 @@ export class EmailImportService { return count > 0; } + /** + * Gleicher Inhalt, andere Datei: Absender erzeugen PDFs beim erneuten + * Versand oft neu (andere Checksumme). Der Text-Fingerprint erkennt das. + */ + private async hasTwinByContent( + attachment: Attachment, + range: string, + ): Promise { + const fingerprint = await this.fingerprints.ensure(attachment); + if (!fingerprint) return false; + return this.fingerprints.hasImportedTwin(attachment.Id, fingerprint, range); + } + // --- Print Preview --- async generatePrintPdf( attachmentId: number, @@ -570,7 +594,9 @@ export class EmailImportService { const ranges = att.splitRanges?.length ? att.splitRanges : [undefined]; for (const range of ranges) { const bereich = range ? `${range.start}-${range.end}` : 'full'; - const key = `${attachment.Checksum || `id:${attachment.Id}`}:${bereich}`; + // Fingerprint vor Checksumme: fängt auch neu erzeugte PDFs ab + const fingerprint = await this.fingerprints.ensure(attachment); + const key = `${fingerprint || attachment.Checksum || `id:${attachment.Id}`}:${bereich}`; const bezeichnung = range ? `„${attachment.FileName}" (Seiten ${bereich})` : `„${attachment.FileName}"`; diff --git a/paperless-backend/src/email/email.module.ts b/paperless-backend/src/email/email.module.ts index bd202dc..2eb5cf2 100644 --- a/paperless-backend/src/email/email.module.ts +++ b/paperless-backend/src/email/email.module.ts @@ -10,6 +10,7 @@ import { EmailPageCacheService } from './email-page-cache.service'; import { EmailImportController } from './email-import.controller'; import { EmailImportService } from './email-import.service'; import { ImapFolderService } from './imap-folder.service'; +import { AttachmentFingerprintService } from './attachment-fingerprint.service'; import { CorrespondentEmailMapping } from '../database/entities/correspondent-email-mapping.entity'; import { Task } from '../database/entities/task.entity'; import { PreprocessingModule } from '../preprocessing/preprocessing.module'; @@ -27,7 +28,12 @@ import { PreprocessingModule } from '../preprocessing/preprocessing.module'; PreprocessingModule, ], controllers: [EmailController, EmailImportController], - providers: [EmailImportService, EmailPageCacheService, ImapFolderService], + providers: [ + EmailImportService, + EmailPageCacheService, + ImapFolderService, + AttachmentFingerprintService, + ], exports: [EmailPageCacheService], }) export class EmailModule {} diff --git a/paperless-backend/src/email/pdf-fingerprint.util.spec.ts b/paperless-backend/src/email/pdf-fingerprint.util.spec.ts new file mode 100644 index 0000000..16e0dbb --- /dev/null +++ b/paperless-backend/src/email/pdf-fingerprint.util.spec.ts @@ -0,0 +1,58 @@ +import { PDFDocument, StandardFonts } from 'pdf-lib'; +import { computeContentFingerprint } from './pdf-fingerprint.util'; + +async function pdfMitText( + zeilen: string[], + erstellt = new Date('2026-09-24T12:00:00Z'), +): Promise { + const doc = await PDFDocument.create(); + doc.setCreationDate(erstellt); + doc.setModificationDate(erstellt); + const font = await doc.embedFont(StandardFonts.Helvetica); + const page = doc.addPage(); + zeilen.forEach((text, i) => + page.drawText(text, { x: 50, y: 700 - i * 20, font, size: 12 }), + ); + return Buffer.from(await doc.save()); +} + +const TEXT = [ + 'Lastschriftankuendigung / Pre-Notification', + 'Mandatsreferenz 0000844118', + 'Betrag 12,50 EUR', +]; + +describe('computeContentFingerprint', () => { + it('liefert für neu erzeugte PDFs mit gleichem Inhalt denselben Fingerprint', async () => { + const a = await pdfMitText(TEXT, new Date('2026-09-24T12:37:00Z')); + const b = await pdfMitText(TEXT, new Date('2026-09-24T13:45:00Z')); + expect(a.equals(b)).toBe(false); + + const fpA = await computeContentFingerprint(a); + expect(fpA).toMatch(/^[0-9a-f]{64}$/); + await expect(computeContentFingerprint(b)).resolves.toBe(fpA); + }); + + it('unterscheidet PDFs mit anderem Inhalt', async () => { + const a = await pdfMitText(TEXT); + const b = await pdfMitText([...TEXT.slice(0, 2), 'Betrag 13,50 EUR']); + + expect(await computeContentFingerprint(a)).not.toBe( + await computeContentFingerprint(b), + ); + }); + + it('liefert einen Leerstring für PDFs ohne Textebene (z.B. Scans)', async () => { + const doc = await PDFDocument.create(); + doc.addPage(); + const leer = Buffer.from(await doc.save()); + + await expect(computeContentFingerprint(leer)).resolves.toBe(''); + }); + + it('liefert null, wenn die Datei kein lesbares PDF ist', async () => { + await expect( + computeContentFingerprint(Buffer.from('kein pdf')), + ).resolves.toBeNull(); + }); +}); diff --git a/paperless-backend/src/email/pdf-fingerprint.util.ts b/paperless-backend/src/email/pdf-fingerprint.util.ts new file mode 100644 index 0000000..a5caf9f --- /dev/null +++ b/paperless-backend/src/email/pdf-fingerprint.util.ts @@ -0,0 +1,37 @@ +import * as crypto from 'crypto'; +import { PDFParse } from 'pdf-parse'; + +/** + * Unterhalb dieser Textlänge gilt ein PDF als "ohne Textebene" (z.B. Scan) – + * ein paar Zeichen reichen nicht, um Dokumente sicher zu unterscheiden. + */ +const MIN_TEXT_LENGTH = 20; + +/** + * Inhaltsbasierter Fingerprint eines PDFs: SHA-256 über den extrahierten, + * whitespace-normalisierten Text. Anders als die Datei-Checksumme bleibt er + * gleich, wenn ein Absender dasselbe Dokument neu erzeugt (andere + * Erstellungszeit, Dokument-ID o.ä.). + * + * @returns Fingerprint (64 Hex-Zeichen), `''` bei PDFs ohne verwertbaren Text + * (bereits berechnet, aber nicht vergleichbar) oder `null`, wenn das + * PDF nicht gelesen werden konnte. + */ +export async function computeContentFingerprint( + pdf: Buffer, +): Promise { + // pdf.js übernimmt den Puffer, deshalb eine Kopie übergeben + const parser = new PDFParse({ data: new Uint8Array(pdf) }); + try { + const { text } = await parser.getText(); + const normalized = text.replace(/\s+/g, ' ').trim(); + // Seitenmarker ("-- 1 of 1 --") zählen nicht als Inhalt + const inhalt = normalized.replace(/-- \d+ of \d+ --/g, '').trim(); + if (inhalt.length < MIN_TEXT_LENGTH) return ''; + return crypto.createHash('sha256').update(normalized).digest('hex'); + } catch { + return null; + } finally { + await parser.destroy().catch(() => undefined); + } +}