From acb90769443ad2164b270fa0e1895a9c5f1df1c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20P=C3=B6ttker?= Date: Fri, 25 Sep 2026 12:42:02 +0200 Subject: [PATCH] =?UTF-8?q?fix(mail):=20=E2=80=9EAnh=C3=A4nge=20pr=C3=BCfe?= =?UTF-8?q?n=E2=80=9C=20erkennt=20Duplikate=20auch=20=C3=BCber=20den=20Inh?= =?UTF-8?q?alt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit check-attachments fragte nur die Checksumme bei Paperless ab. Neu erzeugte PDFs mit gleichem Inhalt wie ein bereits importierter Anhang blieben dadurch auf „Neu“. Findet Paperless die Checksumme nicht, wird jetzt per Text-Fingerprint nach einem importierten Anhang gesucht; die Mail wird dann als verarbeitet markiert und die Paperless-ID übernommen. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../attachment-fingerprint.service.spec.ts | 21 +++++ .../email/attachment-fingerprint.service.ts | 23 ++++++ .../src/email/email.controller.spec.ts | 77 ++++++++++++++++++- .../src/email/email.controller.ts | 22 +++++- 4 files changed, 138 insertions(+), 5 deletions(-) diff --git a/paperless-backend/src/email/attachment-fingerprint.service.spec.ts b/paperless-backend/src/email/attachment-fingerprint.service.spec.ts index 20bad9e..d7105e2 100644 --- a/paperless-backend/src/email/attachment-fingerprint.service.spec.ts +++ b/paperless-backend/src/email/attachment-fingerprint.service.spec.ts @@ -143,6 +143,27 @@ describe('AttachmentFingerprintService', () => { }); }); + describe('findImportedTwin', () => { + it('liefert den importierten Anhang mit gleichem Inhalt samt Paperless-ID', async () => { + query.getOne = jest.fn().mockResolvedValue({ + Id: 3693, + PaperlessDocumentIds: { full: 14138 }, + }); + + await expect( + service.findImportedTwin(pdfAnhang({ ContentFingerprint: 'fp' })), + ).resolves.toEqual({ attachmentId: 3693, documentId: 14138 }); + expect(query.andWhere).toHaveBeenCalledWith('a.ImportStatus = 1'); + }); + + it('liefert null ohne vergleichbaren Fingerprint', async () => { + await expect( + service.findImportedTwin(pdfAnhang({ ContentFingerprint: '' })), + ).resolves.toBeNull(); + expect(attachmentRepo.createQueryBuilder).not.toHaveBeenCalled(); + }); + }); + it('backfill berechnet fehlende Fingerprints portionsweise', async () => { query.getMany .mockResolvedValueOnce([pdfAnhang({ Id: 1 }), pdfAnhang({ Id: 2 })]) diff --git a/paperless-backend/src/email/attachment-fingerprint.service.ts b/paperless-backend/src/email/attachment-fingerprint.service.ts index 45aab44..ab0c7ca 100644 --- a/paperless-backend/src/email/attachment-fingerprint.service.ts +++ b/paperless-backend/src/email/attachment-fingerprint.service.ts @@ -88,6 +88,29 @@ export class AttachmentFingerprintService implements OnApplicationBootstrap { return count > 0; } + /** + * Sucht einen bereits importierten Anhang mit gleichem Inhalt und liefert + * dessen Paperless-Dokument-ID (sofern beim Import ermittelt). + */ + async findImportedTwin( + attachment: Attachment, + ): Promise<{ attachmentId: number; documentId: number | null } | null> { + const fingerprint = await this.ensure(attachment); + if (!fingerprint) return null; + + const twin = await this.attachmentRepo + .createQueryBuilder('a') + .where('a.ContentFingerprint = :fingerprint', { fingerprint }) + .andWhere('a.Id != :attachmentId', { attachmentId: attachment.Id }) + .andWhere('a.ImportStatus = 1') + .orderBy('a.Id', 'ASC') + .getOne(); + if (!twin) return null; + + const ids = twin.PaperlessDocumentIds as Record | null; + return { attachmentId: twin.Id, documentId: ids?.full ?? null }; + } + /** Berechnet fehlende Fingerprints aller PDF-Anhänge in kleinen Portionen. */ async backfill(batchSize = 25): Promise { let lastId = 0; diff --git a/paperless-backend/src/email/email.controller.spec.ts b/paperless-backend/src/email/email.controller.spec.ts index a34b363..b77a39f 100644 --- a/paperless-backend/src/email/email.controller.spec.ts +++ b/paperless-backend/src/email/email.controller.spec.ts @@ -7,6 +7,7 @@ import { Attachment } from '../database/entities/attachment.entity'; import { Content } from '../database/entities/content.entity'; import { PaperlessService } from '../paperless/paperless.service'; import { ImapFolderService } from './imap-folder.service'; +import { AttachmentFingerprintService } from './attachment-fingerprint.service'; import { Reflector } from '@nestjs/core'; import { PERMISSIONS_KEY } from '../auth/permissions.decorator'; import { Permission } from '../auth/permissions.enum'; @@ -51,8 +52,11 @@ describe('EmailController', () => { createQueryBuilder: jest.Mock; findOneOrFail: jest.Mock; save: jest.Mock; + find: jest.Mock; }; - let attachmentRepo: { find: jest.Mock; findOne: jest.Mock }; + let attachmentRepo: { find: jest.Mock; findOne: jest.Mock; save: jest.Mock }; + let paperlessService: { getDocumentIdByChecksum: jest.Mock }; + let fingerprints: { findImportedTwin: jest.Mock }; beforeEach(async () => { queryBuilder = { @@ -66,11 +70,17 @@ describe('EmailController', () => { createQueryBuilder: jest.fn().mockReturnValue(queryBuilder), findOneOrFail: jest.fn().mockResolvedValue({ ...mockEmails[0] }), save: jest.fn().mockImplementation((e: Email) => Promise.resolve(e)), + find: jest.fn().mockResolvedValue([]), }; attachmentRepo = { find: jest.fn().mockResolvedValue(mockAttachments), findOne: jest.fn().mockResolvedValue(mockAttachments[0]), + save: jest.fn(), }; + paperlessService = { + getDocumentIdByChecksum: jest.fn().mockResolvedValue(null), + }; + fingerprints = { findImportedTwin: jest.fn().mockResolvedValue(null) }; const module: TestingModule = await Test.createTestingModule({ controllers: [EmailController], @@ -81,8 +91,12 @@ describe('EmailController', () => { provide: getRepositoryToken(Content), useValue: { findOne: jest.fn() }, }, - { provide: PaperlessService, useValue: {} }, - { provide: ImapFolderService, useValue: {} }, + { provide: PaperlessService, useValue: paperlessService }, + { + provide: ImapFolderService, + useValue: { cleanupImportedEmails: jest.fn().mockResolvedValue(0) }, + }, + { provide: AttachmentFingerprintService, useValue: fingerprints }, ], }).compile(); @@ -223,4 +237,61 @@ describe('EmailController', () => { expect.objectContaining({ Id: 1, Status: 2 }), ); }); + + describe('checkAttachments', () => { + const neueMail = () => ({ + Id: 7071, + Status: 0, + Attachments: [ + { + Id: 3692, + ContentType: 'application/pdf', + Checksum: 'neu-erzeugt', + PaperlessDocumentIds: null, + }, + ], + }); + + it('markiert eine Mail als verarbeitet, wenn Paperless die Checksumme kennt', async () => { + emailRepo.find.mockResolvedValue([neueMail()]); + paperlessService.getDocumentIdByChecksum.mockResolvedValue(14138); + + const result = await controller.checkAttachments(); + + expect(result.updatedCount).toBe(1); + expect(fingerprints.findImportedTwin).not.toHaveBeenCalled(); + }); + + it('erkennt neu erzeugte PDFs über einen importierten Anhang mit gleichem Inhalt', async () => { + emailRepo.find.mockResolvedValue([neueMail()]); + fingerprints.findImportedTwin.mockResolvedValue({ + attachmentId: 3693, + documentId: 14138, + }); + + const result = await controller.checkAttachments(); + + expect(result).toEqual( + expect.objectContaining({ updatedCount: 1, idsUpdated: 1 }), + ); + expect(attachmentRepo.save).toHaveBeenCalledWith( + expect.objectContaining({ + Id: 3692, + PaperlessDocumentIds: { full: 14138 }, + }), + ); + expect(emailRepo.save).toHaveBeenCalledWith( + expect.objectContaining({ Id: 7071, Status: 1 }), + ); + }); + + it('lässt die Mail neu, wenn weder Checksumme noch Inhalt bekannt sind', async () => { + emailRepo.find.mockResolvedValue([neueMail()]); + + const result = await controller.checkAttachments(); + + expect(result.updatedCount).toBe(0); + expect(emailRepo.save).not.toHaveBeenCalled(); + }); + }); }); diff --git a/paperless-backend/src/email/email.controller.ts b/paperless-backend/src/email/email.controller.ts index 2278bd4..6f4caa8 100644 --- a/paperless-backend/src/email/email.controller.ts +++ b/paperless-backend/src/email/email.controller.ts @@ -20,6 +20,7 @@ import { Attachment } from '../database/entities/attachment.entity'; import { Content } from '../database/entities/content.entity'; import { PaperlessService } from '../paperless/paperless.service'; import { ImapFolderService } from './imap-folder.service'; +import { AttachmentFingerprintService } from './attachment-fingerprint.service'; import { RequirePermissions } from '../auth/permissions.decorator'; import { Permission } from '../auth/permissions.enum'; @@ -48,6 +49,7 @@ export class EmailController { private readonly contentRepo: Repository, private readonly paperlessService: PaperlessService, private readonly imapFolderService: ImapFolderService, + private readonly fingerprints: AttachmentFingerprintService, ) {} @Get() @@ -230,7 +232,7 @@ export class EmailController { `Prüfe Checksumme für E-Mail ${email.Id}, Anhang ${attachment.Id} (${attachment.FileName})`, ); try { - const docId = await this.paperlessService.getDocumentIdByChecksum( + let docId = await this.paperlessService.getDocumentIdByChecksum( attachment.Checksum, ); if (docId !== null) { @@ -238,7 +240,23 @@ export class EmailController { `Treffer! Anhang ${attachment.Id} (E-Mail ${email.Id}) → Paperless-Dokument ${docId}.`, ); hasMatch = true; - + } else { + // Neu erzeugtes PDF (andere Checksumme) mit gleichem Inhalt + // wie ein bereits importierter Anhang? + const twin = + await this.fingerprints.findImportedTwin(attachment); + if (twin) { + this.logger.log( + `Treffer über Inhalt! Anhang ${attachment.Id} (E-Mail ${email.Id}) entspricht importiertem Anhang ${twin.attachmentId}` + + (twin.documentId + ? ` → Paperless-Dokument ${twin.documentId}.` + : '.'), + ); + hasMatch = true; + docId = twin.documentId; + } + } + if (docId !== null) { // PaperlessDocumentId hinterlegen, falls noch nicht vorhanden const existingIds: Record = attachment.PaperlessDocumentIds ?? {};