From 4f34855c1226100b8f2274e83310409aeb306bcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20P=C3=B6ttker?= Date: Sat, 31 Oct 2026 20:15:01 +0100 Subject: [PATCH] fix(paperless): Anpassung an Paperless-NGX 3.4 (API v10) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Task-Status: getTask() normalisiert die paginierte v10-Antwort (status klein, result_data, related_document_ids). Bisher wurde jede Task als „nicht gefunden“ gelöscht, ohne Nachbearbeitung. - Checksummen: Paperless ≥ 3.0 nutzt SHA-256 statt MD5. Duplikatprüfung umgestellt; Migration verbreitert Attachments.Checksum und berechnet Bestandswerte per SHA2() neu. - Custom-Field-Suche über custom_field_query, der alte Filter wurde von Paperless ignoriert. Co-Authored-By: Claude Opus 5.5 --- .../database/entities/attachment.entity.ts | 3 +- .../1793500000000-AttachmentChecksumSha256.ts | 35 +++++++++ .../email-download/email-download.service.ts | 5 +- .../src/email/email-import.service.spec.ts | 2 +- .../src/email/email-import.service.ts | 22 +++--- .../inbox-postprocessor.service.ts | 3 +- .../paperless-task-processor.service.spec.ts | 12 +-- .../paperless-task-processor.service.ts | 34 ++++----- .../paperless-task.normalize.spec.ts | 76 +++++++++++++++++++ .../src/paperless/paperless.service.ts | 66 +++++++++++++++- 10 files changed, 218 insertions(+), 40 deletions(-) create mode 100644 paperless-backend/src/database/migrations/1793500000000-AttachmentChecksumSha256.ts create mode 100644 paperless-backend/src/paperless/paperless-task.normalize.spec.ts diff --git a/paperless-backend/src/database/entities/attachment.entity.ts b/paperless-backend/src/database/entities/attachment.entity.ts index 646371c..7c94454 100644 --- a/paperless-backend/src/database/entities/attachment.entity.ts +++ b/paperless-backend/src/database/entities/attachment.entity.ts @@ -28,7 +28,8 @@ export class Attachment { @Column({ type: 'tinyint', width: 1 }) Erechnung!: boolean; - @Column({ type: 'varchar', length: 32, default: '' }) + /** SHA-256 der Datei — entspricht der Checksumme in Paperless ≥ 3.0 */ + @Column({ type: 'varchar', length: 64, default: '' }) Checksum!: string; /** diff --git a/paperless-backend/src/database/migrations/1793500000000-AttachmentChecksumSha256.ts b/paperless-backend/src/database/migrations/1793500000000-AttachmentChecksumSha256.ts new file mode 100644 index 0000000..473843c --- /dev/null +++ b/paperless-backend/src/database/migrations/1793500000000-AttachmentChecksumSha256.ts @@ -0,0 +1,35 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +/** + * Paperless-NGX ≥ 3.0 speichert Checksummen als SHA-256 statt MD5. Damit der + * Duplikat-Abgleich (`checksum__iexact`) wieder trifft, wird + * `Attachments.Checksum` auf 64 Zeichen verbreitert und für alle Anhänge mit + * gespeichertem Inhalt direkt in der Datenbank neu berechnet. + */ +export class AttachmentChecksumSha2561793500000000 implements MigrationInterface { + name = 'AttachmentChecksumSha2561793500000000'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + ALTER TABLE \`Attachments\` + MODIFY \`Checksum\` varchar(64) NOT NULL DEFAULT '' + `); + await queryRunner.query(` + UPDATE \`Attachments\` a + INNER JOIN \`Contents\` c ON c.\`AttachmentEntityId\` = a.\`Id\` + SET a.\`Checksum\` = SHA2(c.\`Content\`, 256) + `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + UPDATE \`Attachments\` a + INNER JOIN \`Contents\` c ON c.\`AttachmentEntityId\` = a.\`Id\` + SET a.\`Checksum\` = MD5(c.\`Content\`) + `); + await queryRunner.query(` + ALTER TABLE \`Attachments\` + MODIFY \`Checksum\` varchar(32) NOT NULL DEFAULT '' + `); + } +} diff --git a/paperless-backend/src/email-download/email-download.service.ts b/paperless-backend/src/email-download/email-download.service.ts index ebdc010..6692b46 100644 --- a/paperless-backend/src/email-download/email-download.service.ts +++ b/paperless-backend/src/email-download/email-download.service.ts @@ -304,7 +304,10 @@ export class EmailDownloadService { attachment.ContentType = contentType.slice(0, 100); attachment.IsEmbedded = isEmbedded; attachment.ContentId = att.cid ? att.cid.slice(0, 255) : null; - attachment.Checksum = crypto.createHash('md5').update(buffer).digest('hex'); + attachment.Checksum = crypto + .createHash('sha256') + .update(buffer) + .digest('hex'); attachment.ContentFingerprint = contentType.toLowerCase() === 'application/pdf' ? await computeContentFingerprint(buffer) diff --git a/paperless-backend/src/email/email-import.service.spec.ts b/paperless-backend/src/email/email-import.service.spec.ts index 6723247..1972b85 100644 --- a/paperless-backend/src/email/email-import.service.spec.ts +++ b/paperless-backend/src/email/email-import.service.spec.ts @@ -184,7 +184,7 @@ describe('EmailImportService – Duplikatprüfung', () => { const [checksum] = paperlessService.checksumExists.mock .calls[0] as string[]; - expect(checksum).toMatch(/^[0-9a-f]{32}$/); + expect(checksum).toMatch(/^[0-9a-f]{64}$/); expect(checksum).not.toBe('abc123'); }); }); diff --git a/paperless-backend/src/email/email-import.service.ts b/paperless-backend/src/email/email-import.service.ts index 34fab64..2b6c9db 100644 --- a/paperless-backend/src/email/email-import.service.ts +++ b/paperless-backend/src/email/email-import.service.ts @@ -307,8 +307,9 @@ export class EmailImportService { const copied = await sliced.copyPages(pdfDoc, indices); copied.forEach((p) => sliced.addPage(p)); + // Paperless ≥ 3.0 vergleicht SHA-256-Checksummen const checksum = crypto - .createHash('md5') + .createHash('sha256') .update(Buffer.from(await sliced.save())) .digest('hex'); return this.paperlessService.checksumExists(checksum); @@ -787,14 +788,17 @@ export class EmailImportService { try { const taskStatus = await this.paperlessService.getTask(paperlessTaskId); - // Paperless returns { results: [ ... ] } for filtered tasks - const statusObj = taskStatus.results - ? taskStatus.results[0] - : Array.isArray(taskStatus) - ? taskStatus[0] - : taskStatus; - if (statusObj && statusObj.related_document) { - docId = statusObj.related_document; + if (taskStatus?.documentId) { + docId = taskStatus.documentId; + break; + } + if ( + taskStatus?.status === 'FAILURE' || + taskStatus?.status === 'REVOKED' + ) { + this.logger.warn( + `Paperless-Task ${paperlessTaskId} fehlgeschlagen: ${taskStatus.message ?? 'keine Meldung'}`, + ); break; } } catch (e) {} diff --git a/paperless-backend/src/inbox-postprocessor/inbox-postprocessor.service.ts b/paperless-backend/src/inbox-postprocessor/inbox-postprocessor.service.ts index 2704547..bf24937 100644 --- a/paperless-backend/src/inbox-postprocessor/inbox-postprocessor.service.ts +++ b/paperless-backend/src/inbox-postprocessor/inbox-postprocessor.service.ts @@ -483,7 +483,8 @@ export class InboxPostprocessorService { // 6. Checksum berechnen und prüfen const buffer = await fs.readFile(pdfPath); - const checksum = crypto.createHash('md5').update(buffer).digest('hex'); + // Paperless ≥ 3.0 vergleicht SHA-256-Checksummen + const checksum = crypto.createHash('sha256').update(buffer).digest('hex'); const checksumExists = await this.paperlessService.checksumExists(checksum); if (checksumExists) { diff --git a/paperless-backend/src/paperless/paperless-task-processor.service.spec.ts b/paperless-backend/src/paperless/paperless-task-processor.service.spec.ts index 04a0d3b..797f67d 100644 --- a/paperless-backend/src/paperless/paperless-task-processor.service.spec.ts +++ b/paperless-backend/src/paperless/paperless-task-processor.service.spec.ts @@ -64,14 +64,14 @@ describe('PaperlessTaskProcessorService.processSuccessfulTask', () => { /** Zugriff auf die private Methode, typsicher für den Test gekapselt. */ function runProcessSuccessfulTask( task: Task, - apiTask: { related_document: number }, + apiTask: { documentId: number }, parentTask: Task | null, ): Promise { return ( service as unknown as { processSuccessfulTask: ( t: Task, - a: { related_document: number }, + a: { documentId: number }, p: Task | null, ) => Promise; } @@ -132,7 +132,7 @@ describe('PaperlessTaskProcessorService.processSuccessfulTask', () => { CustomFieldsJson: JSON.stringify({ '4': 'x' }), }); - await runProcessSuccessfulTask(task, { related_document: 101 }, null); + await runProcessSuccessfulTask(task, { documentId: 101 }, null); expect(paperlessService.updateDocument).toHaveBeenCalledTimes(1); expect(paperlessService.updateDocument).toHaveBeenCalledWith( @@ -164,7 +164,7 @@ describe('PaperlessTaskProcessorService.processSuccessfulTask', () => { }); const task = createTask({ InterneBelegnummer: '2026-000124' }); - await runProcessSuccessfulTask(task, { related_document: 102 }, null); + await runProcessSuccessfulTask(task, { documentId: 102 }, null); expect(lastPatchPayload().owner).toBeNull(); }); @@ -178,7 +178,7 @@ describe('PaperlessTaskProcessorService.processSuccessfulTask', () => { tags: [], }) .mockResolvedValueOnce({ id: 55, title: 'Eltern' }); - paperlessService.getTask.mockResolvedValue([{ related_document: 55 }]); + paperlessService.getTask.mockResolvedValue({ documentId: 55 }); const task = createTask({ DocumentType: 2, TaskReferenceID: 'parent-uuid', @@ -189,7 +189,7 @@ describe('PaperlessTaskProcessorService.processSuccessfulTask', () => { Fertig: 1, }); - await runProcessSuccessfulTask(task, { related_document: 103 }, parentTask); + await runProcessSuccessfulTask(task, { documentId: 103 }, parentTask); const payload = lastPatchPayload(); expect(payload.document_type).toBe(5); diff --git a/paperless-backend/src/paperless/paperless-task-processor.service.ts b/paperless-backend/src/paperless/paperless-task-processor.service.ts index 26507db..e96312a 100644 --- a/paperless-backend/src/paperless/paperless-task-processor.service.ts +++ b/paperless-backend/src/paperless/paperless-task-processor.service.ts @@ -5,7 +5,10 @@ import { Repository, IsNull } from 'typeorm'; import { Task } from '../database/entities/task.entity'; import { Document } from '../database/entities/document.entity'; import { Attachment } from '../database/entities/attachment.entity'; -import { PaperlessService } from './paperless.service'; +import { + PaperlessService, + type PaperlessTaskStatus, +} from './paperless.service'; import { deriveTaskMetadata } from './task-metadata.util'; @Injectable() @@ -58,15 +61,12 @@ export class PaperlessTaskProcessorService { } // Fetch task status from Paperless - const paperlessTasks = await this.paperlessService.getTask(t.TaskId); - const apiResponseTask = Array.isArray(paperlessTasks) - ? paperlessTasks[0] - : null; + const apiResponseTask = await this.paperlessService.getTask(t.TaskId); if (apiResponseTask) { if (apiResponseTask.status === 'SUCCESS') { - const dateDone = apiResponseTask.date_done - ? new Date(apiResponseTask.date_done) + const dateDone = apiResponseTask.dateDone + ? new Date(apiResponseTask.dateDone) : new Date(); const now = new Date(); @@ -74,12 +74,15 @@ export class PaperlessTaskProcessorService { if (dateDone.getTime() + 10000 < now.getTime()) { await this.processSuccessfulTask(t, apiResponseTask, parentTask); } - } else if (apiResponseTask.status === 'FAILURE') { + } else if ( + apiResponseTask.status === 'FAILURE' || + apiResponseTask.status === 'REVOKED' + ) { // Seit Metadaten (inkl. ASN) direkt beim Upload gesetzt werden, kann // z.B. eine ASN-Kollision bereits den Consume scheitern lassen – // vor dem Löschen die Diagnose-Infos festhalten this.logger.error( - `Task ${t.TaskId} in Paperless fehlgeschlagen (Beleg: ${t.InterneBelegnummer || '-'}, Attachment: ${t.SourceAttachmentID ?? '-'}). Paperless-Meldung: ${apiResponseTask.result ?? 'keine'}`, + `Task ${t.TaskId} in Paperless fehlgeschlagen (Beleg: ${t.InterneBelegnummer || '-'}, Attachment: ${t.SourceAttachmentID ?? '-'}). Paperless-Meldung: ${apiResponseTask.message ?? 'keine'}`, ); toDelete.push(t); } @@ -103,10 +106,10 @@ export class PaperlessTaskProcessorService { private async processSuccessfulTask( t: Task, - apiTask: any, + apiTask: PaperlessTaskStatus, parentTask: Task | null, ) { - const documentId = apiTask.related_document; + const documentId = apiTask.documentId; this.logger.log( `[Postprocessing] Task ${t.TaskId} gestartet. DocumentID: ${documentId ?? 'nicht vorhanden'}`, ); @@ -241,16 +244,13 @@ export class PaperlessTaskProcessorService { this.logger.log( `[Postprocessing] Task ${t.TaskId} - Verarbeite als Anlage zu ParentTask ${parentTask.TaskId}`, ); - const parentPaperlessTasks = await this.paperlessService.getTask( + const apiParentTask = await this.paperlessService.getTask( parentTask.TaskId, ); - const apiParentTask = Array.isArray(parentPaperlessTasks) - ? parentPaperlessTasks[0] - : null; - if (apiParentTask && apiParentTask.related_document) { + if (apiParentTask?.documentId) { const parentDoc = await this.paperlessService.getDocument( - apiParentTask.related_document, + apiParentTask.documentId, ); if (parentDoc) { this.logger.log( diff --git a/paperless-backend/src/paperless/paperless-task.normalize.spec.ts b/paperless-backend/src/paperless/paperless-task.normalize.spec.ts new file mode 100644 index 0000000..3a1e2f6 --- /dev/null +++ b/paperless-backend/src/paperless/paperless-task.normalize.spec.ts @@ -0,0 +1,76 @@ +import { normalizePaperlessTask } from './paperless.service'; + +describe('normalizePaperlessTask', () => { + it('API v10: erfolgreicher Consume liefert documentId aus result_data', () => { + const t = normalizePaperlessTask({ + task_id: 'abc', + task_type: 'consume_file', + status: 'success', + date_done: '2026-10-30T16:16:00.091045+01:00', + result_data: { document_id: 4711 }, + related_document_ids: [4711], + }); + expect(t).toEqual({ + taskId: 'abc', + status: 'SUCCESS', + documentId: 4711, + duplicateOf: null, + message: null, + dateDone: '2026-10-30T16:16:00.091045+01:00', + }); + }); + + it('API v10: Duplikat ist FAILURE ohne documentId, mit duplicateOf', () => { + const t = normalizePaperlessTask({ + task_id: 'abc', + status: 'failure', + result_data: { duplicate_of: 12 }, + related_document_ids: [12], + }); + expect(t.status).toBe('FAILURE'); + expect(t.documentId).toBeNull(); + expect(t.duplicateOf).toBe(12); + expect(t.message).toContain('12'); + }); + + it('API v10: Fehler übernimmt error_message', () => { + const t = normalizePaperlessTask({ + task_id: 'abc', + status: 'failure', + result_data: { + error_type: 'ConsumerError', + error_message: 'ASN existiert', + }, + related_document_ids: [], + }); + expect(t.status).toBe('FAILURE'); + expect(t.documentId).toBeNull(); + expect(t.message).toBe('ASN existiert'); + }); + + it('API v10: laufende Task ohne Ergebnis', () => { + const t = normalizePaperlessTask({ + task_id: 'abc', + status: 'started', + result_data: null, + related_document_ids: [], + date_done: null, + }); + expect(t.status).toBe('STARTED'); + expect(t.documentId).toBeNull(); + expect(t.message).toBeNull(); + }); + + it('Legacy-API v9: related_document und result', () => { + const t = normalizePaperlessTask({ + task_id: 'abc', + status: 'SUCCESS', + result: 'Success. New document id 55 created', + related_document: '55', + date_done: '2026-10-30T16:16:00+01:00', + }); + expect(t.status).toBe('SUCCESS'); + expect(t.documentId).toBe(55); + expect(t.message).toBe('Success. New document id 55 created'); + }); +}); diff --git a/paperless-backend/src/paperless/paperless.service.ts b/paperless-backend/src/paperless/paperless.service.ts index bb9cfeb..783635d 100644 --- a/paperless-backend/src/paperless/paperless.service.ts +++ b/paperless-backend/src/paperless/paperless.service.ts @@ -5,6 +5,55 @@ import * as fs from 'fs'; import * as path from 'path'; import FormData = require('form-data'); +export interface PaperlessTaskStatus { + taskId: string; + status: 'PENDING' | 'STARTED' | 'SUCCESS' | 'FAILURE' | 'REVOKED'; + /** Neu angelegtes Dokument (nur bei Erfolg) */ + documentId: number | null; + /** Bereits vorhandenes Dokument, falls Paperless ein Duplikat abgelehnt hat */ + duplicateOf: number | null; + /** Fehlermeldung bzw. Ergebnistext */ + message: string | null; + dateDone: string | null; +} + +/** + * Bildet eine Task aus API v10 (`result_data`, `related_document_ids`) oder + * der Legacy-API v9 (`result`, `related_document`) auf PaperlessTaskStatus ab. + */ +export function normalizePaperlessTask(raw: any): PaperlessTaskStatus { + const resultData = + raw.result_data && typeof raw.result_data === 'object' + ? raw.result_data + : null; + const toId = (v: unknown): number | null => + v === null || v === undefined || v === '' ? null : Number(v); + + const duplicateOf = toId(resultData?.duplicate_of); + const documentId = + toId(resultData?.document_id) ?? + toId(raw.related_document) ?? + (duplicateOf === null ? toId(raw.related_document_ids?.[0]) : null); + + let message: string | null = null; + if (resultData?.error_message) message = String(resultData.error_message); + else if (duplicateOf !== null) + message = `Duplikat von Dokument ${duplicateOf}`; + else if (typeof raw.result_data === 'string') message = raw.result_data; + else if (raw.result != null) message = String(raw.result); + + return { + taskId: raw.task_id, + status: String( + raw.status ?? '', + ).toUpperCase() as PaperlessTaskStatus['status'], + documentId, + duplicateOf, + message, + dateDone: raw.date_done ?? null, + }; +} + @Injectable() export class PaperlessService { private readonly logger = new Logger(PaperlessService.name); @@ -175,11 +224,19 @@ export class PaperlessService { return response.data.results; } - async getTask(taskId: string): Promise { + /** + * Liefert den Status einer Paperless-Task oder null, falls unbekannt. + * Paperless ≥ 3.0 (API v10) antwortet paginiert mit kleingeschriebenem + * Status, `result_data` und `related_document_ids` — hier auf ein festes + * Format normalisiert, damit die Aufrufer unabhängig von der API-Version sind. + */ + async getTask(taskId: string): Promise { const response = await this.client.get('/tasks/', { params: { task_id: taskId }, }); - return response.data; + const data = response.data; + const raw = Array.isArray(data) ? data[0] : data?.results?.[0]; + return raw ? normalizePaperlessTask(raw) : null; } async getCorrespondentByName(name: string): Promise { @@ -311,7 +368,8 @@ export class PaperlessService { /** * Liefert die Paperless-Doc-ID des passenden Dokuments oder null. - * Paperless kann den custom_fields-Filter ignorieren — daher manuell verifizieren. + * Filtert serverseitig per `custom_field_query`; die Treffer werden zusätzlich + * case-insensitiv verifiziert (Paperless kennt für Textfelder kein `iexact`). */ async findDocumentIdByCustomField( fieldId: number, @@ -319,7 +377,7 @@ export class PaperlessService { ): Promise { const response = await this.client.get('/documents/', { params: { - [`custom_fields__${fieldId}__value__iexact`]: value, + custom_field_query: JSON.stringify([fieldId, 'icontains', value]), page_size: 25, truncate_content: true, },