From b1b30fe1dd6de713c0b13bd035af1f79f24e6964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20P=C3=B6ttker?= Date: Tue, 16 Jun 2026 13:53:56 +0200 Subject: [PATCH] feat: auto-move imported emails to IMAP folder and add 90-day cleanup - New ImapFolderService moves emails to configurable "importiert" folder after successful import, creating the folder if it doesn't exist - Daily cron at 03:00 moves emails older than 90 days to trash and empties it - Extract createImapClient() helper in EmailDownloadService - Add ensurePageCache() with in-flight deduplication to BarcodeScannerService - InboxService regenerates page cache on-demand when image file is missing - IMAP_IMPORTED_FOLDER and IMAP_TRASH_FOLDER added to .env.example and docker-compose Co-Authored-By: Claude Sonnet 4.6 --- .env.example | 6 ++ docker-compose.yml | 2 + .../src/barcode/barcode-scanner.service.ts | 39 +++++++++++ .../email-download/email-download.service.ts | 67 ++++++++++++++++--- .../src/email/email-import.service.ts | 8 +++ paperless-backend/src/email/email.module.ts | 3 +- .../src/email/imap-folder.service.ts | 52 ++++++++++++++ paperless-backend/src/inbox/inbox.service.ts | 9 ++- 8 files changed, 172 insertions(+), 14 deletions(-) create mode 100644 paperless-backend/src/email/imap-folder.service.ts diff --git a/.env.example b/.env.example index e62dd5b..f7cec8d 100644 --- a/.env.example +++ b/.env.example @@ -67,3 +67,9 @@ AGRARMONITOR_UPLOAD_CHECK_CRON=0 * * * * * # Upload-Check-Intervall (Standard: # Leer lassen: E-Mails werden ohne Links versendet APP_URL= DAILY_DIGEST_CRON= # Standard: 0 7 * * * (täglich 07:00 Uhr Europe/Berlin) + +# --- IMAP-Ordnerverwaltung --- +# Zielordner für importierte E-Mails (wird automatisch angelegt falls nicht vorhanden) +IMAP_IMPORTED_FOLDER=importiert +# Papierkorb-Ordner für die 90-Tage-Bereinigung (Gmail: "[Gmail]/Papierkorb", Outlook: "Deleted Items") +IMAP_TRASH_FOLDER=Trash diff --git a/docker-compose.yml b/docker-compose.yml index 12a8d0b..11c008d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,6 +36,8 @@ services: - IMAP_USE_SSL=${IMAP_USE_SSL:-true} - IMAP_USERNAME=${IMAP_USERNAME:-} - IMAP_PASSWORD=${IMAP_PASSWORD:-} + - IMAP_IMPORTED_FOLDER=${IMAP_IMPORTED_FOLDER:-importiert} + - IMAP_TRASH_FOLDER=${IMAP_TRASH_FOLDER:-Trash} - BELEGNUMMER_GET_URL=${BELEGNUMMER_GET_URL:-} - BELEGNUMMER_SET_URL=${BELEGNUMMER_SET_URL:-} - AGRARMONITOR_BASE_URL=${AGRARMONITOR_BASE_URL:-https://admin7.agrarmonitor.de} diff --git a/paperless-backend/src/barcode/barcode-scanner.service.ts b/paperless-backend/src/barcode/barcode-scanner.service.ts index 4f335de..8eaea56 100644 --- a/paperless-backend/src/barcode/barcode-scanner.service.ts +++ b/paperless-backend/src/barcode/barcode-scanner.service.ts @@ -34,6 +34,7 @@ export interface MatchedBarcode { export class BarcodeScannerService implements OnApplicationBootstrap { private readonly logger = new Logger(BarcodeScannerService.name); private templatesCache: BarcodeTemplate[] | null = null; + private readonly regenerating = new Map>(); constructor( private readonly pdfService: PdfService, @@ -120,6 +121,44 @@ export class BarcodeScannerService implements OnApplicationBootstrap { return this.matchTemplates(doc.QrCodes ?? []); } + /** + * Stellt sicher, dass der Seiten-Cache (thumb/preview-PNGs) für ein Dokument + * vorhanden ist. Parallele Aufrufe für dasselbe Dokument warten auf dasselbe + * Promise, um doppeltes Rendering zu vermeiden. + */ + async ensurePageCache(documentId: string, pdfPath: string): Promise { + const existing = this.regenerating.get(documentId); + if (existing) return existing; + + const work = this.doRegenerateCache(documentId, pdfPath).finally(() => { + this.regenerating.delete(documentId); + }); + this.regenerating.set(documentId, work); + return work; + } + + private async doRegenerateCache( + documentId: string, + pdfPath: string, + ): Promise { + let images: string[] = []; + try { + images = await this.pdfService.pdfToImages(pdfPath, 200); + await this.pageCache.clear(documentId); + await this.pageCache.generate(documentId, images); + this.logger.log( + `Seiten-Cache regeneriert für ${documentId} (${images.length} Seiten)`, + ); + } catch (err: unknown) { + this.logger.warn( + `Cache-Regenerierung fehlgeschlagen (${documentId}): ${getErrorMessage(err)}`, + ); + throw err; + } finally { + await this.pdfService.cleanup(images); + } + } + private async matchTemplates( qrCodes: StoredQrCode[], ): Promise { diff --git a/paperless-backend/src/email-download/email-download.service.ts b/paperless-backend/src/email-download/email-download.service.ts index cd0cb65..1219f2e 100644 --- a/paperless-backend/src/email-download/email-download.service.ts +++ b/paperless-backend/src/email-download/email-download.service.ts @@ -56,11 +56,62 @@ export class EmailDownloadService { } } + private createImapClient(): ImapFlow { + return new ImapFlow({ + host: this.configService.get('IMAP_HOST', ''), + port: this.configService.get('IMAP_PORT', 993), + secure: this.configService.get('IMAP_USE_SSL', 'true') === 'true', + auth: { + user: this.configService.get('IMAP_USERNAME', ''), + pass: this.configService.get('IMAP_PASSWORD', ''), + }, + logger: false, + }); + } + + @Cron('0 3 * * *', { timeZone: 'Europe/Berlin' }) + async cleanupImportedEmails(): Promise { + if (!this.configService.get('IMAP_HOST')) return; + const importedFolder = this.configService.get('IMAP_IMPORTED_FOLDER', 'importiert'); + const trashFolder = this.configService.get('IMAP_TRASH_FOLDER', 'Trash'); + const client = this.createImapClient(); + try { + await client.connect(); + + // Alte E-Mails (> 90 Tage) in Papierkorb verschieben + try { + await client.mailboxOpen(importedFolder); + const cutoff = new Date(); + cutoff.setDate(cutoff.getDate() - 90); + const oldUids = await client.search({ before: cutoff }, { uid: true }); + if (Array.isArray(oldUids) && oldUids.length > 0) { + await client.messageMove(oldUids, trashFolder, { uid: true }); + this.logger.log(`${oldUids.length} alte E-Mail(s) aus "${importedFolder}" in "${trashFolder}" verschoben.`); + } + } catch (err: any) { + this.logger.warn(`Bereinigung "${importedFolder}" nicht möglich: ${err.message}`); + } + + // Papierkorb leeren + try { + await client.mailboxOpen(trashFolder); + const trashUids = await client.search({ all: true }, { uid: true }); + if (Array.isArray(trashUids) && trashUids.length > 0) { + await client.messageDelete(trashUids, { uid: true }); + this.logger.log(`${trashUids.length} E-Mail(s) aus "${trashFolder}" gelöscht.`); + } + } catch (err: any) { + this.logger.warn(`Papierkorb "${trashFolder}" konnte nicht geleert werden: ${err.message}`); + } + } catch (err: any) { + this.logger.error(`IMAP-Cleanup fehlgeschlagen: ${err.message}`); + } finally { + await client.logout().catch(() => {}); + } + } + private async fetchAndStore(): Promise { const host = this.configService.get('IMAP_HOST'); - const port = this.configService.get('IMAP_PORT', 993); - const secure = - this.configService.get('IMAP_USE_SSL', 'true') === 'true'; const user = this.configService.get('IMAP_USERNAME'); const pass = this.configService.get('IMAP_PASSWORD'); @@ -73,16 +124,10 @@ export class EmailDownloadService { this.logger.log('E-Mail Fetch Job gestartet.'); - const client = new ImapFlow({ - host, - port, - secure, - auth: { user, pass }, - logger: false, - }); + const client = this.createImapClient(); await client.connect(); - this.logger.log(`Verbunden mit IMAP-Server ${host}:${port}`); + this.logger.log(`Verbunden mit IMAP-Server ${host}.`); const lock = await client.getMailboxLock('INBOX'); try { diff --git a/paperless-backend/src/email/email-import.service.ts b/paperless-backend/src/email/email-import.service.ts index 2431c00..ed0b13d 100644 --- a/paperless-backend/src/email/email-import.service.ts +++ b/paperless-backend/src/email/email-import.service.ts @@ -12,6 +12,7 @@ import { Task } from '../database/entities/task.entity'; import { PaperlessService } from '../paperless/paperless.service'; import * as QRCode from 'qrcode'; import { EmailPageCacheService } from './email-page-cache.service'; +import { ImapFolderService } from './imap-folder.service'; import { PdfService } from '../preprocessing/pdf.service'; import * as path from 'path'; import * as os from 'os'; @@ -53,6 +54,7 @@ export class EmailImportService { private readonly paperlessService: PaperlessService, private readonly pdfService: PdfService, private readonly pageCache: EmailPageCacheService, + private readonly imapFolderService: ImapFolderService, ) {} async ensurePreviews(emailId: number): Promise { @@ -653,6 +655,12 @@ export class EmailImportService { this.logger.log( `Email ${firstAtt.EmailMessageId} als verarbeitet markiert.`, ); + const emailEntity = await this.emailRepo.findOne({ where: { Id: firstAtt.EmailMessageId } }); + if (emailEntity) { + this.imapFolderService.moveToImportiert(emailEntity.MessageId).catch(err => + this.logger.error('IMAP-Verschieben fehlgeschlagen: ' + err.message), + ); + } } } diff --git a/paperless-backend/src/email/email.module.ts b/paperless-backend/src/email/email.module.ts index 6b52873..bd202dc 100644 --- a/paperless-backend/src/email/email.module.ts +++ b/paperless-backend/src/email/email.module.ts @@ -9,6 +9,7 @@ import { EmailPageCacheService } from './email-page-cache.service'; import { EmailImportController } from './email-import.controller'; import { EmailImportService } from './email-import.service'; +import { ImapFolderService } from './imap-folder.service'; import { CorrespondentEmailMapping } from '../database/entities/correspondent-email-mapping.entity'; import { Task } from '../database/entities/task.entity'; import { PreprocessingModule } from '../preprocessing/preprocessing.module'; @@ -26,7 +27,7 @@ import { PreprocessingModule } from '../preprocessing/preprocessing.module'; PreprocessingModule, ], controllers: [EmailController, EmailImportController], - providers: [EmailImportService, EmailPageCacheService], + providers: [EmailImportService, EmailPageCacheService, ImapFolderService], exports: [EmailPageCacheService], }) export class EmailModule {} diff --git a/paperless-backend/src/email/imap-folder.service.ts b/paperless-backend/src/email/imap-folder.service.ts new file mode 100644 index 0000000..c4ff0c5 --- /dev/null +++ b/paperless-backend/src/email/imap-folder.service.ts @@ -0,0 +1,52 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { ImapFlow } from 'imapflow'; + +@Injectable() +export class ImapFolderService { + private readonly logger = new Logger(ImapFolderService.name); + + constructor(private readonly configService: ConfigService) {} + + private createClient(): ImapFlow { + return new ImapFlow({ + host: this.configService.get('IMAP_HOST', ''), + port: this.configService.get('IMAP_PORT', 993), + secure: this.configService.get('IMAP_USE_SSL', 'true') === 'true', + auth: { + user: this.configService.get('IMAP_USERNAME', ''), + pass: this.configService.get('IMAP_PASSWORD', ''), + }, + logger: false, + }); + } + + async moveToImportiert(messageId: string): Promise { + if (!this.configService.get('IMAP_HOST')) return; + + const importedFolder = this.configService.get('IMAP_IMPORTED_FOLDER', 'importiert'); + const client = this.createClient(); + try { + await client.connect(); + + const mailboxes = await client.list(); + if (!mailboxes.some(m => m.path === importedFolder)) { + await client.mailboxCreate(importedFolder); + this.logger.log(`IMAP-Ordner "${importedFolder}" erstellt.`); + } + + await client.mailboxOpen('INBOX'); + const uids = await client.search({ header: { 'message-id': messageId } }, { uid: true }); + if (Array.isArray(uids) && uids.length > 0) { + await client.messageMove(uids, importedFolder, { uid: true }); + this.logger.log(`E-Mail ${messageId} → "${importedFolder}" verschoben.`); + } else { + this.logger.warn(`E-Mail ${messageId} nicht in INBOX gefunden (bereits verschoben?).`); + } + } catch (err: any) { + this.logger.error(`IMAP moveToImportiert fehlgeschlagen: ${err.message}`); + } finally { + await client.logout().catch(() => {}); + } + } +} diff --git a/paperless-backend/src/inbox/inbox.service.ts b/paperless-backend/src/inbox/inbox.service.ts index fb3e2e0..e906518 100644 --- a/paperless-backend/src/inbox/inbox.service.ts +++ b/paperless-backend/src/inbox/inbox.service.ts @@ -232,7 +232,7 @@ export class InboxService { variant: 'preview' | 'thumbnail', preferredUsername: string | null, ): Promise { - const { doc } = await this.resolveDocument(id, preferredUsername); + const { doc, pdfPath } = await this.resolveDocument(id, preferredUsername); if (!Number.isInteger(page) || page < 1 || page > doc.PageCount) { throw new NotFoundException('Seite nicht gefunden'); } @@ -244,7 +244,12 @@ export class InboxService { try { await fs.access(filePath); } catch { - throw new NotFoundException('Seite nicht gefunden'); + try { + await this.barcodeScanner.ensurePageCache(doc.Id, pdfPath); + await fs.access(filePath); + } catch { + throw new NotFoundException('Seite nicht gefunden'); + } } return filePath; }