Initial commit with Email Import Wizard and Task Processor updates

This commit is contained in:
2026-05-04 08:02:11 +02:00
commit effdc5d59f
170 changed files with 67739 additions and 0 deletions
+46
View File
@@ -0,0 +1,46 @@
# =============================================================================
# Paperless Manager - Docker Compose Konfiguration
# =============================================================================
# Kopiere diese Datei nach .env und passe die Werte an.
#
# Lokal: VITE_API_URL=http://localhost:3100 (direkter Zugriff)
# Produktion: VITE_API_URL leer lassen (nginx Reverse-Proxy leitet /api weiter)
# =============================================================================
# --- Ports ---
BACKEND_PORT=3100
FRONTEND_PORT=8080
# --- MySQL Datenbank (extern) ---
DB_HOST=192.168.1.x
DB_PORT=3306
DB_USERNAME=root
DB_PASSWORD=changeme
DB_DATABASE=paperlessadd
# --- Paperless-ngx ---
PAPERLESS_URL=http://paperless:8000
PAPERLESS_TOKEN=your_paperless_api_token
# --- Ollama OCR ---
OLLAMA_URL=http://ollama:11434
OLLAMA_MODEL=llava
# --- Scanner ---
SCANNER_WATCH_DIR=/data/scanner
SCANNER_ARCHIVE_DIR=/data/scanner/_processed_archive
# --- OIDC / Authentik ---
OIDC_ISSUER=https://auth.example.com/application/o/your-app-slug
OIDC_CLIENT_ID=your-oidc-client-id
OIDC_REDIRECT_URI=https://dokumente.example.com/auth/callback
# --- Frontend API-URL ---
# Lokal: http://localhost:3100 (Backend direkt)
# Produktion: leer lassen (nginx proxied /api → Backend)
VITE_API_URL=
# --- Interne Belegnummer API ---
# Platzhalter {Jahr} wird zur Laufzeit durch das Jahr des Eingangsdatums ersetzt
BELEGNUMMER_GET_URL=https://beispiel-api.de/get-number/{Jahr}
BELEGNUMMER_SET_URL=https://beispiel-api.de/set-number/{Jahr}/{Nummer}
+16
View File
@@ -0,0 +1,16 @@
# Dependencies
node_modules/
dist/
.env
# OS files
.DS_Store
# IDE files
.vscode/
.agent/
.claude/
.gemini/
# Docker
docker-compose.override.yml
+65
View File
@@ -0,0 +1,65 @@
services:
# ─── Backend (NestJS) ──────────────────────────────────────
backend:
build: ./paperless-backend
container_name: paperless-backend
restart: unless-stopped
ports:
- "${BACKEND_PORT:-3100}:3100"
environment:
- PORT=3100
- DB_HOST=${DB_HOST:-db}
- DB_PORT=${DB_PORT:-3306}
- DB_USERNAME=${DB_USERNAME:-root}
- DB_PASSWORD=${DB_PASSWORD:-changeme}
- DB_DATABASE=${DB_DATABASE:-paperlessadd}
- PAPERLESS_URL=${PAPERLESS_URL:-http://paperless:8000}
- PAPERLESS_TOKEN=${PAPERLESS_TOKEN:-}
- PAPERLESS_PROCESSOR_CRON=${PAPERLESS_PROCESSOR_CRON:-0 * * * * *}
- OLLAMA_URL=${OLLAMA_URL:-http://ollama:11434}
- OLLAMA_MODEL=${OLLAMA_MODEL:-llava}
- SCANNER_ARCHIVE_DIR=${SCANNER_ARCHIVE_DIR:-/data/scanner/_processed_archive}
- OIDC_ISSUER=${OIDC_ISSUER:-}
- SMTP_HOST=${SMTP_HOST:-}
- SMTP_PORT=${SMTP_PORT:-587}
- SMTP_SECURE=${SMTP_SECURE:-false}
- SMTP_USER=${SMTP_USER:-}
- SMTP_PASS=${SMTP_PASS:-}
- SMTP_FROM=${SMTP_FROM:-paperless@localhost}
- POSTPROCESSING_ERROR_TAG=${POSTPROCESSING_ERROR_TAG:-0}
- MANUELL_BEARBEITEN_TAG=${MANUELL_BEARBEITEN_TAG:-6}
- IMAP_HOST=${IMAP_HOST:-}
- IMAP_PORT=${IMAP_PORT:-993}
- IMAP_USE_SSL=${IMAP_USE_SSL:-true}
- IMAP_USERNAME=${IMAP_USERNAME:-}
- IMAP_PASSWORD=${IMAP_PASSWORD:-}
- BELEGNUMMER_GET_URL=${BELEGNUMMER_GET_URL:-}
- BELEGNUMMER_SET_URL=${BELEGNUMMER_SET_URL:-}
volumes:
- /tmp/omv-scans:/mnt/scans
- /tmp/omv-paperlessmanager:/mnt/data
networks:
- paperless-net
# ─── Frontend (React + nginx) ──────────────────────────────
frontend:
build:
context: ./paperless-frontend
container_name: paperless-frontend
restart: unless-stopped
ports:
- "${FRONTEND_PORT:-8080}:80"
environment:
- VITE_API_URL=${VITE_API_URL:-http://localhost:3100}
- VITE_OIDC_AUTHORITY=${OIDC_ISSUER:-}
- VITE_OIDC_CLIENT_ID=${OIDC_CLIENT_ID:-}
- VITE_OIDC_REDIRECT_URI=${OIDC_REDIRECT_URI:-}
networks:
- paperless-net
volumes:
scanner_data:
networks:
paperless-net:
driver: bridge
File diff suppressed because it is too large Load Diff
+22915
View File
File diff suppressed because one or more lines are too long
+8
View File
@@ -0,0 +1,8 @@
# Offene Punkte für spätere Phasen
Hier sind die Todos, die wir vorerst zurückgestellt haben:
- [ ] **Agrarmonitor-Integration**: API-Anbindung (Base URL, Credentials) implementieren und `AgrarmonitorApiService` erstellen.
- [ ] **Korrespondenten-Sync**: Kunden aus Agrarmonitor abfragen und automatisch Korrespondenten in Paperless anlegen, falls diese nicht existieren (basierend auf der Nummer).
- [ ] **Erweiterte Titel-Logik**: Implementierung der spezifischen Präfixe (ERG, EGU, etc.) basierend auf dem Dokumententyp, falls die automatischen Titel-Templates von Paperless nicht ausreichen.
- [ ] **Vollständige Kunden-Synchronisation**: Regelmäßiger Abgleich der Kundendaten zwischen Agrarmonitor und Paperless.
+409
View File
@@ -0,0 +1,409 @@
{
"name": "Paperlessmanaer",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"dependencies": {
"@vudovn/ag-kit": "^2.0.2"
}
},
"node_modules/@vudovn/ag-kit": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/@vudovn/ag-kit/-/ag-kit-2.0.2.tgz",
"integrity": "sha512-MV+IhmF/C8wtzAjl0ymRY83Ux6kPQ3K2OijRSJNJowkQ/njRtdTWJ2nK9vgpVvYuJ3GM3+XXBrn5AqF8XCkGLg==",
"license": "MIT",
"dependencies": {
"chalk": "^5.6.2",
"commander": "^12.1.0",
"fs-extra": "^11.3.3",
"giget": "^2.0.0",
"ora": "^8.2.0"
},
"bin": {
"ag-kit": "bin/index.js"
},
"engines": {
"node": ">=18"
}
},
"node_modules/ansi-regex": {
"version": "6.2.2",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz",
"integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==",
"license": "MIT",
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/chalk/ansi-regex?sponsor=1"
}
},
"node_modules/chalk": {
"version": "5.6.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz",
"integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==",
"license": "MIT",
"engines": {
"node": "^12.17.0 || ^14.13 || >=16.0.0"
},
"funding": {
"url": "https://github.com/chalk/chalk?sponsor=1"
}
},
"node_modules/citty": {
"version": "0.1.6",
"resolved": "https://registry.npmjs.org/citty/-/citty-0.1.6.tgz",
"integrity": "sha512-tskPPKEs8D2KPafUypv2gxwJP8h/OaJmC82QQGGDQcHvXX43xF2VDACcJVmZ0EuSxkpO9Kc4MlrA3q0+FG58AQ==",
"license": "MIT",
"dependencies": {
"consola": "^3.2.3"
}
},
"node_modules/cli-cursor": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-5.0.0.tgz",
"integrity": "sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==",
"license": "MIT",
"dependencies": {
"restore-cursor": "^5.0.0"
},
"engines": {
"node": ">=18"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/cli-spinners": {
"version": "2.9.2",
"resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.9.2.tgz",
"integrity": "sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==",
"license": "MIT",
"engines": {
"node": ">=6"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/commander": {
"version": "12.1.0",
"resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz",
"integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==",
"license": "MIT",
"engines": {
"node": ">=18"
}
},
"node_modules/consola": {
"version": "3.4.2",
"resolved": "https://registry.npmjs.org/consola/-/consola-3.4.2.tgz",
"integrity": "sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==",
"license": "MIT",
"engines": {
"node": "^14.18.0 || >=16.10.0"
}
},
"node_modules/defu": {
"version": "6.1.4",
"resolved": "https://registry.npmjs.org/defu/-/defu-6.1.4.tgz",
"integrity": "sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg==",
"license": "MIT"
},
"node_modules/emoji-regex": {
"version": "10.6.0",
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz",
"integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==",
"license": "MIT"
},
"node_modules/fs-extra": {
"version": "11.3.4",
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.4.tgz",
"integrity": "sha512-CTXd6rk/M3/ULNQj8FBqBWHYBVYybQ3VPBw0xGKFe3tuH7ytT6ACnvzpIQ3UZtB8yvUKC2cXn1a+x+5EVQLovA==",
"license": "MIT",
"dependencies": {
"graceful-fs": "^4.2.0",
"jsonfile": "^6.0.1",
"universalify": "^2.0.0"
},
"engines": {
"node": ">=14.14"
}
},
"node_modules/get-east-asian-width": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.5.0.tgz",
"integrity": "sha512-CQ+bEO+Tva/qlmw24dCejulK5pMzVnUOFOijVogd3KQs07HnRIgp8TGipvCCRT06xeYEbpbgwaCxglFyiuIcmA==",
"license": "MIT",
"engines": {
"node": ">=18"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/giget": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/giget/-/giget-2.0.0.tgz",
"integrity": "sha512-L5bGsVkxJbJgdnwyuheIunkGatUF/zssUoxxjACCseZYAVbaqdh9Tsmmlkl8vYan09H7sbvKt4pS8GqKLBrEzA==",
"license": "MIT",
"dependencies": {
"citty": "^0.1.6",
"consola": "^3.4.0",
"defu": "^6.1.4",
"node-fetch-native": "^1.6.6",
"nypm": "^0.6.0",
"pathe": "^2.0.3"
},
"bin": {
"giget": "dist/cli.mjs"
}
},
"node_modules/graceful-fs": {
"version": "4.2.11",
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
"integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==",
"license": "ISC"
},
"node_modules/is-interactive": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-2.0.0.tgz",
"integrity": "sha512-qP1vozQRI+BMOPcjFzrjXuQvdak2pHNUMZoeG2eRbiSqyvbEf/wQtEOTOX1guk6E3t36RkaqiSt8A/6YElNxLQ==",
"license": "MIT",
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/is-unicode-supported": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-2.1.0.tgz",
"integrity": "sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ==",
"license": "MIT",
"engines": {
"node": ">=18"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/jsonfile": {
"version": "6.2.0",
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.0.tgz",
"integrity": "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==",
"license": "MIT",
"dependencies": {
"universalify": "^2.0.0"
},
"optionalDependencies": {
"graceful-fs": "^4.1.6"
}
},
"node_modules/log-symbols": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-6.0.0.tgz",
"integrity": "sha512-i24m8rpwhmPIS4zscNzK6MSEhk0DUWa/8iYQWxhffV8jkI4Phvs3F+quL5xvS0gdQR0FyTCMMH33Y78dDTzzIw==",
"license": "MIT",
"dependencies": {
"chalk": "^5.3.0",
"is-unicode-supported": "^1.3.0"
},
"engines": {
"node": ">=18"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/log-symbols/node_modules/is-unicode-supported": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-1.3.0.tgz",
"integrity": "sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ==",
"license": "MIT",
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/mimic-function": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/mimic-function/-/mimic-function-5.0.1.tgz",
"integrity": "sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==",
"license": "MIT",
"engines": {
"node": ">=18"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/node-fetch-native": {
"version": "1.6.7",
"resolved": "https://registry.npmjs.org/node-fetch-native/-/node-fetch-native-1.6.7.tgz",
"integrity": "sha512-g9yhqoedzIUm0nTnTqAQvueMPVOuIY16bqgAJJC8XOOubYFNwz6IER9qs0Gq2Xd0+CecCKFjtdDTMA4u4xG06Q==",
"license": "MIT"
},
"node_modules/nypm": {
"version": "0.6.5",
"resolved": "https://registry.npmjs.org/nypm/-/nypm-0.6.5.tgz",
"integrity": "sha512-K6AJy1GMVyfyMXRVB88700BJqNUkByijGJM8kEHpLdcAt+vSQAVfkWWHYzuRXHSY6xA2sNc5RjTj0p9rE2izVQ==",
"license": "MIT",
"dependencies": {
"citty": "^0.2.0",
"pathe": "^2.0.3",
"tinyexec": "^1.0.2"
},
"bin": {
"nypm": "dist/cli.mjs"
},
"engines": {
"node": ">=18"
}
},
"node_modules/nypm/node_modules/citty": {
"version": "0.2.1",
"resolved": "https://registry.npmjs.org/citty/-/citty-0.2.1.tgz",
"integrity": "sha512-kEV95lFBhQgtogAPlQfJJ0WGVSokvLr/UEoFPiKKOXF7pl98HfUVUD0ejsuTCld/9xH9vogSywZ5KqHzXrZpqg==",
"license": "MIT"
},
"node_modules/onetime": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/onetime/-/onetime-7.0.0.tgz",
"integrity": "sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==",
"license": "MIT",
"dependencies": {
"mimic-function": "^5.0.0"
},
"engines": {
"node": ">=18"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/ora": {
"version": "8.2.0",
"resolved": "https://registry.npmjs.org/ora/-/ora-8.2.0.tgz",
"integrity": "sha512-weP+BZ8MVNnlCm8c0Qdc1WSWq4Qn7I+9CJGm7Qali6g44e/PUzbjNqJX5NJ9ljlNMosfJvg1fKEGILklK9cwnw==",
"license": "MIT",
"dependencies": {
"chalk": "^5.3.0",
"cli-cursor": "^5.0.0",
"cli-spinners": "^2.9.2",
"is-interactive": "^2.0.0",
"is-unicode-supported": "^2.0.0",
"log-symbols": "^6.0.0",
"stdin-discarder": "^0.2.2",
"string-width": "^7.2.0",
"strip-ansi": "^7.1.0"
},
"engines": {
"node": ">=18"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/pathe": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz",
"integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==",
"license": "MIT"
},
"node_modules/restore-cursor": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-5.1.0.tgz",
"integrity": "sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==",
"license": "MIT",
"dependencies": {
"onetime": "^7.0.0",
"signal-exit": "^4.1.0"
},
"engines": {
"node": ">=18"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/signal-exit": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz",
"integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==",
"license": "ISC",
"engines": {
"node": ">=14"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/stdin-discarder": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/stdin-discarder/-/stdin-discarder-0.2.2.tgz",
"integrity": "sha512-UhDfHmA92YAlNnCfhmq0VeNL5bDbiZGg7sZ2IvPsXubGkiNa9EC+tUTsjBRsYUAz87btI6/1wf4XoVvQ3uRnmQ==",
"license": "MIT",
"engines": {
"node": ">=18"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/string-width": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz",
"integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==",
"license": "MIT",
"dependencies": {
"emoji-regex": "^10.3.0",
"get-east-asian-width": "^1.0.0",
"strip-ansi": "^7.1.0"
},
"engines": {
"node": ">=18"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/strip-ansi": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz",
"integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==",
"license": "MIT",
"dependencies": {
"ansi-regex": "^6.2.2"
},
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/chalk/strip-ansi?sponsor=1"
}
},
"node_modules/tinyexec": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.0.4.tgz",
"integrity": "sha512-u9r3uZC0bdpGOXtlxUIdwf9pkmvhqJdrVCH9fapQtgy/OeTTMZ1nqH7agtvEfmGui6e1XxjcdrlxvxJvc3sMqw==",
"license": "MIT",
"engines": {
"node": ">=18"
}
},
"node_modules/universalify": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz",
"integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==",
"license": "MIT",
"engines": {
"node": ">= 10.0.0"
}
}
}
}
+5
View File
@@ -0,0 +1,5 @@
{
"dependencies": {
"@vudovn/ag-kit": "^2.0.2"
}
}
+4
View File
@@ -0,0 +1,4 @@
node_modules
dist
.env
.git
+4
View File
@@ -0,0 +1,4 @@
{
"singleQuote": true,
"trailingComma": "all"
}
+18
View File
@@ -0,0 +1,18 @@
# Backend
FROM node:20-alpine AS builder
WORKDIR /app
COPY package*.json ./
RUN npm ci
COPY . .
RUN npm run build
# Production
FROM node:20-alpine
RUN apk add --no-cache ghostscript imagemagick
WORKDIR /app
COPY --from=builder /app/dist ./dist
COPY --from=builder /app/node_modules ./node_modules
COPY --from=builder /app/package.json ./
EXPOSE 3100
CMD ["node", "dist/main.js"]
+98
View File
@@ -0,0 +1,98 @@
<p align="center">
<a href="http://nestjs.com/" target="blank"><img src="https://nestjs.com/img/logo-small.svg" width="120" alt="Nest Logo" /></a>
</p>
[circleci-image]: https://img.shields.io/circleci/build/github/nestjs/nest/master?token=abc123def456
[circleci-url]: https://circleci.com/gh/nestjs/nest
<p align="center">A progressive <a href="http://nodejs.org" target="_blank">Node.js</a> framework for building efficient and scalable server-side applications.</p>
<p align="center">
<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/v/@nestjs/core.svg" alt="NPM Version" /></a>
<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/l/@nestjs/core.svg" alt="Package License" /></a>
<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/dm/@nestjs/common.svg" alt="NPM Downloads" /></a>
<a href="https://circleci.com/gh/nestjs/nest" target="_blank"><img src="https://img.shields.io/circleci/build/github/nestjs/nest/master" alt="CircleCI" /></a>
<a href="https://discord.gg/G7Qnnhy" target="_blank"><img src="https://img.shields.io/badge/discord-online-brightgreen.svg" alt="Discord"/></a>
<a href="https://opencollective.com/nest#backer" target="_blank"><img src="https://opencollective.com/nest/backers/badge.svg" alt="Backers on Open Collective" /></a>
<a href="https://opencollective.com/nest#sponsor" target="_blank"><img src="https://opencollective.com/nest/sponsors/badge.svg" alt="Sponsors on Open Collective" /></a>
<a href="https://paypal.me/kamilmysliwiec" target="_blank"><img src="https://img.shields.io/badge/Donate-PayPal-ff3f59.svg" alt="Donate us"/></a>
<a href="https://opencollective.com/nest#sponsor" target="_blank"><img src="https://img.shields.io/badge/Support%20us-Open%20Collective-41B883.svg" alt="Support us"></a>
<a href="https://twitter.com/nestframework" target="_blank"><img src="https://img.shields.io/twitter/follow/nestframework.svg?style=social&label=Follow" alt="Follow us on Twitter"></a>
</p>
<!--[![Backers on Open Collective](https://opencollective.com/nest/backers/badge.svg)](https://opencollective.com/nest#backer)
[![Sponsors on Open Collective](https://opencollective.com/nest/sponsors/badge.svg)](https://opencollective.com/nest#sponsor)-->
## Description
[Nest](https://github.com/nestjs/nest) framework TypeScript starter repository.
## Project setup
```bash
$ npm install
```
## Compile and run the project
```bash
# development
$ npm run start
# watch mode
$ npm run start:dev
# production mode
$ npm run start:prod
```
## Run tests
```bash
# unit tests
$ npm run test
# e2e tests
$ npm run test:e2e
# test coverage
$ npm run test:cov
```
## Deployment
When you're ready to deploy your NestJS application to production, there are some key steps you can take to ensure it runs as efficiently as possible. Check out the [deployment documentation](https://docs.nestjs.com/deployment) for more information.
If you are looking for a cloud-based platform to deploy your NestJS application, check out [Mau](https://mau.nestjs.com), our official platform for deploying NestJS applications on AWS. Mau makes deployment straightforward and fast, requiring just a few simple steps:
```bash
$ npm install -g @nestjs/mau
$ mau deploy
```
With Mau, you can deploy your application in just a few clicks, allowing you to focus on building features rather than managing infrastructure.
## Resources
Check out a few resources that may come in handy when working with NestJS:
- Visit the [NestJS Documentation](https://docs.nestjs.com) to learn more about the framework.
- For questions and support, please visit our [Discord channel](https://discord.gg/G7Qnnhy).
- To dive deeper and get more hands-on experience, check out our official video [courses](https://courses.nestjs.com/).
- Deploy your application to AWS with the help of [NestJS Mau](https://mau.nestjs.com) in just a few clicks.
- Visualize your application graph and interact with the NestJS application in real-time using [NestJS Devtools](https://devtools.nestjs.com).
- Need help with your project (part-time to full-time)? Check out our official [enterprise support](https://enterprise.nestjs.com).
- To stay in the loop and get updates, follow us on [X](https://x.com/nestframework) and [LinkedIn](https://linkedin.com/company/nestjs).
- Looking for a job, or have a job to offer? Check out our official [Jobs board](https://jobs.nestjs.com).
## Support
Nest is an MIT-licensed open source project. It can grow thanks to the sponsors and support by the amazing backers. If you'd like to join them, please [read more here](https://docs.nestjs.com/support).
## Stay in touch
- Author - [Kamil Myśliwiec](https://twitter.com/kammysliwiec)
- Website - [https://nestjs.com](https://nestjs.com/)
- Twitter - [@nestframework](https://twitter.com/nestframework)
## License
Nest is [MIT licensed](https://github.com/nestjs/nest/blob/master/LICENSE).
View File
+35
View File
@@ -0,0 +1,35 @@
// @ts-check
import eslint from '@eslint/js';
import eslintPluginPrettierRecommended from 'eslint-plugin-prettier/recommended';
import globals from 'globals';
import tseslint from 'typescript-eslint';
export default tseslint.config(
{
ignores: ['eslint.config.mjs'],
},
eslint.configs.recommended,
...tseslint.configs.recommendedTypeChecked,
eslintPluginPrettierRecommended,
{
languageOptions: {
globals: {
...globals.node,
...globals.jest,
},
sourceType: 'commonjs',
parserOptions: {
projectService: true,
tsconfigRootDir: import.meta.dirname,
},
},
},
{
rules: {
'@typescript-eslint/no-explicit-any': 'off',
'@typescript-eslint/no-floating-promises': 'warn',
'@typescript-eslint/no-unsafe-argument': 'warn',
"prettier/prettier": ["error", { endOfLine: "auto" }],
},
},
);
+41
View File
@@ -0,0 +1,41 @@
-- ============================================================
-- Postprocessing Redesign Migration
-- Ausführen: mysql -h HOST -u USER -p DATABASE < migration.sql
-- ============================================================
-- 1) Postprocessings: Statische Filter → JSON
ALTER TABLE Postprocessings ADD COLUMN FilterJson JSON NOT NULL DEFAULT (JSON_OBJECT('combinator', 'AND', 'rules', JSON_ARRAY()));
ALTER TABLE Postprocessings DROP COLUMN IF EXISTS DocumentTypeId;
ALTER TABLE Postprocessings DROP COLUMN IF EXISTS CorrespondentId;
ALTER TABLE Postprocessings DROP COLUMN IF EXISTS OwnerId;
ALTER TABLE Postprocessings DROP COLUMN IF EXISTS TagId;
ALTER TABLE Postprocessings DROP COLUMN IF EXISTS MandantId;
-- 2) PostprocessingActions: Content text → JSON
ALTER TABLE PostprocessingActions MODIFY COLUMN Content JSON NOT NULL;
-- 3) ExportTargets (NEU)
CREATE TABLE IF NOT EXISTS ExportTargets (
Id INT AUTO_INCREMENT PRIMARY KEY,
Name VARCHAR(100) NOT NULL,
Protocol VARCHAR(20) NOT NULL COMMENT 'ftp | webdav',
Host VARCHAR(255) NOT NULL,
Port INT NULL,
Username VARCHAR(100) NULL,
Password VARCHAR(255) NULL,
RemotePath VARCHAR(500) NULL,
IsActive TINYINT NOT NULL DEFAULT 1
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- 4) PostprocessingLogs (NEU)
CREATE TABLE IF NOT EXISTS PostprocessingLogs (
Id INT AUTO_INCREMENT PRIMARY KEY,
PostprocessingId INT NOT NULL,
ActionId INT NULL,
DocumentId INT NOT NULL,
Status VARCHAR(20) NOT NULL COMMENT 'success | error | skipped',
Message TEXT NULL,
CreatedAt DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
INDEX idx_pp_doc (PostprocessingId, DocumentId),
INDEX idx_created (CreatedAt)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+8
View File
@@ -0,0 +1,8 @@
{
"$schema": "https://json.schemastore.org/nest-cli",
"collection": "@nestjs/schematics",
"sourceRoot": "src",
"compilerOptions": {
"deleteOutDir": true
}
}
+12629
View File
File diff suppressed because it is too large Load Diff
+100
View File
@@ -0,0 +1,100 @@
{
"name": "paperless-backend",
"version": "0.0.1",
"description": "",
"author": "",
"private": true,
"license": "UNLICENSED",
"scripts": {
"build": "nest build",
"format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
"start": "nest start",
"start:dev": "nest start --watch",
"start:debug": "nest start --debug --watch",
"start:prod": "node dist/main",
"lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
"test": "jest",
"test:watch": "jest --watch",
"test:cov": "jest --coverage",
"test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
"test:e2e": "jest --config ./test/jest-e2e.json"
},
"dependencies": {
"@nestjs/common": "^11.0.1",
"@nestjs/config": "^4.0.3",
"@nestjs/core": "^11.0.1",
"@nestjs/passport": "^11.0.5",
"@nestjs/platform-express": "^11.0.1",
"@nestjs/schedule": "^6.1.1",
"@nestjs/typeorm": "^11.0.0",
"@types/form-data": "^2.2.1",
"@types/passport-jwt": "^4.0.1",
"@types/uuid": "^10.0.0",
"axios": "^1.14.0",
"basic-ftp": "^5.2.1",
"chokidar": "^4.0.3",
"form-data": "^4.0.5",
"imapflow": "^1.3.2",
"jsqr": "^1.4.0",
"jwks-rsa": "^4.0.1",
"mailparser": "^3.9.8",
"mysql2": "^3.20.0",
"nodemailer": "^8.0.5",
"passport": "^0.7.0",
"passport-jwt": "^4.0.1",
"pdf-lib": "^1.17.1",
"qrcode": "^1.5.4",
"reflect-metadata": "^0.2.2",
"rxjs": "^7.8.1",
"sharp": "^0.34.5",
"typeorm": "^0.3.28",
"uuid": "^13.0.0",
"webdav": "^5.9.0"
},
"devDependencies": {
"@eslint/eslintrc": "^3.2.0",
"@eslint/js": "^9.18.0",
"@nestjs/cli": "^11.0.0",
"@nestjs/schematics": "^11.0.0",
"@nestjs/testing": "^11.0.1",
"@types/express": "^5.0.0",
"@types/jest": "^30.0.0",
"@types/mailparser": "^3.4.6",
"@types/multer": "^2.1.0",
"@types/node": "^22.10.7",
"@types/nodemailer": "^8.0.0",
"@types/qrcode": "^1.5.6",
"@types/supertest": "^6.0.2",
"eslint": "^9.18.0",
"eslint-config-prettier": "^10.0.1",
"eslint-plugin-prettier": "^5.2.2",
"globals": "^16.0.0",
"jest": "^30.0.0",
"prettier": "^3.4.2",
"source-map-support": "^0.5.21",
"supertest": "^7.0.0",
"ts-jest": "^29.2.5",
"ts-loader": "^9.5.2",
"ts-node": "^10.9.2",
"tsconfig-paths": "^4.2.0",
"typescript": "^5.7.3",
"typescript-eslint": "^8.20.0"
},
"jest": {
"moduleFileExtensions": [
"js",
"json",
"ts"
],
"rootDir": "src",
"testRegex": ".*\\.spec\\.ts$",
"transform": {
"^.+\\.(t|j)s$": "ts-jest"
},
"collectCoverageFrom": [
"**/*.(t|j)s"
],
"coverageDirectory": "../coverage",
"testEnvironment": "node"
}
}
@@ -0,0 +1,22 @@
import { Test, TestingModule } from '@nestjs/testing';
import { AppController } from './app.controller';
import { AppService } from './app.service';
describe('AppController', () => {
let appController: AppController;
beforeEach(async () => {
const app: TestingModule = await Test.createTestingModule({
controllers: [AppController],
providers: [AppService],
}).compile();
appController = app.get<AppController>(AppController);
});
describe('root', () => {
it('should return "Hello World!"', () => {
expect(appController.getHello()).toBe('Hello World!');
});
});
});
+12
View File
@@ -0,0 +1,12 @@
import { Controller, Get } from '@nestjs/common';
import { AppService } from './app.service';
@Controller()
export class AppController {
constructor(private readonly appService: AppService) {}
@Get()
getHello(): string {
return this.appService.getHello();
}
}
+48
View File
@@ -0,0 +1,48 @@
import { Module } from '@nestjs/common';
import { ConfigModule } from '@nestjs/config';
import { ScheduleModule } from '@nestjs/schedule';
import { DatabaseModule } from './database/database.module';
import { AuthModule } from './auth/auth.module';
import { ScannerModule } from './scanner/scanner.module';
import { PaperlessModule } from './paperless/paperless.module';
import { PreprocessingModule } from './preprocessing/preprocessing.module';
import { PostprocessingModule } from './postprocessing/postprocessing.module';
import { WebhookModule } from './webhook/webhook.module';
import { InboxModule } from './inbox/inbox.module';
import { EmailModule } from './email/email.module';
import { EmailDownloadModule } from './email-download/email-download.module';
import { SettingsModule } from './settings/settings.module';
import { KontonummernModule } from './kontonummern/kontonummern.module';
import { StatsModule } from './stats/stats.module';
import { BarcodeModule } from './barcode/barcode.module';
import { InboxPostprocessorModule } from './inbox-postprocessor/inbox-postprocessor.module';
import * as path from 'path';
@Module({
imports: [
ConfigModule.forRoot({
isGlobal: true,
envFilePath: [
path.resolve(__dirname, '../../.env'), // Root .env (zentral)
path.resolve(__dirname, '../.env'), // Lokale .env (Fallback)
],
}),
ScheduleModule.forRoot(),
DatabaseModule,
AuthModule,
ScannerModule,
PaperlessModule,
PreprocessingModule,
PostprocessingModule,
WebhookModule,
InboxModule,
EmailModule,
EmailDownloadModule,
SettingsModule,
KontonummernModule,
StatsModule,
BarcodeModule,
InboxPostprocessorModule,
],
})
export class AppModule {}
+8
View File
@@ -0,0 +1,8 @@
import { Injectable } from '@nestjs/common';
@Injectable()
export class AppService {
getHello(): string {
return 'Hello World!';
}
}
@@ -0,0 +1,37 @@
import { CanActivate, ExecutionContext, Injectable, UnauthorizedException } from '@nestjs/common';
import { ApiKeysService } from './api-keys.service';
@Injectable()
export class ApiKeyGuard implements CanActivate {
constructor(private readonly apiKeysService: ApiKeysService) {}
async canActivate(context: ExecutionContext): Promise<boolean> {
const request = context.switchToHttp().getRequest();
// Check header (X-API-Key)
let apiKey = request.headers['x-api-key'] || request.headers['X-API-Key'];
// Fallback to query parameter (apiKey)
if (!apiKey) {
apiKey = request.query['apiKey'];
}
if (!apiKey) {
throw new UnauthorizedException('API Key missing');
}
try {
const keyEntry = await this.apiKeysService.validateKey(apiKey as string);
// Attach metadata to request if needed later
request.apiKeyMetadata = {
id: keyEntry.id,
name: keyEntry.name,
};
return true;
} catch (err) {
throw new UnauthorizedException(err.message || 'Invalid API Key');
}
}
}
@@ -0,0 +1,26 @@
import { Controller, Get, Post, Delete, Body, Param, UseGuards } from '@nestjs/common';
import { ApiKeysService } from './api-keys.service';
import { JwtAuthGuard } from './jwt-auth.guard';
@Controller('api/api-keys')
@UseGuards(JwtAuthGuard)
export class ApiKeysController {
constructor(private readonly apiKeysService: ApiKeysService) {}
@Post()
async create(@Body() body: { name: string; expiresDays?: number }) {
// Note: The plainKey is only returned here once.
return this.apiKeysService.createApiKey(body.name, body.expiresDays);
}
@Get()
async findAll() {
// Note: This returns hashed keys and prefixes, not the plain keys.
return this.apiKeysService.listKeys();
}
@Delete(':id')
async remove(@Param('id') id: string) {
return this.apiKeysService.deleteKey(id);
}
}
@@ -0,0 +1,71 @@
import { Injectable, UnauthorizedException } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { ApiKey } from '../database/entities/api-key.entity';
import * as crypto from 'crypto';
@Injectable()
export class ApiKeysService {
constructor(
@InjectRepository(ApiKey)
private readonly apiKeyRepo: Repository<ApiKey>,
) {}
async createApiKey(name: string, expiresDays?: number): Promise<{ plainKey: string; entity: ApiKey }> {
const prefix = 'pm_';
const randomPart = crypto.randomBytes(24).toString('hex'); // 48 chars hex
const plainKey = `${prefix}${randomPart}`;
const keyHash = this.hashKey(plainKey);
const apiKey = this.apiKeyRepo.create({
name,
keyPrefix: prefix,
keyHash,
expiresAt: expiresDays ? new Date(Date.now() + expiresDays * 24 * 60 * 60 * 1000) : null,
});
const savedKey = await this.apiKeyRepo.save(apiKey);
return {
plainKey,
entity: savedKey,
};
}
async validateKey(plainKey: string): Promise<ApiKey> {
const keyHash = this.hashKey(plainKey);
const apiKey = await this.apiKeyRepo.findOne({
where: { keyHash },
});
if (!apiKey) {
throw new UnauthorizedException('Invalid API Key');
}
if (apiKey.expiresAt && apiKey.expiresAt < new Date()) {
throw new UnauthorizedException('API Key has expired');
}
// Update last used timestamp (async, don't wait for it to return response faster)
apiKey.lastUsedAt = new Date();
this.apiKeyRepo.save(apiKey).catch(err => console.error('Error updating lastUsedAt:', err));
return apiKey;
}
async listKeys(): Promise<ApiKey[]> {
return this.apiKeyRepo.find({
order: { createdAt: 'DESC' },
});
}
async deleteKey(id: string): Promise<void> {
await this.apiKeyRepo.delete(id);
}
private hashKey(key: string): string {
return crypto.createHash('sha256').update(key).digest('hex');
}
}
+38
View File
@@ -0,0 +1,38 @@
import { Module } from '@nestjs/common';
import { PassportModule } from '@nestjs/passport';
import { TypeOrmModule } from '@nestjs/typeorm';
import { APP_GUARD } from '@nestjs/core';
import { JwtStrategy } from './jwt.strategy';
import { JwtAuthGuard } from './jwt-auth.guard';
import { ApiKeysService } from './api-keys.service';
import { ApiKeysController } from './api-keys.controller';
import { ApiKeyGuard } from './api-key.guard';
import { JwtOrApiKeyGuard } from './jwt-or-apikey.guard';
import { ApiKey } from '../database/entities/api-key.entity';
import { PermissionsGuard } from './permissions.guard';
@Module({
imports: [
PassportModule.register({ defaultStrategy: 'jwt' }),
TypeOrmModule.forFeature([ApiKey]),
],
controllers: [ApiKeysController],
providers: [
JwtStrategy,
JwtAuthGuard,
ApiKeysService,
ApiKeyGuard,
JwtOrApiKeyGuard,
PermissionsGuard,
{
provide: APP_GUARD,
useClass: JwtAuthGuard,
},
{
provide: APP_GUARD,
useClass: PermissionsGuard,
},
],
exports: [PassportModule, ApiKeysService, ApiKeyGuard, JwtAuthGuard, JwtOrApiKeyGuard, PermissionsGuard, TypeOrmModule],
})
export class AuthModule {}
@@ -0,0 +1,21 @@
import { Injectable, ExecutionContext } from '@nestjs/common';
import { AuthGuard } from '@nestjs/passport';
import { Reflector } from '@nestjs/core';
import { IS_PUBLIC_KEY } from './public.decorator';
@Injectable()
export class JwtAuthGuard extends AuthGuard('jwt') {
constructor(private reflector: Reflector) {
super();
}
canActivate(context: ExecutionContext) {
const isPublic = this.reflector.getAllAndOverride<boolean>(IS_PUBLIC_KEY, [
context.getHandler(),
context.getClass(),
]);
if (isPublic) return true;
return super.canActivate(context);
}
}
@@ -0,0 +1,31 @@
import { CanActivate, ExecutionContext, Injectable } from '@nestjs/common';
import { JwtAuthGuard } from './jwt-auth.guard';
import { ApiKeyGuard } from './api-key.guard';
import { lastValueFrom, isObservable } from 'rxjs';
/**
* Combined guard that accepts either a valid JWT Bearer token
* or a valid API key (X-API-Key header / apiKey query param).
* Tries JWT first, falls back to API key.
*/
@Injectable()
export class JwtOrApiKeyGuard implements CanActivate {
constructor(
private readonly jwtGuard: JwtAuthGuard,
private readonly apiKeyGuard: ApiKeyGuard,
) {}
async canActivate(context: ExecutionContext): Promise<boolean> {
// Try JWT first
try {
const result = this.jwtGuard.canActivate(context);
const jwtOk = isObservable(result) ? await lastValueFrom(result) : await result;
if (jwtOk) return true;
} catch {
// JWT failed, try API key
}
// Fall back to API key
return this.apiKeyGuard.canActivate(context);
}
}
@@ -0,0 +1,38 @@
import { Injectable } from '@nestjs/common';
import { PassportStrategy } from '@nestjs/passport';
import { Strategy, ExtractJwt } from 'passport-jwt';
import { ConfigService } from '@nestjs/config';
import { passportJwtSecret } from 'jwks-rsa';
import { mapGroupsToPermissions } from './permissions.enum';
@Injectable()
export class JwtStrategy extends PassportStrategy(Strategy, 'jwt') {
constructor(configService: ConfigService) {
const issuer = configService.get<string>('OIDC_ISSUER', '');
super({
jwtFromRequest: ExtractJwt.fromAuthHeaderAsBearerToken(),
ignoreExpiration: false,
issuer,
algorithms: ['RS256'],
secretOrKeyProvider: passportJwtSecret({
cache: true,
rateLimit: true,
jwksRequestsPerMinute: 5,
jwksUri: `${issuer.endsWith('/') ? issuer.slice(0, -1) : issuer}/jwks/`,
}),
});
}
validate(payload: any): { userId: string; email: string; name: string; preferredUsername: string | null; groups: string[]; permissions: any[] } {
const groups = payload.groups || [];
return {
userId: payload.sub,
email: payload.email,
name: payload.name || payload.preferred_username,
preferredUsername: payload.preferred_username ?? null,
groups: groups,
permissions: mapGroupsToPermissions(groups),
};
}
}
@@ -0,0 +1,5 @@
import { SetMetadata } from '@nestjs/common';
import { Permission } from './permissions.enum';
export const PERMISSIONS_KEY = 'permissions';
export const RequirePermissions = (...permissions: Permission[]) => SetMetadata(PERMISSIONS_KEY, permissions);
@@ -0,0 +1,35 @@
export const Permission = {
MANAGE_ALL: 'MANAGE_ALL',
PROCESS_MANUALLY: 'PROCESS_MANUALLY',
VIEW_MAIL: 'VIEW_MAIL',
VIEW_INBOX: 'VIEW_INBOX',
VIEW_SCANNER: 'VIEW_SCANNER',
MANAGE_SETTINGS: 'MANAGE_SETTINGS',
} as const;
export type Permission = typeof Permission[keyof typeof Permission];
export function mapGroupsToPermissions(groups: string[] | undefined | null): Permission[] {
const permissions = new Set<Permission>();
if (!groups || !Array.isArray(groups)) {
return [];
}
if (groups.includes('PM_Admin')) {
permissions.add(Permission.MANAGE_ALL);
permissions.add(Permission.PROCESS_MANUALLY);
permissions.add(Permission.VIEW_MAIL);
permissions.add(Permission.VIEW_INBOX);
permissions.add(Permission.VIEW_SCANNER);
permissions.add(Permission.MANAGE_SETTINGS);
return Array.from(permissions);
}
if (groups.includes('PM_Belege')) permissions.add(Permission.PROCESS_MANUALLY);
if (groups.includes('PM_Maileingang')) permissions.add(Permission.VIEW_MAIL);
if (groups.includes('PM_Posteingang')) permissions.add(Permission.VIEW_INBOX);
if (groups.includes('PM_Scanner')) permissions.add(Permission.VIEW_SCANNER);
return Array.from(permissions);
}
@@ -0,0 +1,40 @@
import { Injectable, CanActivate, ExecutionContext } from '@nestjs/common';
import { Reflector } from '@nestjs/core';
import { PERMISSIONS_KEY } from './permissions.decorator';
import { Permission } from './permissions.enum';
@Injectable()
export class PermissionsGuard implements CanActivate {
constructor(private reflector: Reflector) {}
canActivate(context: ExecutionContext): boolean {
const requiredPermissions = this.reflector.getAllAndOverride<Permission[]>(PERMISSIONS_KEY, [
context.getHandler(),
context.getClass(),
]);
if (!requiredPermissions) {
return true;
}
const { user } = context.switchToHttp().getRequest();
// Let API Key requests bypass the permissions check for now, unless explicitly denied.
// Usually API keys have different scopes, but assuming they act as Admins for automated uploads.
if (user && user.apiKey) {
return true;
}
if (!user || !user.permissions) {
return false;
}
const userPermissions = user.permissions as Permission[];
if (userPermissions.includes(Permission.MANAGE_ALL)) {
return true;
}
return requiredPermissions.some((permission) => userPermissions.includes(permission));
}
}
@@ -0,0 +1,4 @@
import { SetMetadata } from '@nestjs/common';
export const IS_PUBLIC_KEY = 'isPublic';
export const Public = () => SetMetadata(IS_PUBLIC_KEY, true);
@@ -0,0 +1,218 @@
import { Injectable, Logger, OnApplicationBootstrap } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import * as fs from 'fs/promises';
import { PdfService } from '../preprocessing/pdf.service';
import { QrCodeService } from '../preprocessing/qr-code.service';
import {
BarcodeTemplate,
type BarcodeActionType,
} from '../database/entities/barcode-template.entity';
import { InboxDocument, type StoredQrCode } from '../database/entities/inbox-document.entity';
import { PageCacheService } from './page-cache.service';
export interface MatchedBarcode {
page: number;
value: string;
templateId: number | null;
templateName: string | null;
splitBefore: boolean;
actions: BarcodeActionType[];
}
@Injectable()
export class BarcodeScannerService implements OnApplicationBootstrap {
private readonly logger = new Logger(BarcodeScannerService.name);
private templatesCache: BarcodeTemplate[] | null = null;
constructor(
private readonly pdfService: PdfService,
private readonly qrCodeService: QrCodeService,
private readonly pageCache: PageCacheService,
@InjectRepository(BarcodeTemplate)
private readonly templateRepo: Repository<BarcodeTemplate>,
@InjectRepository(InboxDocument)
private readonly documentRepo: Repository<InboxDocument>,
) {}
async onApplicationBootstrap(): Promise<void> {
await this.migrateLegacySplitBefore();
}
invalidateTemplates(): void {
this.templatesCache = null;
}
private async migrateLegacySplitBefore(): Promise<void> {
let rows: BarcodeTemplate[];
try {
rows = await this.templateRepo.find();
} catch (err: any) {
this.logger.warn(`Template-Migration: Query fehlgeschlagen: ${err.message}`);
return;
}
let migrated = 0;
for (const tpl of rows) {
const actions = (tpl.Actions ?? []) as string[];
if (actions.includes('SPLIT_BEFORE')) {
tpl.SplitBefore = true;
tpl.Actions = actions.filter((a) => a !== 'SPLIT_BEFORE') as BarcodeActionType[];
await this.templateRepo.save(tpl);
migrated += 1;
}
}
if (migrated > 0) {
this.logger.log(`Template-Migration: ${migrated} Vorlage(n) auf SplitBefore-Flag umgestellt`);
}
}
/**
* Rendert alle Seiten, extrahiert QR-Codes, persistiert Page-Cache + DB-Row.
* Wird nach dem Move aus dem Watcher und beim Backfill aufgerufen.
*/
async scanAndMatch(doc: InboxDocument): Promise<MatchedBarcode[]> {
const pdfPath = this.pageCache.documentPdfPath(doc.Id);
const { qrCodes, pageCount } = await this.performScan(doc.Id, pdfPath);
doc.QrCodes = qrCodes;
doc.PageCount = pageCount;
try {
await this.documentRepo.save(doc);
} catch (err: any) {
this.logger.warn(`Scan-Ergebnis konnte nicht gespeichert werden (${doc.Id}): ${err.message}`);
}
return this.matchTemplates(qrCodes);
}
/**
* Scannt nur, wenn die Row noch keine Seitenanzahl hat (= noch nie gescannt).
*/
async ensureScanned(doc: InboxDocument): Promise<boolean> {
if (doc.PageCount > 0) return false;
await this.scanAndMatch(doc);
return true;
}
/**
* Read-only: mapped die persistierten QR-Codes auf MatchedBarcodes.
*/
async getMatched(doc: InboxDocument): Promise<MatchedBarcode[]> {
return this.matchTemplates(doc.QrCodes ?? []);
}
private async matchTemplates(qrCodes: StoredQrCode[]): Promise<MatchedBarcode[]> {
if (qrCodes.length === 0) return [];
const templates = await this.getTemplates();
return qrCodes.map((qr) => {
const tpl = this.firstMatch(qr.value, templates);
return {
page: qr.page,
value: qr.value,
templateId: tpl?.Id ?? null,
templateName: tpl?.Name ?? null,
splitBefore: tpl?.SplitBefore ?? false,
actions: tpl?.Actions ?? [],
};
});
}
private firstMatch(value: string, templates: BarcodeTemplate[]): BarcodeTemplate | null {
for (const tpl of templates) {
try {
const re = new RegExp(tpl.Regex);
if (re.test(value)) return tpl;
} catch {
// ignore invalid regex
}
}
return null;
}
private async getTemplates(): Promise<BarcodeTemplate[]> {
if (!this.templatesCache) {
this.templatesCache = await this.templateRepo.find({ order: { Id: 'ASC' } });
}
return this.templatesCache;
}
private async performScan(
documentId: string,
pdfPath: string,
): Promise<{ qrCodes: StoredQrCode[]; pageCount: number }> {
let images: string[] = [];
try {
images = await this.pdfService.pdfToImages(pdfPath, 400);
const qrCodes: StoredQrCode[] = [];
const templates = await this.getTemplates();
for (let i = 0; i < images.length; i++) {
try {
const buffer = await fs.readFile(images[i]);
const qrs = await this.qrCodeService.extractFromImage(buffer);
// Nur QR-Codes speichern, die zu einer Eingangsdokumentart passen.
// Mehrere passende QRs pro Seite werden alle übernommen.
for (const qr of qrs) {
if (this.firstMatch(qr.data, templates)) {
qrCodes.push({ page: i + 1, value: qr.data });
}
}
} catch (err: any) {
this.logger.warn(`QR-Scan fehlgeschlagen (${pdfPath}, Seite ${i + 1}): ${err.message}`);
}
}
await this.pageCache.clear(documentId);
await this.pageCache.generate(documentId, images);
return { qrCodes, pageCount: images.length };
} catch (err: any) {
this.logger.warn(`Kein QR-Scan möglich für ${pdfPath}: ${err.message}`);
return { qrCodes: [], pageCount: 0 };
} finally {
await this.pdfService.cleanup(images);
}
}
/**
* Rescannt alle Inbox-Dokumente — wird nach Änderungen an Eingangsdokumentarten aufgerufen.
* Läuft sequenziell, um PDF-Rendering nicht zu überlasten. Fire-and-forget vom Caller.
*/
async rescanAll(): Promise<{ scanned: number; failed: number }> {
this.invalidateTemplates();
let docs: InboxDocument[];
try {
docs = await this.documentRepo.find();
} catch (err: any) {
this.logger.warn(`Rescan: DB-Query fehlgeschlagen: ${err.message}`);
return { scanned: 0, failed: 0 };
}
if (docs.length === 0) return { scanned: 0, failed: 0 };
this.logger.log(`Rescan: starte Neuerfassung für ${docs.length} Inbox-Dokument(e)`);
let scanned = 0;
let failed = 0;
for (const doc of docs) {
try {
const pdfPath = this.pageCache.documentPdfPath(doc.Id);
try {
await fs.access(pdfPath);
} catch {
this.logger.warn(`Rescan: PDF fehlt für ${doc.Id} (${pdfPath})`);
failed++;
continue;
}
const { qrCodes, pageCount } = await this.performScan(doc.Id, pdfPath);
doc.QrCodes = qrCodes;
doc.PageCount = pageCount;
await this.documentRepo.save(doc);
scanned++;
} catch (err: any) {
this.logger.warn(`Rescan fehlgeschlagen für ${doc.Id}: ${err.message}`);
failed++;
}
}
this.logger.log(`Rescan abgeschlossen: ${scanned} ok, ${failed} fehlgeschlagen`);
return { scanned, failed };
}
}
@@ -0,0 +1,135 @@
import {
Body,
Controller,
Delete,
Get,
Logger,
Param,
ParseIntPipe,
Post,
Put,
BadRequestException,
NotFoundException,
} from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { BarcodeTemplate, type BarcodeActionType } from '../database/entities/barcode-template.entity';
import { RequirePermissions } from '../auth/permissions.decorator';
import { Permission } from '../auth/permissions.enum';
import { BarcodeScannerService } from './barcode-scanner.service';
const VALID_ACTIONS: BarcodeActionType[] = ['SEND_TO_PAPERLESS', 'SEND_BY_EMAIL'];
interface UpsertDto {
Name?: string;
Regex?: string;
SplitBefore?: boolean;
DateinameTemplate?: string | null;
Actions?: BarcodeActionType[];
}
function validate(dto: UpsertDto, partial = false): void {
if (!partial || dto.Name !== undefined) {
if (typeof dto.Name !== 'string' || !dto.Name.trim()) {
throw new BadRequestException('Name ist erforderlich');
}
}
if (!partial || dto.Regex !== undefined) {
if (typeof dto.Regex !== 'string' || !dto.Regex.trim()) {
throw new BadRequestException('Regex ist erforderlich');
}
try {
new RegExp(dto.Regex);
} catch {
throw new BadRequestException('Regex ist ungültig');
}
}
if (dto.SplitBefore !== undefined && typeof dto.SplitBefore !== 'boolean') {
throw new BadRequestException('SplitBefore muss ein Boolean sein');
}
if (dto.DateinameTemplate !== undefined && dto.DateinameTemplate !== null &&
typeof dto.DateinameTemplate !== 'string') {
throw new BadRequestException('DateinameTemplate muss ein String sein');
}
if (!partial || dto.Actions !== undefined) {
if (!Array.isArray(dto.Actions)) {
throw new BadRequestException('Actions muss eine Liste sein');
}
for (const a of dto.Actions) {
if (!VALID_ACTIONS.includes(a)) {
throw new BadRequestException(`Unbekannte Aktion: ${a}`);
}
}
}
}
@Controller('api/barcode-templates')
@RequirePermissions(Permission.MANAGE_SETTINGS)
export class BarcodeTemplatesController {
private readonly logger = new Logger(BarcodeTemplatesController.name);
constructor(
@InjectRepository(BarcodeTemplate)
private readonly repo: Repository<BarcodeTemplate>,
private readonly scanner: BarcodeScannerService,
) {}
private triggerRescan(): void {
this.scanner.rescanAll().catch((err) => {
this.logger.error(`Rescan nach Template-Änderung fehlgeschlagen: ${err.message}`);
});
}
@Get()
async list() {
return this.repo.find({ order: { Id: 'ASC' } });
}
@Post()
async create(@Body() dto: UpsertDto) {
validate(dto);
const entity = this.repo.create({
Name: dto.Name!.trim(),
Regex: dto.Regex!,
SplitBefore: dto.SplitBefore ?? false,
DateinameTemplate: dto.DateinameTemplate ?? null,
Actions: dto.Actions!,
});
const saved = await this.repo.save(entity);
this.scanner.invalidateTemplates();
this.triggerRescan();
return saved;
}
@Put(':id')
async update(@Param('id', ParseIntPipe) id: number, @Body() dto: UpsertDto) {
validate(dto, true);
const existing = await this.repo.findOneBy({ Id: id });
if (!existing) throw new NotFoundException('Vorlage nicht gefunden');
const regexChanged = dto.Regex !== undefined && dto.Regex !== existing.Regex;
if (dto.Name !== undefined) existing.Name = dto.Name.trim();
if (dto.Regex !== undefined) existing.Regex = dto.Regex;
if (dto.SplitBefore !== undefined) existing.SplitBefore = dto.SplitBefore;
if (dto.DateinameTemplate !== undefined) existing.DateinameTemplate = dto.DateinameTemplate ?? null;
if (dto.Actions !== undefined) existing.Actions = dto.Actions;
const saved = await this.repo.save(existing);
this.scanner.invalidateTemplates();
if (regexChanged) {
// Nur bei Regex-Änderungen rescannen — Name/Aktionen ändern nichts am Match-Set.
this.triggerRescan();
}
return saved;
}
@Delete(':id')
async remove(@Param('id', ParseIntPipe) id: number) {
const res = await this.repo.delete(id);
if (!res.affected) throw new NotFoundException('Vorlage nicht gefunden');
this.scanner.invalidateTemplates();
this.triggerRescan();
return { ok: true };
}
}
@@ -0,0 +1,19 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { BarcodeTemplate } from '../database/entities/barcode-template.entity';
import { InboxDocument } from '../database/entities/inbox-document.entity';
import { BarcodeTemplatesController } from './barcode-templates.controller';
import { BarcodeScannerService } from './barcode-scanner.service';
import { PageCacheService } from './page-cache.service';
import { PreprocessingModule } from '../preprocessing/preprocessing.module';
@Module({
imports: [
TypeOrmModule.forFeature([BarcodeTemplate, InboxDocument]),
PreprocessingModule,
],
controllers: [BarcodeTemplatesController],
providers: [BarcodeScannerService, PageCacheService],
exports: [BarcodeScannerService, PageCacheService],
})
export class BarcodeModule {}
@@ -0,0 +1,117 @@
import { Injectable, Logger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import * as path from 'path';
import * as fs from 'fs/promises';
import sharp from 'sharp';
const THUMBNAIL_WIDTH = 180;
@Injectable()
export class PageCacheService {
private readonly logger = new Logger(PageCacheService.name);
private readonly inboxRoot: string;
constructor(configService: ConfigService) {
this.inboxRoot = configService.get<string>('INBOX_DATA_DIR', '/mnt/data/inbox');
}
documentDir(documentId: string): string {
return path.join(this.inboxRoot, documentId);
}
documentPdfPath(documentId: string): string {
return path.join(this.documentDir(documentId), 'document.pdf');
}
previewPath(documentId: string, page: number): string {
return path.join(this.documentDir(documentId), `page-${page}.preview.png`);
}
thumbnailPath(documentId: string, page: number): string {
return path.join(this.documentDir(documentId), `page-${page}.thumb.png`);
}
/**
* Übernimmt die bereits gerenderten 200-dpi-PNGs als preview.png und
* erzeugt parallel eine kleinere thumb.png pro Seite.
*/
async generate(documentId: string, renderedImages: string[]): Promise<void> {
const dir = this.documentDir(documentId);
await fs.mkdir(dir, { recursive: true });
for (let i = 0; i < renderedImages.length; i++) {
const page = i + 1;
const src = renderedImages[i];
const previewDest = this.previewPath(documentId, page);
const thumbDest = this.thumbnailPath(documentId, page);
try {
await fs.copyFile(src, previewDest);
await sharp(src).resize({ width: THUMBNAIL_WIDTH }).png().toFile(thumbDest);
} catch (err: any) {
this.logger.warn(
`Seiten-Cache fehlgeschlagen (${documentId} Seite ${page}): ${err.message}`,
);
}
}
}
/**
* Entfernt alle page-*.png-Dateien eines Dokuments (Original-PDF bleibt).
*/
async clear(documentId: string): Promise<void> {
const dir = this.documentDir(documentId);
let entries: string[];
try {
entries = await fs.readdir(dir);
} catch {
return;
}
for (const name of entries) {
if (!/^page-\d+\.(preview|thumb)\.png$/.test(name)) continue;
await fs.unlink(path.join(dir, name)).catch(() => undefined);
}
}
/**
* Verschiebt Page-Cache-Dateien nach dem Löschen einer Seite:
* page-N.*.png weg, page-(N+1..oldPageCount) rutschen um 1 nach vorne.
*/
async shiftAfterPageDelete(
documentId: string,
deletedPage: number,
oldPageCount: number,
): Promise<void> {
const dir = this.documentDir(documentId);
await fs
.unlink(path.join(dir, `page-${deletedPage}.thumb.png`))
.catch(() => undefined);
await fs
.unlink(path.join(dir, `page-${deletedPage}.preview.png`))
.catch(() => undefined);
for (let n = deletedPage + 1; n <= oldPageCount; n++) {
for (const variant of ['thumb', 'preview'] as const) {
const from = path.join(dir, `page-${n}.${variant}.png`);
const to = path.join(dir, `page-${n - 1}.${variant}.png`);
try {
await fs.rename(from, to);
} catch (err: any) {
this.logger.warn(
`Cache-Shift fehlgeschlagen (${documentId} Seite ${n} ${variant}): ${err.message}`,
);
}
}
}
}
async hasPreview(documentId: string, page: number): Promise<boolean> {
try {
await fs.access(this.previewPath(documentId, page));
return true;
} catch {
return false;
}
}
}
@@ -0,0 +1,73 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { ConfigModule, ConfigService } from '@nestjs/config';
import {
Client,
DocumentType,
DocumentField,
Task,
Postprocessing,
PostprocessingAction,
PostprocessingLog,
ExportTarget,
Setting,
Kontonummer,
Document,
UserClient,
Email,
Attachment,
Content,
ApiKey,
CorrespondentSetting,
BarcodeTemplate,
InboxDocument,
InboxPostprocessingAction,
CorrespondentEmailMapping,
} from './entities';
const entities = [
Client,
DocumentType,
DocumentField,
Task,
Postprocessing,
PostprocessingAction,
PostprocessingLog,
ExportTarget,
Setting,
Kontonummer,
Document,
UserClient,
Email,
Attachment,
Content,
ApiKey,
CorrespondentSetting,
BarcodeTemplate,
InboxDocument,
InboxPostprocessingAction,
CorrespondentEmailMapping,
];
@Module({
imports: [
TypeOrmModule.forRootAsync({
imports: [ConfigModule],
inject: [ConfigService],
useFactory: (config: ConfigService) => ({
type: 'mysql' as const,
host: config.get<string>('DB_HOST', 'localhost'),
port: config.get<number>('DB_PORT', 3306),
username: config.get<string>('DB_USERNAME', 'root'),
password: config.get<string>('DB_PASSWORD', ''),
database: config.get<string>('DB_DATABASE', 'paperlessadd'),
entities,
synchronize: true,
charset: 'utf8mb4',
}),
}),
TypeOrmModule.forFeature(entities),
],
exports: [TypeOrmModule],
})
export class DatabaseModule {}
@@ -0,0 +1,28 @@
import { Entity, PrimaryGeneratedColumn, Column, CreateDateColumn, UpdateDateColumn } from 'typeorm';
@Entity('api_keys')
export class ApiKey {
@PrimaryGeneratedColumn('uuid')
id!: string;
@Column({ type: 'varchar', length: 100 })
name!: string;
@Column({ type: 'varchar', length: 10 })
keyPrefix!: string;
@Column({ type: 'varchar', length: 64, unique: true })
keyHash!: string;
@CreateDateColumn()
createdAt!: Date;
@UpdateDateColumn()
updatedAt!: Date;
@Column({ type: 'datetime', nullable: true })
lastUsedAt!: Date | null;
@Column({ type: 'datetime', nullable: true })
expiresAt!: Date | null;
}
@@ -0,0 +1,54 @@
import { Entity, PrimaryGeneratedColumn, Column, ManyToOne, OneToOne, JoinColumn, Index } from 'typeorm';
import { Email } from './email.entity';
import { Content } from './content.entity';
@Entity('Attachments')
export class Attachment {
@PrimaryGeneratedColumn({ type: 'int' })
Id!: number;
@Index('IX_Attachments_EmailMessageId')
@Column({ type: 'int' })
EmailMessageId!: number;
@Column({ type: 'varchar', length: 255 })
FileName!: string;
@Column({ type: 'varchar', length: 100 })
ContentType!: string;
@Column({ type: 'tinyint', width: 1 })
Erechnung!: boolean;
@Column({ type: 'varchar', length: 32, default: '' })
Checksum!: string;
@Column({ type: 'varchar', length: 255, nullable: true })
ContentId!: string | null;
@Column({ type: 'tinyint', width: 1 })
IsEmbedded!: boolean;
@Index('IX_Attachments_ParentId')
@Column({ type: 'int', nullable: true })
ParentId!: number | null;
@Column({ type: 'json', nullable: true })
PaperlessDocumentIds!: any | null;
@Column({ type: 'int', default: 0 })
ImportStatus!: number;
@Column({ type: 'varchar', length: 255, nullable: true })
InterneBelegnummer!: string | null;
@Column({ type: 'int', default: 0 })
PageCount!: number;
@ManyToOne(() => Email, (email) => email.Attachments)
@JoinColumn({ name: 'EmailMessageId' })
EmailMessage!: Email;
@OneToOne(() => Content, (content) => content.AttachmentEntity)
Content!: Content;
}
@@ -0,0 +1,36 @@
import {
Entity,
PrimaryGeneratedColumn,
Column,
CreateDateColumn,
UpdateDateColumn,
} from 'typeorm';
export type BarcodeActionType = 'SEND_TO_PAPERLESS' | 'SEND_BY_EMAIL';
@Entity('barcode_templates')
export class BarcodeTemplate {
@PrimaryGeneratedColumn()
Id!: number;
@Column({ type: 'varchar', length: 100 })
Name!: string;
@Column({ type: 'varchar', length: 500 })
Regex!: string;
@Column({ type: 'boolean', default: false })
SplitBefore!: boolean;
@Column({ type: 'varchar', length: 500, nullable: true })
DateinameTemplate!: string | null;
@Column({ type: 'json' })
Actions!: BarcodeActionType[];
@CreateDateColumn()
CreatedAt!: Date;
@UpdateDateColumn()
UpdatedAt!: Date;
}
@@ -0,0 +1,13 @@
import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm';
@Entity('clients')
export class Client {
@PrimaryGeneratedColumn()
Id!: number;
@Column({ type: 'varchar', length: 45 })
Name!: string;
@Column({ type: 'int' })
PaperlessUserId!: number;
}
@@ -0,0 +1,22 @@
import { Entity, PrimaryGeneratedColumn, Column, OneToOne, JoinColumn, Index } from 'typeorm';
import { Attachment } from './attachment.entity';
@Entity('Contents')
export class Content {
@PrimaryGeneratedColumn({ type: 'int' })
Id!: number;
@Index('IX_Contents_AttachmentEntityId', { unique: true })
@Column({ type: 'int' })
AttachmentEntityId!: number;
@Column({ name: 'Content', type: 'longblob' })
Content1!: Buffer;
@Column({ type: 'bigint' })
ContentLength!: number;
@OneToOne(() => Attachment, (attachment) => attachment.Content)
@JoinColumn({ name: 'AttachmentEntityId' })
AttachmentEntity!: Attachment;
}
@@ -0,0 +1,14 @@
import { Entity, PrimaryGeneratedColumn, Column, Index } from 'typeorm';
@Entity('CorrespondentEmailMappings')
export class CorrespondentEmailMapping {
@PrimaryGeneratedColumn({ type: 'int' })
Id!: number;
@Index('IX_CorrespondentEmailMapping_EmailAddress', { unique: true })
@Column({ type: 'varchar', length: 255 })
EmailAddress!: string;
@Column({ type: 'int' })
PaperlessCorrespondentId!: number;
}
@@ -0,0 +1,10 @@
import { Entity, PrimaryColumn, Column } from 'typeorm';
@Entity('CorrespondentSettings')
export class CorrespondentSetting {
@PrimaryColumn({ type: 'int' })
CorrespondentId!: number;
@Column({ type: 'int', nullable: true })
AgrarmonitorId!: number | null;
}
@@ -0,0 +1,28 @@
import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm';
@Entity('DocumentFields')
export class DocumentField {
@PrimaryGeneratedColumn()
Id!: number;
@Column({ type: 'int' })
DocumentType!: number;
@Column({ type: 'int' })
Type!: number;
@Column({ type: 'int', nullable: true })
TypeIndex!: number | null;
@Column({ type: 'tinyint', default: 0 })
IsRequired!: boolean;
@Column({ type: 'tinyint', default: 0 })
IsRequiredPosteingang!: boolean;
@Column({ type: 'varchar', length: 250, nullable: true })
Hinweis!: string | null;
@Column({ type: 'tinyint', default: 0 })
VisiblePosteingang!: boolean;
}
@@ -0,0 +1,19 @@
import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm';
@Entity('DocumentTypes')
export class DocumentType {
@PrimaryGeneratedColumn()
Id!: number;
@Column({ type: 'int' })
DocumentTypeId!: number;
@Column({ type: 'text' })
TitelTemplate!: string;
@Column({ type: 'int', nullable: true })
TagNotReady!: number | null;
@Column({ type: 'int', nullable: true })
TagReady!: number | null;
}
@@ -0,0 +1,13 @@
import { Entity, PrimaryColumn, Column } from 'typeorm';
@Entity('documents')
export class Document {
@PrimaryColumn({ type: 'int' })
documentId!: number;
@Column({ type: 'varchar', length: 32, nullable: true })
checksum!: string | null;
@Column({ type: 'varchar', length: 1024, nullable: true })
filename!: string | null;
}
@@ -0,0 +1,32 @@
import { Entity, PrimaryGeneratedColumn, Column, OneToMany } from 'typeorm';
import { Attachment } from './attachment.entity';
@Entity('Emails')
export class Email {
@PrimaryGeneratedColumn()
Id!: number;
@Column({ type: 'varchar', length: 255 })
MessageId!: string;
@Column({ name: 'From', type: 'varchar', length: 255 })
SenderAddress!: string;
@Column({ name: 'To', type: 'varchar', length: 255 })
RecipientAddress!: string;
@Column({ type: 'varchar', length: 500 })
Subject!: string;
@Column({ type: 'datetime' })
Date!: Date;
@Column({ type: 'longtext' })
Body!: string;
@Column({ type: 'int', default: 0 })
Status!: number;
@OneToMany(() => Attachment, (attachment) => attachment.EmailMessage)
Attachments!: Attachment[];
}
@@ -0,0 +1,31 @@
import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm';
@Entity('ExportTargets')
export class ExportTarget {
@PrimaryGeneratedColumn()
Id!: number;
@Column({ type: 'varchar', length: 100 })
Name!: string;
@Column({ type: 'varchar', length: 20 })
Protocol!: string; // 'ftp' | 'webdav'
@Column({ type: 'varchar', length: 255 })
Host!: string;
@Column({ type: 'int', nullable: true })
Port!: number | null;
@Column({ type: 'varchar', length: 100, nullable: true })
Username!: string | null;
@Column({ type: 'varchar', length: 255, nullable: true })
Password!: string | null;
@Column({ type: 'varchar', length: 500, nullable: true })
RemotePath!: string | null;
@Column({ type: 'tinyint', default: 1 })
IsActive!: boolean;
}
@@ -0,0 +1,64 @@
import {
Entity,
PrimaryColumn,
Column,
CreateDateColumn,
UpdateDateColumn,
Index,
} from 'typeorm';
export type InboxSource = 'all' | 'user';
export interface StoredQrCode {
page: number;
value: string;
}
@Entity('inbox_documents')
@Index(['Source', 'OwnerUsername'])
export class InboxDocument {
@PrimaryColumn({ type: 'char', length: 36 })
Id!: string;
@Column({ type: 'varchar', length: 255 })
OriginalName!: string;
@Column({ type: 'varchar', length: 10 })
Source!: InboxSource;
@Column({ type: 'varchar', length: 100, nullable: true })
OwnerUsername!: string | null;
@Column({ type: 'int', default: 0 })
PageCount!: number;
@Column({ type: 'json' })
QrCodes!: StoredQrCode[];
@Column({
type: 'json',
nullable: true,
transformer: {
to: (v: number[] | null | undefined) => (v && v.length ? v : null),
from: (v: number[] | null) => v ?? [],
},
})
DeletedPages!: number[];
@Column({
type: 'json',
nullable: true,
transformer: {
to: (v: Record<string, number> | null | undefined) =>
v && Object.keys(v).length ? v : null,
from: (v: Record<string, number> | null) => v ?? {},
},
})
Rotations!: Record<string, number>;
@CreateDateColumn()
CreatedAt!: Date;
@UpdateDateColumn()
UpdatedAt!: Date;
}
@@ -0,0 +1,36 @@
import {
Entity,
PrimaryGeneratedColumn,
Column,
CreateDateColumn,
UpdateDateColumn,
} from 'typeorm';
export type InboxActionType = 'MAIL' | 'EXPORT' | 'PAPERLESS';
@Entity('inbox_postprocessing_actions')
export class InboxPostprocessingAction {
@PrimaryGeneratedColumn()
Id!: number;
@Column({ type: 'varchar', length: 30 })
ActionType!: InboxActionType;
@Column({ type: 'json' })
Content!: Record<string, any>;
@Column({ type: 'int', nullable: true })
BarcodeTemplateId!: number | null;
@Column({ type: 'int', default: 0 })
Order!: number;
@Column({ type: 'tinyint', default: 1 })
IsActive!: boolean;
@CreateDateColumn()
CreatedAt!: Date;
@UpdateDateColumn()
UpdatedAt!: Date;
}
@@ -0,0 +1,21 @@
export { Client } from './client.entity';
export { DocumentType } from './document-type.entity';
export { DocumentField } from './document-field.entity';
export { Task } from './task.entity';
export { Postprocessing } from './postprocessing.entity';
export { PostprocessingAction } from './postprocessing-action.entity';
export { PostprocessingLog } from './postprocessing-log.entity';
export { ExportTarget } from './export-target.entity';
export { Setting } from './setting.entity';
export { Kontonummer } from './kontonummer.entity';
export { Document } from './document.entity';
export { UserClient } from './user-client.entity';
export { Email } from './email.entity';
export { Attachment } from './attachment.entity';
export { Content } from './content.entity';
export { ApiKey } from './api-key.entity';
export { CorrespondentSetting } from './correspondent-setting.entity';
export { BarcodeTemplate } from './barcode-template.entity';
export { InboxDocument } from './inbox-document.entity';
export { InboxPostprocessingAction } from './inbox-postprocessing-action.entity';
export { CorrespondentEmailMapping } from './correspondent-email-mapping.entity';
@@ -0,0 +1,13 @@
import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm';
@Entity('Kontonummern')
export class Kontonummer {
@PrimaryGeneratedColumn()
KontonummerId: number;
@Column({ type: 'int' })
CorrespondentId: number;
@Column({ type: 'varchar', length: 100 })
Nummer: string;
}
@@ -0,0 +1,27 @@
import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm';
@Entity('PostprocessingActions')
export class PostprocessingAction {
@PrimaryGeneratedColumn()
Id!: number;
@Column({ type: 'int' })
PostprocessingId!: number;
@Column({ type: 'int' })
ActionType!: number;
// 1 = Export (FTP/WebDAV)
// 2 = Mail
// 3 = Tag setzen/entfernen
// 4 = Custom Field setzen
// 5 = Webhook
@Column({ type: 'json' })
Content!: Record<string, any>;
@Column({ type: 'int' })
Order!: number;
@Column({ type: 'tinyint' })
IsActive!: boolean;
}
@@ -0,0 +1,25 @@
import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm';
@Entity('PostprocessingLogs')
export class PostprocessingLog {
@PrimaryGeneratedColumn()
Id!: number;
@Column({ type: 'int' })
PostprocessingId!: number;
@Column({ type: 'int', nullable: true })
ActionId!: number | null;
@Column({ type: 'int' })
DocumentId!: number;
@Column({ type: 'varchar', length: 20 })
Status!: string; // 'success' | 'error' | 'skipped'
@Column({ type: 'text', nullable: true })
Message!: string | null;
@Column({ type: 'datetime', default: () => 'CURRENT_TIMESTAMP' })
CreatedAt!: Date;
}
@@ -0,0 +1,33 @@
import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm';
export interface FilterCondition {
field: string; // 'document_type' | 'correspondent' | 'owner' | 'tag' | 'custom_field_<id>' | 'title' | 'archive_serial_number'
operator: string; // 'equals' | 'not_equals' | 'contains' | 'not_contains' | 'is_set' | 'is_not_set'
value: any;
}
export interface FilterGroup {
combinator: 'AND' | 'OR';
rules: (FilterCondition | FilterGroup)[];
}
@Entity('Postprocessings')
export class Postprocessing {
@PrimaryGeneratedColumn()
Id!: number;
@Column({ type: 'varchar', length: 255 })
Name!: string;
@Column({ type: 'json' })
FilterJson!: FilterGroup;
@Column({ type: 'int', default: 0 })
Order!: number;
@Column({ type: 'tinyint', default: 0 })
NoFurther!: boolean;
@Column({ type: 'tinyint', default: 1 })
IsActive!: boolean;
}
@@ -0,0 +1,16 @@
import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm';
@Entity('settings')
export class Setting {
@PrimaryGeneratedColumn()
ID!: number;
@Column({ type: 'int' })
Typ!: number;
@Column({ type: 'varchar', length: 255, nullable: true })
Wert!: string | null;
@Column({ type: 'varchar', length: 45, nullable: true })
Tag!: string | null;
}
@@ -0,0 +1,61 @@
import { Entity, PrimaryColumn, Column } from 'typeorm';
@Entity('tasks')
export class Task {
@PrimaryColumn({ type: 'varchar', length: 36 })
TaskId!: string;
@Column({ type: 'varchar', length: 50 })
InterneBelegnummer!: string;
@Column({ type: 'int', nullable: true })
DocumentType!: number | null;
@Column({ type: 'datetime', nullable: true })
Eingangsdatum!: Date | null;
@Column({ type: 'tinyint', nullable: true })
Fertig!: number | null;
@Column({ type: 'varchar', length: 50, nullable: true })
Tags!: string | null;
@Column({ type: 'int', nullable: true })
BetriebID!: number | null;
@Column({ type: 'varchar', length: 100, nullable: true })
Lieferant!: string | null;
@Column({ type: 'varchar', length: 100, nullable: true })
externeBelegnummer!: string | null;
@Column({ type: 'int', nullable: true })
EinkaufID!: number | null;
@Column({ type: 'datetime', nullable: true })
Belegdatum!: Date | null;
@Column({ type: 'int', nullable: true })
PaperlessDocumentID!: number | null;
@Column({ type: 'varchar', length: 36, nullable: true })
TaskReferenceID!: string | null;
@Column({ type: 'longtext', nullable: true })
BarcodeJson!: string | null;
@Column({ type: 'int', nullable: true })
DuplikatZU!: number | null;
@Column({ type: 'longtext', nullable: true })
CustomFieldsJson!: string | null;
@Column({ type: 'varchar', length: 50, nullable: true })
Asn!: string | null;
@Column({ type: 'int', nullable: true })
SourceAttachmentID!: number | null;
@Column({ type: 'varchar', length: 50, nullable: true })
SourceAttachmentRange!: string | null;
}
@@ -0,0 +1,16 @@
import { Entity, PrimaryGeneratedColumn, Column } from 'typeorm';
@Entity('UserClients')
export class UserClient {
@PrimaryGeneratedColumn()
Id!: number;
@Column({ type: 'varchar', length: 255 })
UserId!: string;
@Column({ type: 'int' })
ClientId!: number;
@Column({ type: 'enum', enum: ['viewer', 'editor', 'admin'], default: 'editor' })
Role!: 'viewer' | 'editor' | 'admin';
}
@@ -0,0 +1,28 @@
import { Controller, Post, Logger, HttpCode, HttpStatus } from '@nestjs/common';
import { RequirePermissions } from '../auth/permissions.decorator';
import { Permission } from '../auth/permissions.enum';
import { EmailDownloadService } from './email-download.service';
@Controller('api/emails')
export class EmailDownloadController {
private readonly logger = new Logger(EmailDownloadController.name);
constructor(private readonly emailDownloadService: EmailDownloadService) {}
@Post('fetch')
@HttpCode(HttpStatus.OK)
async triggerFetch() {
this.logger.log('Manueller E-Mail-Abruf wurde ausgelöst.');
await this.emailDownloadService.handleCron();
return { message: 'E-Mail-Abruf abgeschlossen.' };
}
@Post('backfill-thumbnails')
@RequirePermissions(Permission.MANAGE_ALL)
@HttpCode(HttpStatus.OK)
async backfillThumbnails() {
this.logger.log('Manueller Backfill für Thumbnails wurde ausgelöst.');
const result = await this.emailDownloadService.backfillThumbnailsForNewEmails();
return { message: 'Backfill abgeschlossen.', result };
}
}
@@ -0,0 +1,22 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { Email } from '../database/entities/email.entity';
import { Attachment } from '../database/entities/attachment.entity';
import { Content } from '../database/entities/content.entity';
import { EmailDownloadService } from './email-download.service';
import { EmailDownloadController } from './email-download.controller';
import { PreprocessingModule } from '../preprocessing/preprocessing.module';
import { EmailModule } from '../email/email.module';
@Module({
imports: [
TypeOrmModule.forFeature([Email, Attachment, Content]),
PreprocessingModule,
EmailModule
],
controllers: [EmailDownloadController],
providers: [EmailDownloadService],
exports: [EmailDownloadService],
})
export class EmailDownloadModule {}
@@ -0,0 +1,248 @@
import { Injectable, Logger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { Cron, CronExpression } from '@nestjs/schedule';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { ImapFlow, type FetchMessageObject } from 'imapflow';
import { simpleParser, type AddressObject, type Attachment as MailAttachment } from 'mailparser';
import * as crypto from 'crypto';
import * as os from 'os';
import * as path from 'path';
import * as fs from 'fs/promises';
import { PdfService } from '../preprocessing/pdf.service';
import { EmailPageCacheService } from '../email/email-page-cache.service';
import { Email } from '../database/entities/email.entity';
import { Attachment } from '../database/entities/attachment.entity';
import { Content } from '../database/entities/content.entity';
import { isERechnung } from './zugferd.util';
@Injectable()
export class EmailDownloadService {
private readonly logger = new Logger(EmailDownloadService.name);
private running = false;
constructor(
private readonly configService: ConfigService,
private readonly pdfService: PdfService,
private readonly pageCache: EmailPageCacheService,
@InjectRepository(Email) private readonly emailRepo: Repository<Email>,
@InjectRepository(Attachment) private readonly attachmentRepo: Repository<Attachment>,
@InjectRepository(Content) private readonly contentRepo: Repository<Content>,
) {}
@Cron(CronExpression.EVERY_5_MINUTES)
async handleCron() {
if (this.running) {
this.logger.warn('E-Mail-Download läuft noch überspringe Tick.');
return;
}
this.running = true;
try {
await this.fetchAndStore();
} catch (err: any) {
this.logger.error(`Fehler im E-Mail-Download-Job: ${err.message}`, err.stack);
} finally {
this.running = false;
}
}
private async fetchAndStore(): Promise<void> {
const host = this.configService.get<string>('IMAP_HOST');
const port = this.configService.get<number>('IMAP_PORT', 993);
const secure = this.configService.get<string>('IMAP_USE_SSL', 'true') === 'true';
const user = this.configService.get<string>('IMAP_USERNAME');
const pass = this.configService.get<string>('IMAP_PASSWORD');
if (!host || !user || !pass) {
this.logger.warn('IMAP-Konfiguration unvollständig Job wird übersprungen.');
return;
}
this.logger.log('E-Mail Fetch Job gestartet.');
const client = new ImapFlow({
host,
port,
secure,
auth: { user, pass },
logger: false,
});
await client.connect();
this.logger.log(`Verbunden mit IMAP-Server ${host}:${port}`);
const lock = await client.getMailboxLock('INBOX');
try {
const status = await client.status('INBOX', { messages: true });
this.logger.log(`Posteingang geöffnet. Anzahl der Nachrichten: ${status.messages ?? 0}`);
if (!status.messages || status.messages === 0) {
return;
}
const iter = client.fetch(
'1:*',
{ envelope: true, uid: true, source: true },
);
for await (const msg of iter) {
const messageId = msg.envelope?.messageId;
if (!messageId) continue;
try {
const existing = await this.emailRepo.findOne({ where: { MessageId: messageId } });
if (existing) {
this.logger.debug(`E-Mail mit MessageId ${messageId} bereits vorhanden.`);
continue;
}
await this.processMessage(msg);
} catch (err: any) {
this.logger.error(`Fehler beim Abrufen der Nachricht ${messageId}: ${err.message}`, err.stack);
}
}
this.logger.log('Alle neuen E-Mails und deren Anhänge wurden in der Datenbank gespeichert.');
} finally {
lock.release();
await client.logout().catch(() => undefined);
this.logger.log('Verbindung zum IMAP-Server geschlossen.');
}
}
private async processMessage(msg: FetchMessageObject): Promise<void> {
if (!msg.source) return;
const parsed = await simpleParser(msg.source);
const messageId = msg.envelope?.messageId ?? parsed.messageId ?? '';
const email = new Email();
email.MessageId = messageId;
email.SenderAddress = formatAddress(parsed.from);
email.RecipientAddress = formatAddress(parsed.to);
email.Subject = (msg.envelope?.subject ?? parsed.subject ?? '').slice(0, 500);
email.Date = msg.envelope?.date ?? parsed.date ?? new Date();
email.Body = parsed.html || parsed.text || '';
email.Status = 0;
const attachmentsToPersist: Array<{ attachment: Attachment; buffer: Buffer }> = [];
for (const att of parsed.attachments) {
const entry = await this.buildAttachment(att);
if (entry) attachmentsToPersist.push(entry);
}
// Double-Check: nochmal gegen DB prüfen (Race-Condition-Schutz wie in C#)
const existing2 = await this.emailRepo.findOne({ where: { MessageId: messageId } });
if (existing2) {
this.logger.debug(`E-Mail mit MessageId ${messageId} nach dem Download bereits vorhanden.`);
return;
}
const savedEmail = await this.emailRepo.save(email);
for (const { attachment, buffer } of attachmentsToPersist) {
attachment.EmailMessageId = savedEmail.Id;
const savedAttachment = await this.attachmentRepo.save(attachment);
const content = new Content();
content.AttachmentEntityId = savedAttachment.Id;
content.Content1 = buffer;
content.ContentLength = buffer.length;
await this.contentRepo.save(content);
// Generate PDF thumbnails if it's a PDF
if (savedAttachment.ContentType === 'application/pdf') {
await this.generateThumbnailsForAttachment(savedAttachment, buffer);
}
}
this.logger.debug(`Neue E-Mail mit MessageId ${messageId} hinzugefügt (${attachmentsToPersist.length} Anhänge).`);
}
public async generateThumbnailsForAttachment(attachment: Attachment, buffer: Buffer): Promise<void> {
try {
const tempPdfPath = path.join(os.tmpdir(), `email-att-${attachment.Id}.pdf`);
await fs.writeFile(tempPdfPath, buffer);
const images = await this.pdfService.pdfToImages(tempPdfPath, 400);
await this.pageCache.generate(attachment.Id, images);
attachment.PageCount = images.length;
await this.attachmentRepo.save(attachment);
await this.pdfService.cleanup(images);
await fs.unlink(tempPdfPath).catch(() => {});
} catch (err: any) {
this.logger.warn(`Konnte Vorschaubilder für Anhang ${attachment.Id} nicht generieren: ${err.message}`);
}
}
public async backfillThumbnailsForNewEmails(): Promise<{ processed: number; failed: number }> {
const emails = await this.emailRepo.find({
where: { Status: 0 },
relations: ['Attachments', 'Attachments.Content']
});
let processed = 0;
let failed = 0;
for (const email of emails) {
for (const attachment of email.Attachments) {
if (attachment.ContentType === 'application/pdf' && attachment.PageCount === 0 && attachment.Content?.Content1) {
this.logger.log(`Backfill: Generiere Thumbnails für Attachment ${attachment.Id} (Email ${email.Id})`);
try {
await this.generateThumbnailsForAttachment(attachment, attachment.Content.Content1);
processed++;
} catch (err) {
failed++;
}
}
}
}
this.logger.log(`Backfill abgeschlossen: ${processed} erfolgreich, ${failed} fehlgeschlagen.`);
return { processed, failed };
}
private async buildAttachment(
att: MailAttachment,
): Promise<{ attachment: Attachment; buffer: Buffer } | null> {
const buffer = att.content;
if (!Buffer.isBuffer(buffer) || buffer.length === 0) return null;
const filename = att.filename ?? att.cid ?? 'unbenannt';
let contentType = att.contentType ?? 'application/octet-stream';
const isEmbedded = att.contentDisposition === 'inline';
const isPdfByName = filename.toLowerCase().endsWith('.pdf');
const isPdfByType = contentType.toLowerCase() === 'application/pdf';
const isOctet = contentType.toLowerCase() === 'application/octet-stream';
// Nur PDFs (inkl. als octet-stream deklarierter PDFs) und embedded rfc822 werden übernommen.
const isRfc822 = contentType.toLowerCase() === 'message/rfc822';
if (!isPdfByType && !isRfc822 && !(isOctet && isPdfByName)) {
return null;
}
if (isPdfByName && !isPdfByType) {
contentType = 'application/pdf';
}
const attachment = new Attachment();
attachment.FileName = filename.slice(0, 255);
attachment.ContentType = contentType.slice(0, 100);
attachment.IsEmbedded = isEmbedded;
attachment.ContentId = att.cid ? att.cid.slice(0, 255) : null;
attachment.Checksum = crypto.createHash('md5').update(buffer).digest('hex');
attachment.Erechnung = contentType.toLowerCase() === 'application/pdf' ? isERechnung(buffer) : false;
attachment.ParentId = null;
return { attachment, buffer };
}
}
function formatAddress(addr: AddressObject | AddressObject[] | undefined): string {
if (!addr) return '';
const first = Array.isArray(addr) ? addr[0] : addr;
return (first?.text ?? '').slice(0, 255);
}
@@ -0,0 +1,10 @@
const KNOWN_NAMES = [
'factur-x.xml',
'zugferd-invoice.xml',
'xrechnung.xml',
];
export function isERechnung(pdfBuffer: Buffer): boolean {
const asText = pdfBuffer.toString('latin1').toLowerCase();
return KNOWN_NAMES.some((n) => asText.includes(n));
}
@@ -0,0 +1,134 @@
import { Controller, Get, Post, Body, Param, Query, Res, HttpException, HttpStatus, Logger, Delete } from '@nestjs/common';
import type { Response } from 'express';
import { EmailImportService } from './email-import.service';
import { EmailPageCacheService } from './email-page-cache.service';
import { RequirePermissions } from '../auth/permissions.decorator';
import { Permission } from '../auth/permissions.enum';
@Controller('api/email-import')
export class EmailImportController {
private readonly logger = new Logger(EmailImportController.name);
constructor(
private readonly importService: EmailImportService,
private readonly pageCache: EmailPageCacheService,
) {}
// --- Korrespondenten Mapping ---
@Get('mappings')
@RequirePermissions(Permission.MANAGE_SETTINGS)
async getMappings() {
return this.importService.getMappings();
}
@Post('mappings')
@RequirePermissions(Permission.MANAGE_SETTINGS)
async addMapping(@Body() body: { emailAddress: string; paperlessCorrespondentId: number }) {
if (!body.emailAddress || !body.paperlessCorrespondentId) {
throw new HttpException('Missing emailAddress or paperlessCorrespondentId', HttpStatus.BAD_REQUEST);
}
return this.importService.addMapping(body.emailAddress, body.paperlessCorrespondentId);
}
@Delete('mappings/:id')
@RequirePermissions(Permission.MANAGE_SETTINGS)
async deleteMapping(@Param('id') id: string) {
return this.importService.deleteMapping(parseInt(id, 10));
}
@Get('correspondent')
@RequirePermissions(Permission.VIEW_MAIL)
async getCorrespondent(@Query('email') emailAddress: string) {
const id = await this.importService.getCorrespondentByEmail(emailAddress);
return { paperlessCorrespondentId: id };
}
@Post('emails/:id/ensure-previews')
@RequirePermissions(Permission.VIEW_MAIL)
async ensurePreviews(@Param('id') id: string) {
await this.importService.ensurePreviews(parseInt(id, 10));
return { success: true };
}
// --- Belegnummern ---
@Get('belegnummer')
@RequirePermissions(Permission.VIEW_MAIL)
async getBelegnummer(@Query('date') date: string) {
if (!date) throw new HttpException('Date query parameter required', HttpStatus.BAD_REQUEST);
const nummer = await this.importService.getBelegnummer(date);
return { nummer };
}
@Post('belegnummer/release')
@RequirePermissions(Permission.VIEW_MAIL)
async releaseBelegnummer(@Body() body: { date: string; number: string }) {
if (!body.date || !body.number) throw new HttpException('Date and number required', HttpStatus.BAD_REQUEST);
await this.importService.releaseBelegnummer(body.date, body.number);
return { success: true };
}
// --- Print Preview ---
@Post('attachments/:attachmentId/print-preview')
@RequirePermissions(Permission.VIEW_MAIL)
async printPreview(
@Param('attachmentId') attachmentId: number,
@Body() barcodeData: any,
@Res() res: Response,
) {
try {
const pdfBuffer = await this.importService.generatePrintPdf(attachmentId, barcodeData);
this.logger.log(`Print preview generated for attachment ${attachmentId}: ${pdfBuffer.length} bytes`);
res.setHeader('Content-Type', 'application/pdf');
res.setHeader('Content-Disposition', `inline; filename="preview-${attachmentId}.pdf"`);
res.send(pdfBuffer);
} catch (err: any) {
this.logger.error(`Error generating print preview: ${err.message}`);
throw new HttpException(err.message, HttpStatus.INTERNAL_SERVER_ERROR);
}
}
// --- Page Thumbnails ---
@Get('attachments/:attachmentId/pages/:page/thumb')
@RequirePermissions(Permission.VIEW_MAIL)
async getPageThumbnail(
@Param('attachmentId') attachmentId: number,
@Param('page') page: number,
@Res() res: Response,
) {
const hasPreview = await this.pageCache.hasPreview(attachmentId, page);
if (!hasPreview) {
throw new HttpException('Preview not found', HttpStatus.NOT_FOUND);
}
const thumbPath = this.pageCache.thumbnailPath(attachmentId, page);
res.sendFile(thumbPath);
}
// --- Page Previews (larger) ---
@Get('attachments/:attachmentId/pages/:page/preview')
@RequirePermissions(Permission.VIEW_MAIL)
async getPagePreview(
@Param('attachmentId') attachmentId: number,
@Param('page') page: number,
@Res() res: Response,
) {
const hasPreview = await this.pageCache.hasPreview(attachmentId, page);
if (!hasPreview) {
throw new HttpException('Preview not found', HttpStatus.NOT_FOUND);
}
const previewPath = this.pageCache.previewPath(attachmentId, page);
res.sendFile(previewPath);
}
// --- Final Import ---
@Post('execute')
@RequirePermissions(Permission.MANAGE_ALL)
async executeImport(@Body() importData: any) {
try {
const result = await this.importService.executeImport(importData);
return result;
} catch (err: any) {
this.logger.error(`Error executing import: ${err.message}`);
throw new HttpException(err.message, HttpStatus.INTERNAL_SERVER_ERROR);
}
}
}
@@ -0,0 +1,446 @@
import { Injectable, Logger, HttpException, HttpStatus } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import axios from 'axios';
import { PDFDocument, rgb, StandardFonts } from 'pdf-lib';
import { Attachment } from '../database/entities/attachment.entity';
import { Email } from '../database/entities/email.entity';
import { Content } from '../database/entities/content.entity';
import { CorrespondentEmailMapping } from '../database/entities/correspondent-email-mapping.entity';
import { Task } from '../database/entities/task.entity';
import { PaperlessService } from '../paperless/paperless.service';
import * as QRCode from 'qrcode';
import { EmailPageCacheService } from './email-page-cache.service';
import { PdfService } from '../preprocessing/pdf.service';
import * as path from 'path';
import * as os from 'os';
import * as fs from 'fs/promises';
@Injectable()
export class EmailImportService {
private readonly logger = new Logger(EmailImportService.name);
constructor(
private readonly configService: ConfigService,
@InjectRepository(Email) private readonly emailRepo: Repository<Email>,
@InjectRepository(Attachment) private readonly attachmentRepo: Repository<Attachment>,
@InjectRepository(Content) private readonly contentRepo: Repository<Content>,
@InjectRepository(CorrespondentEmailMapping) private readonly mappingRepo: Repository<CorrespondentEmailMapping>,
@InjectRepository(Task) private readonly taskRepo: Repository<Task>,
private readonly paperlessService: PaperlessService,
private readonly pdfService: PdfService,
private readonly pageCache: EmailPageCacheService,
) {}
async ensurePreviews(emailId: number): Promise<void> {
const attachments = await this.attachmentRepo.find({
where: { EmailMessageId: emailId, ContentType: 'application/pdf' },
relations: ['Content'],
});
for (const attachment of attachments) {
const hasPreview = await this.pageCache.hasPreview(attachment.Id, 1);
if (!hasPreview && attachment.Content?.Content1) {
this.logger.log(`Generiere fehlende Vorschaubilder für Anhang ${attachment.Id} (Email ${emailId})`);
try {
const tempPdfPath = path.join(os.tmpdir(), `email-att-gen-${attachment.Id}.pdf`);
await fs.writeFile(tempPdfPath, attachment.Content.Content1);
const images = await this.pdfService.pdfToImages(tempPdfPath, 400);
await this.pageCache.generate(attachment.Id, images);
attachment.PageCount = images.length;
await this.attachmentRepo.save(attachment);
await this.pdfService.cleanup(images);
await fs.unlink(tempPdfPath).catch(() => {});
} catch (err: any) {
this.logger.warn(`Fehler bei on-demand Vorschau-Generierung für Anhang ${attachment.Id}: ${err.message}`);
}
}
}
}
// --- Korrespondenten Mapping ---
async getMappings() {
return this.mappingRepo.find();
}
async addMapping(emailAddress: string, paperlessCorrespondentId: number) {
let mapping = await this.mappingRepo.findOne({ where: { EmailAddress: emailAddress } });
if (!mapping) {
mapping = this.mappingRepo.create({ EmailAddress: emailAddress, PaperlessCorrespondentId: paperlessCorrespondentId });
} else {
mapping.PaperlessCorrespondentId = paperlessCorrespondentId;
}
return this.mappingRepo.save(mapping);
}
async deleteMapping(id: number) {
return this.mappingRepo.delete(id);
}
async getCorrespondentByEmail(emailAddress: string): Promise<number | null> {
const mapping = await this.mappingRepo.findOne({ where: { EmailAddress: emailAddress } });
return mapping ? mapping.PaperlessCorrespondentId : null;
}
// --- Belegnummern API ---
private buildUrl(urlTemplate: string, dateStr: string): string {
const dateObj = new Date(dateStr);
const year = (isNaN(dateObj.getTime()) ? new Date() : dateObj).getFullYear().toString();
return urlTemplate.replace('{Jahr}', year);
}
async getBelegnummer(emailDate: string): Promise<string> {
const urlTemplate = this.configService.get<string>('BELEGNUMMER_GET_URL');
if (!urlTemplate) throw new HttpException('BELEGNUMMER_GET_URL not configured', HttpStatus.INTERNAL_SERVER_ERROR);
const url = this.buildUrl(urlTemplate, emailDate);
try {
this.logger.debug(`Fetching Belegnummer from ${url}`);
const response = await axios.get(url);
// If the response is an object, try to extract 'nummer' or 'number'
let result = response.data;
if (result && typeof result === 'object') {
result = result.nummer || result.number || result.data?.nummer || JSON.stringify(result);
}
this.logger.debug(`Received Belegnummer: ${result}`);
return String(result);
} catch (error: any) {
const status = error.response?.status || 'UNKNOWN';
const detail = error.response?.data ? JSON.stringify(error.response.data) : error.message;
this.logger.error(`Failed to fetch Belegnummer from ${url}. Status: ${status}, Detail: ${detail}`);
throw new HttpException(`Fehler beim Abrufen der Belegnummer: ${detail}`, HttpStatus.BAD_GATEWAY);
}
}
async releaseBelegnummer(emailDate: string, number: string): Promise<void> {
const urlTemplate = this.configService.get<string>('BELEGNUMMER_RELEASE_URL');
if (!urlTemplate) {
this.logger.warn('BELEGNUMMER_RELEASE_URL not configured, skipping release.');
return;
}
const cleanNumber = number.replace(/^0+/, '') || '0';
let url = this.buildUrl(urlTemplate, emailDate);
url = url.replace('{Nummer}', cleanNumber);
try {
this.logger.log(`Releasing Belegnummer: ${cleanNumber} (original: ${number}) via ${url}`);
await axios.get(url);
} catch (error: any) {
this.logger.error(`Failed to release Belegnummer at ${url}: ${error.message}`);
}
}
async setBelegnummer(emailDate: string, number: string): Promise<void> {
const urlTemplate = this.configService.get<string>('BELEGNUMMER_SET_URL');
if (!urlTemplate) throw new HttpException('BELEGNUMMER_SET_URL not configured', HttpStatus.INTERNAL_SERVER_ERROR);
const cleanNumber = number.replace(/^0+/, '') || '0';
let url = this.buildUrl(urlTemplate, emailDate);
url = url.replace('{Nummer}', cleanNumber);
try {
this.logger.log(`Setting Belegnummer: ${cleanNumber} (original: ${number}) via ${url}`);
await axios.get(url);
} catch (error: any) {
this.logger.error(`Failed to set Belegnummer at ${url}: ${error.message}`);
throw new HttpException('Fehler beim Setzen der Belegnummer', HttpStatus.BAD_GATEWAY);
}
}
// --- Print Preview ---
async generatePrintPdf(attachmentId: number, barcodeData: any): Promise<Buffer> {
const content = await this.contentRepo.findOne({ where: { AttachmentEntityId: attachmentId } });
if (!content) throw new HttpException('Inhalt nicht gefunden', HttpStatus.NOT_FOUND);
return this.applyBarcodeToPdf(content.Content1, barcodeData);
}
async applyBarcodeToPdf(pdfBytes: Buffer, barcodeData: any): Promise<Buffer> {
this.logger.debug(`applyBarcodeToPdf: Input size = ${pdfBytes.length} bytes`);
let currentPdfBytes = pdfBytes;
const tempInputPath = path.join(os.tmpdir(), `input-${Date.now()}.pdf`);
await fs.writeFile(tempInputPath, pdfBytes);
try {
// First try to load to check encryption
let pdfDoc = await PDFDocument.load(currentPdfBytes, { ignoreEncryption: true });
if (pdfDoc.isEncrypted) {
this.logger.log('PDF ist verschlüsselt, versuche Bereinigung via Ghostscript...');
const sanitizedPath = await this.pdfService.sanitizePdf(tempInputPath);
currentPdfBytes = await fs.readFile(sanitizedPath);
await fs.unlink(sanitizedPath).catch(() => {});
// Reload sanitized PDF
pdfDoc = await PDFDocument.load(currentPdfBytes);
}
const pages = pdfDoc.getPages();
this.logger.debug(`applyBarcodeToPdf: Pages = ${pages.length}, Encrypted = ${pdfDoc.isEncrypted}`);
if (pages.length === 0) {
this.logger.warn('applyBarcodeToPdf: Keine Seiten gefunden');
return Buffer.from(await pdfDoc.save());
}
const firstPage = pages[0];
const { x, y, nummer, datum, jahr } = barcodeData;
// Parse date
const d = new Date(datum);
const yyyy = (isNaN(d.getTime()) ? new Date() : d).getFullYear().toString();
const mm = String((isNaN(d.getTime()) ? new Date() : d).getMonth() + 1).padStart(2, '0');
const dd = String((isNaN(d.getTime()) ? new Date() : d).getDate()).padStart(2, '0');
const qrDateStr = `${yyyy}${mm}${dd}`; // yyyyMMdd
const qrContent = `${String(jahr).padStart(4, '0')}-${String(nummer).padStart(6, '0')}-${qrDateStr}`;
const printDateStr = `${dd}.${mm}.${yyyy}`;
// Dimensions: 57x32 mm
const PT_PER_MM = 2.83465;
const boxW = 57 * PT_PER_MM;
const boxH = 32 * PT_PER_MM;
// A4 dimensions: 210x297 mm
const PAGE_H_PT = 297 * PT_PER_MM;
// Convert mm to points (Y is from bottom in pdf-lib)
const startX = Number(x) * PT_PER_MM;
const startY = PAGE_H_PT - (Number(y) * PT_PER_MM) - boxH;
// 1. Draw Background Box (White with Black border)
firstPage.drawRectangle({
x: startX,
y: startY,
width: boxW,
height: boxH,
color: rgb(1, 1, 1),
borderColor: rgb(0, 0, 0),
borderWidth: 1,
});
// 2. Draw QR Code
const qrBuffer = await QRCode.toBuffer(qrContent, {
errorCorrectionLevel: 'H',
margin: 0,
width: 300,
color: { dark: '#000000', light: '#FFFFFF' }
});
const qrImage = await pdfDoc.embedPng(qrBuffer);
// QR Code size: 27x27 mm (10% smaller than 30x30)
const qrSize = 27 * PT_PER_MM;
const padding = (32 - 27) / 2; // Center vertically in 32mm box
const qrX = startX + (padding * PT_PER_MM);
const qrY = startY + (padding * PT_PER_MM);
firstPage.drawImage(qrImage, {
x: qrX,
y: qrY,
width: qrSize,
height: qrSize,
});
// 3. Draw Texts
const helveticaBold = await pdfDoc.embedFont(StandardFonts.HelveticaBold);
// Helper to draw centered text in a specific area
const drawCenteredInArea = (text: string, relX: number, relY: number, areaW: number, areaH: number, fontSize: number) => {
const textWidth = helveticaBold.widthOfTextAtSize(text, fontSize);
const absX = startX + (relX * PT_PER_MM) + (areaW * PT_PER_MM / 2) - (textWidth / 2);
const absY = (startY + boxH) - (relY * PT_PER_MM) - (areaH * PT_PER_MM / 2) - (fontSize / 2.5);
firstPage.drawText(text, {
x: absX,
y: absY,
size: fontSize,
font: helveticaBold,
color: rgb(0, 0, 0),
});
};
const isNeu = barcodeData.isNeu === true;
// Text Area X: +33.3mm, Width: 21mm
// Year: Y + 3mm, Height: 7.5mm
drawCenteredInArea(String(jahr).padStart(4, '0'), 33.3, 3, 21, 7.5, 12);
// Number: Y + 10.5mm, Height: 7.5mm
const numberText = isNeu ? '<- neu ->' : String(nummer).padStart(6, '0');
drawCenteredInArea(numberText, 33.3, 10.5, 21, 7.5, 12);
// "Eingegangen": Y + 19mm, Height: 4mm, Size 8
drawCenteredInArea('Eingegangen', 33.3, 19, 21, 4, 8);
// Date: Y + 19 + 3.5 = 22.5mm, Height: 4mm, Size 8
drawCenteredInArea(printDateStr, 33.3, 22.5, 21, 4, 8);
return Buffer.from(await pdfDoc.save());
} finally {
await fs.unlink(tempInputPath).catch(() => {});
}
}
// --- Import Logic ---
async executeImport(data: {
attachments: {
attachmentId: number;
type: 'MAIN' | 'ATTACHMENT' | 'IGNORE';
paperlessCorrespondentId?: number | null;
parentDocumentId?: number | null; // Used if type is ATTACHMENT (should map to a Custom Field theoretically, or just tags. For now, CF if configured, but we pass it)
splitRanges?: { start: number; end: number }[]; // 1-based pages, e.g. [{start: 1, end: 3}, {start: 4, end: 5}]
barcode?: { x: number; y: number; nummer: string; datum: string; jahr: string };
belegnummer?: string;
}[];
emailDate: string;
}): Promise<{ success: boolean; results: any[] }> {
const fs = require('fs/promises');
const path = require('path');
const os = require('os');
const crypto = require('crypto');
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'paperless-mail-import-'));
const results = [];
try {
for (const att of data.attachments) {
if (att.type === 'IGNORE') continue;
const attachmentEntity = await this.attachmentRepo.findOne({ where: { Id: att.attachmentId } });
if (!attachmentEntity) continue;
const content = await this.contentRepo.findOne({ where: { AttachmentEntityId: att.attachmentId } });
if (!content) continue;
const originalPdfBytes = content.Content1;
const baseFilename = attachmentEntity.FileName.replace(/\.pdf$/i, '');
const paperlessIds: any = {};
const uploadPromises = [];
// Formatting the date for Paperless (ISO format)
const createdDate = new Date(data.emailDate).toISOString();
if (att.splitRanges && att.splitRanges.length > 0) {
// SPLIT PDF
const pdfDoc = await PDFDocument.load(originalPdfBytes, { ignoreEncryption: true });
for (const range of att.splitRanges) {
const newPdf = await PDFDocument.create();
// Pages are 0-indexed in pdf-lib
const pageIndices = Array.from({ length: range.end - range.start + 1 }, (_, i) => range.start - 1 + i);
const copiedPages = await newPdf.copyPages(pdfDoc, pageIndices);
copiedPages.forEach((p) => newPdf.addPage(p));
const splitPdfBytes = await newPdf.save();
const tempFilePath = path.join(tempDir, `${baseFilename}_${range.start}-${range.end}.pdf`);
await fs.writeFile(tempFilePath, Buffer.from(splitPdfBytes));
uploadPromises.push({
path: tempFilePath,
filename: `${baseFilename}_${range.start}-${range.end}`,
rangeKey: `${range.start}-${range.end}`,
});
}
} else {
// Process Full Attachment
const tempFilePath = path.join(tempDir, `${baseFilename}.pdf`);
await fs.writeFile(tempFilePath, originalPdfBytes);
uploadPromises.push({
path: tempFilePath,
filename: baseFilename,
rangeKey: 'full',
});
}
// 0. Check if ASN already exists
if (att.belegnummer) {
await this.paperlessService.validateAsnNotExists(att.belegnummer);
}
// Upload all generated PDFs
for (const uploadItem of uploadPromises) {
const options: any = {
filename: uploadItem.filename,
title: att.belegnummer ? `Beleg ${att.belegnummer}` : uploadItem.filename,
created: createdDate,
owner: null,
};
if (att.paperlessCorrespondentId) options.correspondent = att.paperlessCorrespondentId;
const paperlessTaskId = await this.paperlessService.uploadDocument(uploadItem.path, options);
// Create background task for enrichment (same logic as Inbox)
const backgroundTask = this.taskRepo.create({
TaskId: paperlessTaskId,
InterneBelegnummer: att.belegnummer || '',
Eingangsdatum: att.barcode?.datum ? new Date(att.barcode.datum) : createdDate,
Belegdatum: createdDate,
BarcodeJson: att.barcode ? JSON.stringify(att.barcode) : null,
BetriebID: null, // Owner
Fertig: 0,
DocumentType: att.type === 'MAIN' ? null : 5, // 5 = Anlage
SourceAttachmentID: att.attachmentId,
SourceAttachmentRange: uploadItem.rangeKey,
});
await this.taskRepo.save(backgroundTask);
// Still poll for Doc ID so we can return it to the frontend for immediate preview
let docId = null;
for (let i = 0; i < 30; i++) {
await new Promise(resolve => setTimeout(resolve, 2000));
try {
const taskStatus = await this.paperlessService.getTask(paperlessTaskId);
const statusObj = Array.isArray(taskStatus) ? taskStatus[0] : taskStatus;
if (statusObj && statusObj.related_document) {
docId = statusObj.related_document;
break;
}
} catch(e) {}
}
if (docId) {
paperlessIds[uploadItem.rangeKey] = docId;
}
}
// Update Database
attachmentEntity.PaperlessDocumentIds = paperlessIds;
attachmentEntity.ImportStatus = 1;
if (att.belegnummer) {
attachmentEntity.InterneBelegnummer = att.belegnummer;
}
await this.attachmentRepo.save(attachmentEntity);
// Confirm Belegnummer if used
if (att.belegnummer && att.barcode?.nummer) {
await this.setBelegnummer(data.emailDate, att.barcode.nummer).catch(e => this.logger.warn(`Failed to set Belegnummer: ${e.message}`));
}
results.push({ attachmentId: att.attachmentId, paperlessIds });
}
// Mark Email as processed (Status = 1)
if (data.attachments.length > 0) {
const firstAtt = await this.attachmentRepo.findOne({
where: { Id: data.attachments[0].attachmentId }
});
if (firstAtt) {
await this.emailRepo.update(firstAtt.EmailMessageId, { Status: 1 });
this.logger.log(`Email ${firstAtt.EmailMessageId} als verarbeitet markiert.`);
}
}
return { success: true, results };
} finally {
// Clean up temp dir
await fs.rm(tempDir, { recursive: true, force: true }).catch(() => {});
}
}
}
@@ -0,0 +1,57 @@
import { Injectable, Logger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import * as path from 'path';
import * as fs from 'fs/promises';
import sharp from 'sharp';
const THUMBNAIL_WIDTH = 180;
@Injectable()
export class EmailPageCacheService {
private readonly logger = new Logger(EmailPageCacheService.name);
private readonly mailsRoot: string;
constructor(configService: ConfigService) {
this.mailsRoot = configService.get<string>('MAILS_DATA_DIR', '/mnt/data/mails');
}
attachmentDir(attachmentId: number | string): string {
return path.join(this.mailsRoot, attachmentId.toString());
}
previewPath(attachmentId: number | string, page: number): string {
return path.join(this.attachmentDir(attachmentId), `page-${page}.preview.png`);
}
thumbnailPath(attachmentId: number | string, page: number): string {
return path.join(this.attachmentDir(attachmentId), `page-${page}.thumb.png`);
}
async generate(attachmentId: number | string, renderedImages: string[]): Promise<void> {
const dir = this.attachmentDir(attachmentId);
await fs.mkdir(dir, { recursive: true });
for (let i = 0; i < renderedImages.length; i++) {
const page = i + 1;
const src = renderedImages[i];
const previewDest = this.previewPath(attachmentId, page);
const thumbDest = this.thumbnailPath(attachmentId, page);
try {
await fs.copyFile(src, previewDest);
await sharp(src).resize({ width: THUMBNAIL_WIDTH }).png().toFile(thumbDest);
} catch (err: any) {
this.logger.warn(`E-Mail Page Cache fehlgeschlagen (Attachment ${attachmentId} Seite ${page}): ${err.message}`);
}
}
}
async hasPreview(attachmentId: number | string, page: number): Promise<boolean> {
try {
await fs.access(this.previewPath(attachmentId, page));
return true;
} catch {
return false;
}
}
}
@@ -0,0 +1,55 @@
import { Test, TestingModule } from '@nestjs/testing';
import { getRepositoryToken } from '@nestjs/typeorm';
import { EmailController } from './email.controller';
import { Email } from '../database/entities/email.entity';
const mockEmails: Partial<Email>[] = [
{ Id: 1, MessageId: 'msg-1', SenderAddress: 'a@test.de', RecipientAddress: 'b@test.de', Subject: 'Test 1', Date: new Date(), Body: 'body', Status: 0 },
{ Id: 2, MessageId: 'msg-2', SenderAddress: 'c@test.de', RecipientAddress: 'd@test.de', Subject: 'Test 2', Date: new Date(), Body: 'body2', Status: 1 },
];
const mockQueryBuilder = {
orderBy: jest.fn().mockReturnThis(),
take: jest.fn().mockReturnThis(),
where: jest.fn().mockReturnThis(),
getMany: jest.fn().mockResolvedValue(mockEmails),
};
const mockRepo = {
createQueryBuilder: jest.fn().mockReturnValue(mockQueryBuilder),
findOneByOrFail: jest.fn().mockResolvedValue(mockEmails[0]),
};
describe('EmailController', () => {
let controller: EmailController;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
controllers: [EmailController],
providers: [{ provide: getRepositoryToken(Email), useValue: mockRepo }],
}).compile();
controller = module.get<EmailController>(EmailController);
});
it('should be defined', () => {
expect(controller).toBeDefined();
});
it('getEmails returns list', async () => {
const result = await controller.getEmails();
expect(result).toHaveLength(2);
expect(mockQueryBuilder.orderBy).toHaveBeenCalledWith('e.Date', 'DESC');
});
it('getEmails filters by status', async () => {
await controller.getEmails('1');
expect(mockQueryBuilder.where).toHaveBeenCalledWith('e.Status = :status', { status: 1 });
});
it('getEmail returns single item', async () => {
const result = await controller.getEmail('1');
expect(result).toEqual(mockEmails[0]);
expect(mockRepo.findOneByOrFail).toHaveBeenCalledWith({ Id: 1 });
});
});
@@ -0,0 +1,149 @@
import { Controller, Get, Post, Param, Query, Res, Logger, NotFoundException, Patch, Body } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import type { Response } from 'express';
import { Email } from '../database/entities/email.entity';
import { Attachment } from '../database/entities/attachment.entity';
import { Content } from '../database/entities/content.entity';
import { PaperlessService } from '../paperless/paperless.service';
import { RequirePermissions } from '../auth/permissions.decorator';
import { Permission } from '../auth/permissions.enum';
@Controller('api/emails')
export class EmailController {
private readonly logger = new Logger(EmailController.name);
constructor(
@InjectRepository(Email) private readonly emailRepo: Repository<Email>,
@InjectRepository(Attachment) private readonly attachmentRepo: Repository<Attachment>,
@InjectRepository(Content) private readonly contentRepo: Repository<Content>,
private readonly paperlessService: PaperlessService,
) {}
@Get()
async getEmails(
@Query('status') status?: string,
@Query('limit') limit?: string,
) {
const qb = this.emailRepo.createQueryBuilder('e')
.leftJoinAndSelect('e.Attachments', 'a')
.orderBy('e.Date', 'DESC')
.take(parseInt(limit ?? '50', 10));
if (status !== undefined) {
qb.where('e.Status = :status', { status: parseInt(status, 10) });
}
return qb.getMany();
}
@Get(':id')
async getEmail(@Param('id') id: string) {
return this.emailRepo.findOneOrFail({
where: { Id: parseInt(id, 10) },
relations: ['Attachments'],
});
}
@Get(':id/attachments')
async getAttachments(@Param('id') id: string) {
return this.attachmentRepo.find({
where: { EmailMessageId: parseInt(id, 10) },
order: { Id: 'ASC' },
});
}
@Get('attachments/:attachmentId/content')
async getAttachmentContent(
@Param('attachmentId') attachmentId: string,
@Res() res: Response,
) {
const id = parseInt(attachmentId, 10);
const attachment = await this.attachmentRepo.findOne({ where: { Id: id } });
if (!attachment) throw new NotFoundException('Anhang nicht gefunden');
const content = await this.contentRepo.findOne({ where: { AttachmentEntityId: id } });
if (!content) throw new NotFoundException('Inhalt nicht gefunden');
res.setHeader('Content-Type', attachment.ContentType || 'application/octet-stream');
res.setHeader('Content-Disposition', `inline; filename="${encodeURIComponent(attachment.FileName)}"`);
res.send(content.Content1);
}
@Patch(':id/status')
@RequirePermissions(Permission.MANAGE_ALL)
async updateStatus(
@Param('id') id: string,
@Body('status') status: number,
) {
const email = await this.emailRepo.findOneOrFail({ where: { Id: parseInt(id, 10) } });
email.Status = status;
await this.emailRepo.save(email);
this.logger.log(`E-Mail ${id} auf Status ${status} gesetzt.`);
return { message: 'Status aktualisiert' };
}
@Post('check-attachments')
@RequirePermissions(Permission.MANAGE_ALL)
async checkAttachments() {
this.logger.log('Starte manuelle Prüfung der E-Mail-Anhänge in Paperless...');
try {
// Hole alle neuen E-Mails (Status = 0) inkl. Anhängen
const emails = await this.emailRepo.find({
where: { Status: 0 },
relations: ['Attachments'],
});
this.logger.log(`Gefunden: ${emails.length} E-Mails mit Status "Neu" (0). Beginne Prüfung...`);
let updatedCount = 0;
let skippedCount = 0;
for (const [index, email] of emails.entries()) {
if (!email.Attachments || email.Attachments.length === 0) {
skippedCount++;
continue;
}
let hasMatch = false;
for (const attachment of email.Attachments) {
// Prüfe nur PDFs und wenn eine Checksumme vorhanden ist
if (attachment.ContentType === 'application/pdf' && attachment.Checksum) {
this.logger.debug(`Prüfe Checksumme für E-Mail ${email.Id}, Anhang ${attachment.Id} (${attachment.FileName})`);
try {
const exists = await this.paperlessService.checksumExists(attachment.Checksum);
if (exists) {
this.logger.log(`Treffer! Anhang ${attachment.Id} (E-Mail ${email.Id}) in Paperless gefunden.`);
hasMatch = true;
break; // Ein Treffer reicht für diese E-Mail
}
} catch (err: any) {
this.logger.error(`Fehler bei Checksummen-Prüfung für Attachment ${attachment.Id}: ${err.message}`, err.stack);
}
}
}
// Wenn mindestens ein Anhang in Paperless existiert, markiere die Mail als verarbeitet (Status = 1)
if (hasMatch) {
email.Status = 1;
await this.emailRepo.save(email);
updatedCount++;
this.logger.log(`E-Mail ${email.Id} auf Status 1 (Verarbeitet) gesetzt.`);
}
// Zwischenstand loggen
if ((index + 1) % 10 === 0) {
this.logger.log(`Zwischenstand: ${index + 1} von ${emails.length} E-Mails geprüft.`);
}
}
this.logger.log(`Prüfung abgeschlossen. ${updatedCount} aktualisiert, ${skippedCount} ohne (PDF-)Anhänge übersprungen.`);
return { updatedCount };
} catch (error: any) {
this.logger.error(`Kritischer Fehler bei checkAttachments: ${error.message}`, error.stack);
throw error;
}
}
}
@@ -0,0 +1,26 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { Email } from '../database/entities/email.entity';
import { Attachment } from '../database/entities/attachment.entity';
import { Content } from '../database/entities/content.entity';
import { EmailController } from './email.controller';
import { PaperlessModule } from '../paperless/paperless.module';
import { EmailPageCacheService } from './email-page-cache.service';
import { EmailImportController } from './email-import.controller';
import { EmailImportService } from './email-import.service';
import { CorrespondentEmailMapping } from '../database/entities/correspondent-email-mapping.entity';
import { Task } from '../database/entities/task.entity';
import { PreprocessingModule } from '../preprocessing/preprocessing.module';
@Module({
imports: [
TypeOrmModule.forFeature([Email, Attachment, Content, CorrespondentEmailMapping, Task]),
PaperlessModule,
PreprocessingModule,
],
controllers: [EmailController, EmailImportController],
providers: [EmailImportService, EmailPageCacheService],
exports: [EmailPageCacheService],
})
export class EmailModule {}
@@ -0,0 +1,70 @@
import * as fs from 'fs/promises';
import * as os from 'os';
import * as path from 'path';
import { PDFDocument, degrees } from 'pdf-lib';
import type { InboxDocument } from '../database/entities/inbox-document.entity';
/**
* Wendet die virtuellen Edits (DeletedPages, Rotations) auf das Original-PDF an
* und schreibt das Ergebnis in eine temporäre Datei. Gibt den Pfad zurück.
* Aufrufer ist verantwortlich für das Aufräumen.
*/
export async function applyEditsToTemp(
doc: InboxDocument,
pdfPath: string,
): Promise<string> {
const bytes = await fs.readFile(pdfPath);
const pdf = await PDFDocument.load(bytes, { ignoreEncryption: true });
const rotations = doc.Rotations ?? {};
for (const [pageStr, rot] of Object.entries(rotations)) {
const pageNum = Number(pageStr);
if (!Number.isInteger(pageNum)) continue;
const idx = pageNum - 1;
if (idx < 0 || idx >= pdf.getPageCount()) continue;
const normalized = ((Math.round(rot / 90) * 90) % 360 + 360) % 360;
if (normalized === 0) continue;
pdf.getPage(idx).setRotation(degrees(normalized));
}
// Seiten in absteigender Reihenfolge entfernen, damit Indizes stabil bleiben
const deleted = [...(doc.DeletedPages ?? [])].sort((a, b) => b - a);
for (const pageNum of deleted) {
const idx = pageNum - 1;
if (idx < 0 || idx >= pdf.getPageCount()) continue;
pdf.removePage(idx);
}
const out = await pdf.save();
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'inbox-pp-'));
const tmpPath = path.join(tmpDir, 'document.pdf');
await fs.writeFile(tmpPath, out);
return tmpPath;
}
export async function cleanupTemp(filePath: string | null): Promise<void> {
if (!filePath) return;
try {
await fs.unlink(filePath);
await fs.rmdir(path.dirname(filePath)).catch(() => undefined);
} catch {
// ignore
}
}
export async function extractSectionToTemp(
pdfPath: string,
pageIndices: number[],
): Promise<string> {
const bytes = await fs.readFile(pdfPath);
const srcPdf = await PDFDocument.load(bytes, { ignoreEncryption: true });
const outPdf = await PDFDocument.create();
const copied = await outPdf.copyPages(srcPdf, pageIndices);
copied.forEach(p => outPdf.addPage(p));
const out = await outPdf.save();
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'inbox-section-'));
const tmpPath = path.join(tmpDir, 'section.pdf');
await fs.writeFile(tmpPath, out);
return tmpPath;
}
@@ -0,0 +1,22 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { InboxPostprocessingAction } from '../database/entities/inbox-postprocessing-action.entity';
import { InboxDocument } from '../database/entities/inbox-document.entity';
import { BarcodeTemplate } from '../database/entities/barcode-template.entity';
import { Task } from '../database/entities/task.entity';
import { InboxPostprocessorService } from './inbox-postprocessor.service';
import { BarcodeModule } from '../barcode/barcode.module';
import { PaperlessModule } from '../paperless/paperless.module';
import { PostprocessingModule } from '../postprocessing/postprocessing.module';
@Module({
imports: [
TypeOrmModule.forFeature([InboxPostprocessingAction, InboxDocument, BarcodeTemplate, Task]),
BarcodeModule,
PaperlessModule,
PostprocessingModule,
],
providers: [InboxPostprocessorService],
exports: [InboxPostprocessorService],
})
export class InboxPostprocessorModule {}
@@ -0,0 +1,488 @@
import { Injectable, Logger, NotFoundException } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import * as fs from 'fs/promises';
import * as path from 'path';
import * as crypto from 'crypto';
import {
InboxPostprocessingAction,
} from '../database/entities/inbox-postprocessing-action.entity';
import { InboxDocument } from '../database/entities/inbox-document.entity';
import { BarcodeTemplate } from '../database/entities/barcode-template.entity';
import { Task } from '../database/entities/task.entity';
import { PageCacheService } from '../barcode/page-cache.service';
import { PaperlessService } from '../paperless/paperless.service';
import { MailService } from '../postprocessing/mail.service';
import { ExportService } from '../postprocessing/export.service';
import { applyEditsToTemp, cleanupTemp, extractSectionToTemp } from './edit-applier';
import { applyTemplate, buildVariables } from './variable-resolver';
function parseFlexDate(s: string): Date | null {
if (!s) return null;
// ISO: YYYY-MM-DD
if (/^\d{4}-\d{2}-\d{2}$/.test(s)) {
const d = new Date(s);
return isNaN(d.getTime()) ? null : d;
}
// German: DD.MM.YYYY
const m = s.match(/^(\d{1,2})\.(\d{1,2})\.(\d{4})$/);
if (m) {
const d = new Date(`${m[3]}-${m[2].padStart(2, '0')}-${m[1].padStart(2, '0')}`);
return isNaN(d.getTime()) ? null : d;
}
return null;
}
export interface ActionResult {
sectionIndex: number;
actionId: number;
actionType: string;
ok: boolean;
skipped?: boolean;
message?: string;
duplicateOfDocumentId?: number;
}
export interface RunForDocumentResult {
results: ActionResult[];
totalSections: number;
}
interface PaperlessRunResult {
skipped?: boolean;
message?: string;
duplicateOfDocumentId?: number;
}
@Injectable()
export class InboxPostprocessorService {
private readonly logger = new Logger(InboxPostprocessorService.name);
constructor(
private readonly pageCache: PageCacheService,
private readonly paperlessService: PaperlessService,
private readonly mailService: MailService,
private readonly exportService: ExportService,
@InjectRepository(InboxPostprocessingAction)
private readonly actionRepo: Repository<InboxPostprocessingAction>,
@InjectRepository(InboxDocument)
private readonly docRepo: Repository<InboxDocument>,
@InjectRepository(BarcodeTemplate)
private readonly templateRepo: Repository<BarcodeTemplate>,
@InjectRepository(Task)
private readonly taskRepo: Repository<Task>,
) {}
async runForDocument(
documentId: string,
preferredUsername: string | null,
sectionOffset: number = 0,
processOnlyOne: boolean = false,
replaceDuplicate: boolean = false,
): Promise<RunForDocumentResult> {
const doc = await this.docRepo.findOne({ where: { Id: documentId } });
if (!doc) throw new NotFoundException('Dokument nicht gefunden');
if (doc.Source === 'user' && doc.OwnerUsername !== preferredUsername) {
throw new NotFoundException('Dokument nicht gefunden');
}
const sourcePdf = this.pageCache.documentPdfPath(doc.Id);
try {
await fs.access(sourcePdf);
} catch {
throw new NotFoundException('Dokument-PDF fehlt');
}
const templates = await this.templateRepo.find({ order: { Id: 'ASC' } });
const matchedTemplateIds = [
...new Set(
doc.QrCodes
.map((qr) => {
const tpl = templates.find((t) => {
try { return new RegExp(t.Regex).test(qr.value); }
catch { return false; }
});
return tpl?.Id ?? null;
})
.filter((id): id is number => id !== null),
),
];
if (matchedTemplateIds.length === 0) return { results: [], totalSections: 0 };
const actions = await this.actionRepo.find({
where: matchedTemplateIds.map((tid) => ({ BarcodeTemplateId: tid, IsActive: true })),
order: { BarcodeTemplateId: 'ASC', Order: 'ASC', Id: 'ASC' },
});
if (actions.length === 0) {
return { results: [], totalSections: 0 };
}
// Aktionen nach Template gruppieren, damit jede Gruppe ihre eigenen Barcode-Variablen bekommt
const actionsByTemplate = new Map<number, InboxPostprocessingAction[]>();
for (const action of actions) {
const tid = action.BarcodeTemplateId!;
if (!actionsByTemplate.has(tid)) actionsByTemplate.set(tid, []);
actionsByTemplate.get(tid)!.push(action);
}
let processedPdfPath: string | null = null;
let abortProcessing = false;
const results: ActionResult[] = [];
try {
processedPdfPath = await applyEditsToTemp(doc, sourcePdf);
// Überlebende Seiten berechnen (1-basiert auf Original)
const deletedSet = new Set(doc.DeletedPages ?? []);
const survivingOriginalPages: number[] = [];
for (let p = 1; p <= doc.PageCount; p++) {
if (!deletedSet.has(p)) survivingOriginalPages.push(p);
}
// Mapping: Original-Seite → 0-basierter Index in processedPdf
const processedPageIndex = new Map<number, number>();
survivingOriginalPages.forEach((origPage, idx) => processedPageIndex.set(origPage, idx));
// QR-Codes mit ihrem Index in der verarbeiteten PDF (gelöschte QR-Seiten herausfiltern)
const qrsWithIdx = doc.QrCodes
.filter(qr => processedPageIndex.has(qr.page))
.map(qr => ({ page: qr.page, value: qr.value, processedIdx: processedPageIndex.get(qr.page)! }))
.sort((a, b) => a.processedIdx - b.processedIdx);
// Split-Punkte: 0-basierte Indizes in der verarbeiteten PDF wo SplitBefore-Templates beginnen
const splitPoints: number[] = [];
for (const qr of qrsWithIdx) {
const tplMatch = templates.find(t => { try { return new RegExp(t.Regex).test(qr.value); } catch { return false; } });
if (tplMatch?.SplitBefore) splitPoints.push(qr.processedIdx);
}
const processedPageCount = survivingOriginalPages.length;
// Gesamtzahl aktiver Abschnitte vorab berechnen (für Stepper-UI im Frontend)
let totalSections = 0;
for (let i = 0; i < qrsWithIdx.length; i++) {
const qr = qrsWithIdx[i];
const tpl = templates.find(t => { try { return new RegExp(t.Regex).test(qr.value); } catch { return false; } });
if (!tpl) continue;
if (i > 0 && !tpl.SplitBefore) continue;
totalSections++;
}
// Ersten Barcode immer verarbeiten (SplitBefore egal).
// Weitere Barcodes nur wenn SplitBefore=true → neues Dokument.
// sectionOffset: erste N aktive Abschnitte überspringen (für "Nächstes Dokument").
// processOnlyOne: nach dem ersten verarbeiteten Abschnitt abbrechen (für Wizard).
let activeSectionCount = 0;
let processedSection = false;
for (let i = 0; i < qrsWithIdx.length; i++) {
if (abortProcessing) break;
if (processedSection && processOnlyOne) break;
const qr = qrsWithIdx[i];
const tpl = templates.find(t => { try { return new RegExp(t.Regex).test(qr.value); } catch { return false; } });
if (!tpl) continue;
if (i > 0 && !tpl.SplitBefore) continue;
if (activeSectionCount < sectionOffset) {
activeSectionCount++;
continue;
}
const currentSectionIndex = activeSectionCount;
activeSectionCount++;
const tplActions = actionsByTemplate.get(tpl.Id);
if (!tplActions || tplActions.length === 0) continue;
const variables = buildVariables({ doc, template: tpl, matchingQrValue: qr.value });
// Abschnitt aus der verarbeiteten PDF extrahieren
const startIdx = qr.processedIdx;
const nextSplitIdx = splitPoints.find(sp => sp > startIdx);
const endIdx = nextSplitIdx !== undefined ? nextSplitIdx - 1 : processedPageCount - 1;
const pageIndices = Array.from({ length: endIdx - startIdx + 1 }, (_, i) => startIdx + i);
const sectionPdfPath = await extractSectionToTemp(processedPdfPath, pageIndices);
const defaultFilenameBase = tpl.DateinameTemplate
? applyTemplate(tpl.DateinameTemplate, variables)
: undefined;
try {
for (const action of tplActions) {
if (abortProcessing) break;
if (action.ActionType === 'PAPERLESS') {
try {
const res = await this.runPaperless(action.Content ?? {}, sectionPdfPath, variables, defaultFilenameBase, replaceDuplicate);
results.push({
sectionIndex: currentSectionIndex,
actionId: action.Id,
actionType: action.ActionType,
ok: true,
skipped: res.skipped,
message: res.message,
duplicateOfDocumentId: res.duplicateOfDocumentId,
});
if (res.skipped) {
abortProcessing = true;
break;
}
} catch (err: any) {
this.logger.error(
`Aktion PAPERLESS#${action.Id} für Dokument ${doc.Id} fehlgeschlagen: ${err.message}`,
);
results.push({ sectionIndex: currentSectionIndex, actionId: action.Id, actionType: action.ActionType, ok: false, message: err.message });
}
} else {
try {
await this.runAction(action, sectionPdfPath, doc, variables, defaultFilenameBase);
results.push({ sectionIndex: currentSectionIndex, actionId: action.Id, actionType: action.ActionType, ok: true });
} catch (err: any) {
this.logger.error(
`Aktion ${action.ActionType}#${action.Id} für Dokument ${doc.Id} fehlgeschlagen: ${err.message}`,
);
results.push({ sectionIndex: currentSectionIndex, actionId: action.Id, actionType: action.ActionType, ok: false, message: err.message });
}
}
}
} finally {
await cleanupTemp(sectionPdfPath);
}
processedSection = true;
}
return { results, totalSections };
} finally {
await cleanupTemp(processedPdfPath);
}
}
private async runAction(
action: InboxPostprocessingAction,
pdfPath: string,
doc: InboxDocument,
variables: Record<string, string>,
defaultFilenameBase?: string,
): Promise<void> {
const content = action.Content ?? {};
switch (action.ActionType) {
case 'MAIL':
return this.runMail(content, pdfPath, doc, variables, defaultFilenameBase);
case 'EXPORT':
return this.runExport(content, pdfPath, variables, defaultFilenameBase);
default:
throw new Error(`Unbekannter ActionType: ${action.ActionType}`);
}
}
private async runMail(
content: Record<string, any>,
pdfPath: string,
doc: InboxDocument,
variables: Record<string, string>,
defaultFilenameBase?: string,
): Promise<void> {
const to = applyTemplate(String(content.to ?? ''), variables).trim();
if (!to) throw new Error('Empfänger fehlt');
const subject = applyTemplate(String(content.subject ?? ''), variables);
const body = applyTemplate(String(content.body ?? ''), variables);
const filenameTpl = String(content.filenameTemplate ?? '').trim();
const filename = filenameTpl
? `${applyTemplate(filenameTpl, variables)}.pdf`
: defaultFilenameBase
? `${defaultFilenameBase}.pdf`
: doc.OriginalName;
const buffer = await fs.readFile(pdfPath);
await this.mailService.sendMail({
to,
subject,
body,
attachments: [{ filename, content: buffer }],
});
}
private async runExport(
content: Record<string, any>,
pdfPath: string,
variables: Record<string, string>,
defaultFilenameBase?: string,
): Promise<void> {
const targetId = Number(content.exportTargetId);
if (!targetId) throw new Error('Export-Ziel fehlt');
const filenameTpl = String(content.filenameTemplate ?? '').trim();
const filename = filenameTpl
? `${applyTemplate(filenameTpl, variables)}.pdf`
: defaultFilenameBase
? `${defaultFilenameBase}.pdf`
: `${path.basename(pdfPath, '.pdf')}.pdf`;
const buffer = await fs.readFile(pdfPath);
await this.exportService.exportFile(targetId, filename, buffer);
}
private async runPaperless(
content: Record<string, any>,
pdfPath: string,
variables: Record<string, string>,
defaultFilenameBase?: string,
replaceDuplicate: boolean = false,
): Promise<PaperlessRunResult> {
// 1. Interne Belegnummer auflösen (Pflicht)
const intNrTpl = String(content.interneBelegnummer ?? '').trim();
if (!intNrTpl) throw new Error('Interne Belegnummer ist in der Aktion nicht konfiguriert');
const interneBelegnummer = applyTemplate(intNrTpl, variables).trim();
if (!interneBelegnummer) throw new Error('Interne Belegnummer konnte nicht aufgelöst werden');
// 2. ASN bestimmen (vor Duplikat-Checks, damit Paperless danach gefragt werden kann)
const asnTpl = String(content.asn ?? '').trim();
const asn = asnTpl ? applyTemplate(asnTpl, variables).trim() : null;
let archiveSerialNumber: number | undefined;
if (asn) {
const n = parseInt(asn.replace(/[^0-9]/g, ''), 10);
if (!Number.isNaN(n)) archiveSerialNumber = n;
}
if (archiveSerialNumber === undefined) {
const n = parseInt(interneBelegnummer.replace(/[^0-9]/g, ''), 10);
if (!Number.isNaN(n)) archiveSerialNumber = n;
}
if (replaceDuplicate) {
this.logger.log(`Duplikat-Check übersprungen (replaceDuplicate=true) für ${interneBelegnummer}`);
} else {
// 3. Duplikat-Check lokal (tasks-Tabelle)
const existingTask = await this.taskRepo.findOneBy({ InterneBelegnummer: interneBelegnummer });
if (existingTask) {
this.logger.warn(`Duplikat (lokal): Belegnummer ${interneBelegnummer} bereits in tasks-Tabelle`);
return {
skipped: true,
message: `Duplikat Belegnummer ${interneBelegnummer} bereits vorhanden`,
duplicateOfDocumentId: existingTask.PaperlessDocumentID ?? undefined,
};
}
// 4. Duplikat-Check Paperless API (Custom Field 7 = InterneBelegnummer)
const cf7DocId = await this.paperlessService.findDocumentIdByCustomField(7, interneBelegnummer);
if (cf7DocId !== null) {
this.logger.warn(`Duplikat (Paperless CF7): Belegnummer ${interneBelegnummer} bereits in Paperless (Doc ${cf7DocId})`);
return {
skipped: true,
message: `Duplikat Belegnummer ${interneBelegnummer} bereits in Paperless`,
duplicateOfDocumentId: cf7DocId,
};
}
// 5. Duplikat-Check Paperless API (archive_serial_number)
if (archiveSerialNumber !== undefined) {
const asnDocId = await this.paperlessService.findDocumentIdByAsn(archiveSerialNumber);
if (asnDocId !== null) {
this.logger.warn(`Duplikat (Paperless ASN): ${archiveSerialNumber} bereits in Paperless (Doc ${asnDocId})`);
return {
skipped: true,
message: `Duplikat ASN ${archiveSerialNumber} bereits in Paperless`,
duplicateOfDocumentId: asnDocId,
};
}
}
// 6. Checksum berechnen und prüfen
const buffer = await fs.readFile(pdfPath);
const checksum = crypto.createHash('md5').update(buffer).digest('hex');
const checksumExists = await this.paperlessService.checksumExists(checksum);
if (checksumExists) {
this.logger.warn(`Duplikat (Checksum): ${checksum}`);
return { skipped: true, message: 'Duplikat (Checksum-Übereinstimmung)' };
}
}
// 7. Restliche Metadaten auflösen
const titleTpl = String(content.title ?? '').trim();
const title = titleTpl
? applyTemplate(titleTpl, variables)
: defaultFilenameBase || undefined;
const tags = Array.isArray(content.tags)
? content.tags.map((t: any) => Number(t)).filter((n: number) => Number.isFinite(n))
: undefined;
const documentType = content.documentType ? Number(content.documentType) : undefined;
const correspondent = content.correspondent ? Number(content.correspondent) : undefined;
const owner = content.owner !== undefined && content.owner !== null && content.owner !== ''
? Number(content.owner)
: undefined;
const rawCustomFields: Record<string, string> | null =
content.customFields && typeof content.customFields === 'object'
? Object.fromEntries(
Object.entries(content.customFields as Record<string, any>).map(([k, v]) => [
k,
applyTemplate(String(v ?? ''), variables),
]),
)
: null;
// Eingangsdatum: Priorität 1 = dediziertes Feld, Priorität 2 = Custom Field 9, Fallback = heute
const eingangsdatumTpl = String(content.eingangsdatum ?? '').trim();
let eingangsdatum: Date;
if (eingangsdatumTpl) {
eingangsdatum = parseFlexDate(applyTemplate(eingangsdatumTpl, variables).trim()) ?? new Date();
} else if (rawCustomFields?.['9']) {
const parsed = parseFlexDate(rawCustomFields['9']);
if (parsed) {
eingangsdatum = parsed;
delete rawCustomFields['9']; // Task-Processor schreibt Field 9 in korrektem ISO-Format
} else {
eingangsdatum = new Date();
}
} else {
eingangsdatum = new Date();
}
// Custom Fields für den Upload zusammenstellen:
// - CF7 = InterneBelegnummer (für Duplikat-Check via Paperless API)
// - CF9 = Eingangsdatum in ISO-Format
// - User-konfigurierte Felder aus rawCustomFields
const uploadCustomFields: Record<string, string> = {};
if (rawCustomFields) {
for (const [k, v] of Object.entries(rawCustomFields)) uploadCustomFields[k] = v;
}
uploadCustomFields['7'] = interneBelegnummer;
uploadCustomFields['9'] = `${eingangsdatum.getFullYear()}-${String(eingangsdatum.getMonth() + 1).padStart(2, '0')}-${String(eingangsdatum.getDate()).padStart(2, '0')}`;
// 6. Upload
const taskIdRaw = await this.paperlessService.uploadDocument(pdfPath, {
title,
filename: defaultFilenameBase || undefined,
documentType,
correspondent,
owner,
tags,
archiveSerialNumber,
customFields: uploadCustomFields,
});
const taskId = String(taskIdRaw).replace(/"/g, '');
// 7. Task anlegen
const task = this.taskRepo.create({
TaskId: taskId,
InterneBelegnummer: interneBelegnummer,
DocumentType: documentType ?? null,
Eingangsdatum: eingangsdatum,
Fertig: 0,
Tags: tags && tags.length > 0 ? tags.join(',') : null,
BetriebID: owner ?? null,
externeBelegnummer: null,
CustomFieldsJson: rawCustomFields && Object.keys(rawCustomFields).length > 0
? JSON.stringify(rawCustomFields)
: null,
Asn: asn || null,
Lieferant: null,
EinkaufID: null,
Belegdatum: null,
TaskReferenceID: null,
BarcodeJson: null,
DuplikatZU: null,
});
await this.taskRepo.save(task);
this.logger.log(`Dokument hochgeladen und Task angelegt: ${interneBelegnummer}${taskId}`);
return {};
}
}
@@ -0,0 +1,66 @@
import type { BarcodeTemplate } from '../database/entities/barcode-template.entity';
import type { InboxDocument } from '../database/entities/inbox-document.entity';
export interface ResolverContext {
doc: InboxDocument;
template: BarcodeTemplate;
matchingQrValue: string | null;
now?: Date;
}
function pad(n: number): string {
return n < 10 ? `0${n}` : String(n);
}
function dateVars(d: Date): Record<string, string> {
const yyyy = String(d.getFullYear());
const mm = pad(d.getMonth() + 1);
const dd = pad(d.getDate());
return {
datum: `${yyyy}-${mm}-${dd}`,
jahr: yyyy,
monat: mm,
tag: dd,
zeitstempel: d.toISOString(),
};
}
function sanitizeKey(name: string): string {
return name.replace(/[^A-Za-z0-9_]/g, '_');
}
/**
* Berechnet alle Platzhalter für eine Aktion einer bestimmten Barcode-Vorlage:
* - Datums-Variablen: {datum}, {jahr}, {monat}, {tag}, {zeitstempel}
* - {barcode} = gesamter Barcode-Wert
* - {barcode.<gruppe>} = Named Capture Group aus der Vorlage-Regex
*/
export function buildVariables(ctx: ResolverContext): Record<string, string> {
const vars: Record<string, string> = {};
Object.assign(vars, dateVars(ctx.now ?? new Date()));
if (ctx.matchingQrValue !== null) {
vars['barcode'] = ctx.matchingQrValue;
try {
const m = ctx.matchingQrValue.match(new RegExp(ctx.template.Regex));
if (m?.groups) {
for (const [g, gv] of Object.entries(m.groups)) {
if (gv !== undefined) vars[`barcode.${sanitizeKey(g)}`] = gv;
}
}
} catch {}
}
return vars;
}
/**
* Ersetzt Platzhalter der Form `{name}` und `{name.group}` im Template.
* Unbekannte Platzhalter bleiben unverändert.
*/
export function applyTemplate(template: string, vars: Record<string, string>): string {
if (!template) return template;
return template.replace(/\{([A-Za-z0-9_.]+)\}/g, (full, name: string) => {
return name in vars ? vars[name] : full;
});
}
@@ -0,0 +1,29 @@
import { Controller, Get, Request, Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository, In } from 'typeorm';
import { Client } from '../database/entities/client.entity';
import { UserClient } from '../database/entities/user-client.entity';
@Controller('api/clients')
export class ClientsController {
private readonly logger = new Logger(ClientsController.name);
constructor(
@InjectRepository(Client) private readonly clientRepo: Repository<Client>,
@InjectRepository(UserClient) private readonly userClientRepo: Repository<UserClient>,
) {}
@Get()
async getMyClients(@Request() req: any) {
const userId = req.user.userId;
const mappings = await this.userClientRepo.find({ where: { UserId: userId } });
const clientIds = mappings.map((m) => m.ClientId);
if (clientIds.length === 0) {
// Fallback to match old C# behavior where it returned all clients
// or if admin users shouldn't be restricted initially.
return this.clientRepo.find();
}
return this.clientRepo.find({ where: { Id: In(clientIds) } });
}
}
@@ -0,0 +1,139 @@
import { Injectable, Logger, OnApplicationBootstrap } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { InjectRepository } from '@nestjs/typeorm';
import { DataSource, Repository } from 'typeorm';
import { randomUUID } from 'crypto';
import * as path from 'path';
import * as fs from 'fs/promises';
import {
InboxDocument,
type InboxSource,
type StoredQrCode,
} from '../database/entities/inbox-document.entity';
import { PageCacheService } from '../barcode/page-cache.service';
interface LegacyScanRow {
QrCodes: string | StoredQrCode[];
}
@Injectable()
export class InboxMigrationService implements OnApplicationBootstrap {
private readonly logger = new Logger(InboxMigrationService.name);
private readonly legacyRoot: string;
constructor(
private readonly configService: ConfigService,
private readonly pageCache: PageCacheService,
private readonly dataSource: DataSource,
@InjectRepository(InboxDocument)
private readonly documentRepo: Repository<InboxDocument>,
) {
this.legacyRoot = this.configService.get<string>('SCANS_DATA_DIR', '/mnt/data/scans');
}
async onApplicationBootstrap(): Promise<void> {
let subdirs: string[];
try {
const entries = await fs.readdir(this.legacyRoot, { withFileTypes: true });
subdirs = entries.filter((e) => e.isDirectory()).map((e) => e.name);
} catch (err: any) {
if (err.code !== 'ENOENT') {
this.logger.warn(`Migration: ${this.legacyRoot} nicht lesbar: ${err.message}`);
}
return;
}
let migrated = 0;
for (const subdir of subdirs) {
const dir = path.join(this.legacyRoot, subdir);
let files: string[];
try {
files = await fs.readdir(dir);
} catch (err: any) {
this.logger.warn(`Migration: ${dir} nicht lesbar: ${err.message}`);
continue;
}
for (const name of files) {
if (path.extname(name).toLowerCase() !== '.pdf') continue;
const src = path.join(dir, name);
try {
await this.migrateFile(src, subdir, name);
migrated += 1;
} catch (err: any) {
this.logger.error(`Migration fehlgeschlagen (${src}): ${err.message}`);
}
}
await fs.rmdir(dir).catch(() => undefined);
}
if (migrated > 0) {
this.logger.log(`Migration: ${migrated} Datei(en) übernommen`);
} else {
this.logger.log('Migration: keine Altdaten gefunden');
}
}
private async migrateFile(src: string, subdir: string, name: string): Promise<void> {
const id = randomUUID();
const source: InboxSource = subdir === 'all' ? 'all' : 'user';
const owner = source === 'all' ? null : subdir;
const targetDir = this.pageCache.documentDir(id);
const targetPdf = this.pageCache.documentPdfPath(id);
await fs.mkdir(targetDir, { recursive: true });
await this.move(src, targetPdf);
const qrCodes = await this.loadLegacyQrCodes(src);
const doc = this.documentRepo.create({
Id: id,
OriginalName: name,
Source: source,
OwnerUsername: owner,
PageCount: 0,
QrCodes: qrCodes,
});
await this.documentRepo.save(doc);
}
private async move(src: string, dest: string): Promise<void> {
try {
await fs.rename(src, dest);
return;
} catch (err: any) {
if (err.code !== 'EXDEV') throw err;
}
await fs.copyFile(src, dest);
try {
await fs.unlink(src);
} catch (err) {
await fs.unlink(dest).catch(() => undefined);
throw err;
}
}
private async loadLegacyQrCodes(oldFilePath: string): Promise<StoredQrCode[]> {
try {
const rows = await this.dataSource.query<LegacyScanRow[]>(
'SELECT QrCodes FROM barcode_scans WHERE FilePath = ? LIMIT 1',
[oldFilePath],
);
if (!rows || rows.length === 0) return [];
const raw = rows[0].QrCodes;
if (typeof raw === 'string') {
try {
return JSON.parse(raw) as StoredQrCode[];
} catch {
return [];
}
}
return Array.isArray(raw) ? raw : [];
} catch {
// Tabelle fehlt oder anderes DB-Problem: Backfill scannt später.
return [];
}
}
}
@@ -0,0 +1,145 @@
import {
BadRequestException,
Body,
Controller,
Delete,
Get,
HttpCode,
Param,
ParseIntPipe,
Post,
Put,
Request,
Res,
StreamableFile,
} from '@nestjs/common';
import type { Response } from 'express';
import { createReadStream } from 'fs';
import { InboxService } from './inbox.service';
import { InboxPostprocessorService } from '../inbox-postprocessor/inbox-postprocessor.service';
import { BarcodeScannerService } from '../barcode/barcode-scanner.service';
import { RequirePermissions } from '../auth/permissions.decorator';
import { Permission } from '../auth/permissions.enum';
@Controller('api/inbox')
@RequirePermissions(Permission.VIEW_SCANNER)
export class InboxController {
constructor(
private readonly inboxService: InboxService,
private readonly postprocessor: InboxPostprocessorService,
private readonly barcodeScanner: BarcodeScannerService,
) {}
@Get()
async list(@Request() req: any) {
const preferredUsername: string | null = req.user?.preferredUsername ?? null;
return this.inboxService.listFiles(preferredUsername);
}
@Post('rescan')
async rescan() {
return this.barcodeScanner.rescanAll();
}
@Get(':id/preview')
async preview(
@Param('id') id: string,
@Request() req: any,
@Res({ passthrough: true }) res: Response,
): Promise<StreamableFile> {
const preferredUsername: string | null = req.user?.preferredUsername ?? null;
const { doc, pdfPath } = await this.inboxService.resolveDocument(id, preferredUsername);
res.setHeader('Content-Type', 'application/pdf');
res.setHeader('Content-Disposition', `inline; filename="${doc.OriginalName}"`);
return new StreamableFile(createReadStream(pdfPath));
}
@Get(':id/pages/:page/thumbnail')
async pageThumbnail(
@Param('id') id: string,
@Param('page', ParseIntPipe) page: number,
@Request() req: any,
@Res({ passthrough: true }) res: Response,
): Promise<StreamableFile> {
const preferredUsername: string | null = req.user?.preferredUsername ?? null;
const filePath = await this.inboxService.resolvePageImage(id, page, 'thumbnail', preferredUsername);
res.setHeader('Content-Type', 'image/png');
res.setHeader('Cache-Control', 'private, max-age=3600');
return new StreamableFile(createReadStream(filePath));
}
@Delete(':id')
@HttpCode(204)
async remove(@Param('id') id: string, @Request() req: any): Promise<void> {
const preferredUsername: string | null = req.user?.preferredUsername ?? null;
await this.inboxService.deleteDocument(id, preferredUsername);
}
@Delete(':id/pages/:page')
@HttpCode(204)
async removePage(
@Param('id') id: string,
@Param('page', ParseIntPipe) page: number,
@Request() req: any,
): Promise<void> {
const preferredUsername: string | null = req.user?.preferredUsername ?? null;
await this.inboxService.deletePage(id, page, preferredUsername);
}
@Post(':id/reset-edits')
@HttpCode(204)
async resetEdits(@Param('id') id: string, @Request() req: any): Promise<void> {
const preferredUsername: string | null = req.user?.preferredUsername ?? null;
await this.inboxService.resetEdits(id, preferredUsername);
}
@Post(':id/postprocess')
async postprocess(
@Param('id') id: string,
@Request() req: any,
@Body() body: { sectionOffset?: number; processOnlyOne?: boolean; replaceDuplicate?: boolean },
) {
const preferredUsername: string | null = req.user?.preferredUsername ?? null;
const { results, totalSections } = await this.postprocessor.runForDocument(
id,
preferredUsername,
body?.sectionOffset ?? 0,
body?.processOnlyOne ?? false,
body?.replaceDuplicate ?? false,
);
return { results, totalSections };
}
@Put(':id/pages/:page/rotation')
@HttpCode(204)
async setPageRotation(
@Param('id') id: string,
@Param('page', ParseIntPipe) page: number,
@Body() body: { rotation?: number },
@Request() req: any,
): Promise<void> {
const rotation = Number(body?.rotation);
if (!Number.isFinite(rotation)) {
throw new BadRequestException('rotation muss eine Zahl sein');
}
const preferredUsername: string | null = req.user?.preferredUsername ?? null;
await this.inboxService.setPageRotation(id, page, rotation, preferredUsername);
}
@Get(':id/pages/:page/preview')
async pagePreview(
@Param('id') id: string,
@Param('page', ParseIntPipe) page: number,
@Request() req: any,
@Res({ passthrough: true }) res: Response,
): Promise<StreamableFile> {
const preferredUsername: string | null = req.user?.preferredUsername ?? null;
const filePath = await this.inboxService.resolvePageImage(id, page, 'preview', preferredUsername);
res.setHeader('Content-Type', 'image/png');
res.setHeader('Cache-Control', 'private, max-age=3600');
return new StreamableFile(createReadStream(filePath));
}
}
@@ -0,0 +1,23 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { Client } from '../database/entities/client.entity';
import { UserClient } from '../database/entities/user-client.entity';
import { InboxDocument } from '../database/entities/inbox-document.entity';
import { InboxController } from './inbox.controller';
import { ClientsController } from './clients.controller';
import { InboxService } from './inbox.service';
import { InboxMigrationService } from './inbox-migration.service';
import { BarcodeModule } from '../barcode/barcode.module';
import { InboxPostprocessorModule } from '../inbox-postprocessor/inbox-postprocessor.module';
@Module({
imports: [
TypeOrmModule.forFeature([Client, UserClient, InboxDocument]),
BarcodeModule,
InboxPostprocessorModule,
],
controllers: [InboxController, ClientsController],
providers: [InboxService, InboxMigrationService],
exports: [InboxService],
})
export class InboxModule {}
@@ -0,0 +1,193 @@
import {
ConflictException,
Injectable,
Logger,
NotFoundException,
} from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import * as fs from 'fs/promises';
import { BarcodeScannerService, type MatchedBarcode } from '../barcode/barcode-scanner.service';
import { PageCacheService } from '../barcode/page-cache.service';
import {
InboxDocument,
type InboxSource,
} from '../database/entities/inbox-document.entity';
export interface InboxFile {
id: string;
name: string;
source: InboxSource;
pageCount: number;
deletedPages: number[];
rotations: Record<string, number>;
barcodes: MatchedBarcode[];
createdAt: string;
}
export interface ResolvedDocument {
doc: InboxDocument;
pdfPath: string;
}
@Injectable()
export class InboxService {
private readonly logger = new Logger(InboxService.name);
constructor(
private readonly barcodeScanner: BarcodeScannerService,
private readonly pageCache: PageCacheService,
@InjectRepository(InboxDocument)
private readonly documentRepo: Repository<InboxDocument>,
) {}
async listFiles(preferredUsername: string | null): Promise<InboxFile[]> {
const where = preferredUsername
? [{ Source: 'all' as InboxSource }, { Source: 'user' as InboxSource, OwnerUsername: preferredUsername }]
: [{ Source: 'all' as InboxSource }];
const docs = await this.documentRepo.find({
where,
order: { CreatedAt: 'DESC' },
});
const files: InboxFile[] = [];
for (const doc of docs) {
files.push({
id: doc.Id,
name: doc.OriginalName,
source: doc.Source,
pageCount: doc.PageCount,
deletedPages: [...(doc.DeletedPages ?? [])].sort((a, b) => a - b),
rotations: { ...(doc.Rotations ?? {}) },
barcodes: await this.barcodeScanner.getMatched(doc),
createdAt: doc.CreatedAt.toISOString(),
});
}
return files;
}
async resolveDocument(id: string, preferredUsername: string | null): Promise<ResolvedDocument> {
const doc = await this.documentRepo.findOne({ where: { Id: id } });
if (!doc) throw new NotFoundException('Dokument nicht gefunden');
if (doc.Source === 'user' && doc.OwnerUsername !== preferredUsername) {
throw new NotFoundException('Dokument nicht gefunden');
}
const pdfPath = this.pageCache.documentPdfPath(doc.Id);
try {
const stat = await fs.stat(pdfPath);
if (!stat.isFile()) throw new Error('not a file');
} catch (err: any) {
this.logger.warn(`Datei fehlt trotz DB-Eintrag (${doc.Id}): ${err.message}`);
throw new NotFoundException('Dokument nicht gefunden');
}
return { doc, pdfPath };
}
/**
* Markiert eine Seite virtuell zum Löschen. Die PDF und der Page-Cache
* bleiben unverändert; die eigentliche Anwendung passiert später bei
* der Weiterverarbeitung.
*/
async deletePage(
id: string,
page: number,
preferredUsername: string | null,
): Promise<void> {
const { doc } = await this.resolveDocument(id, preferredUsername);
if (!Number.isInteger(page) || page < 1 || page > doc.PageCount) {
throw new NotFoundException('Seite nicht gefunden');
}
const deleted = new Set<number>(doc.DeletedPages ?? []);
if (deleted.has(page)) return; // schon markiert
const remaining = doc.PageCount - deleted.size;
if (remaining <= 1) {
throw new ConflictException('Mindestens eine Seite muss übrig bleiben');
}
deleted.add(page);
doc.DeletedPages = Array.from(deleted).sort((a, b) => a - b);
await this.documentRepo.save(doc);
}
/**
* Setzt eine Seitenrotation virtuell. Wert wird auf 0/90/180/270
* normalisiert; 0 entfernt den Eintrag.
*/
async setPageRotation(
id: string,
page: number,
rotation: number,
preferredUsername: string | null,
): Promise<void> {
const { doc } = await this.resolveDocument(id, preferredUsername);
if (!Number.isInteger(page) || page < 1 || page > doc.PageCount) {
throw new NotFoundException('Seite nicht gefunden');
}
const normalized = ((Math.round(rotation / 90) * 90) % 360 + 360) % 360;
const next: Record<string, number> = { ...(doc.Rotations ?? {}) };
if (normalized === 0) {
delete next[String(page)];
} else {
next[String(page)] = normalized;
}
doc.Rotations = next;
await this.documentRepo.save(doc);
}
/**
* Setzt alle markierten Bearbeitungen (DeletedPages, Rotations) zurück.
*/
async resetEdits(id: string, preferredUsername: string | null): Promise<void> {
const { doc } = await this.resolveDocument(id, preferredUsername);
let changed = false;
if (doc.DeletedPages && doc.DeletedPages.length > 0) {
doc.DeletedPages = [];
changed = true;
}
if (doc.Rotations && Object.keys(doc.Rotations).length > 0) {
doc.Rotations = {};
changed = true;
}
if (changed) await this.documentRepo.save(doc);
}
async deleteDocument(id: string, preferredUsername: string | null): Promise<void> {
const { doc } = await this.resolveDocument(id, preferredUsername);
const dir = this.pageCache.documentDir(doc.Id);
await this.documentRepo.delete(doc.Id);
try {
await fs.rm(dir, { recursive: true, force: true });
} catch (err: any) {
this.logger.warn(`Dokument-Ordner konnte nicht gelöscht werden (${dir}): ${err.message}`);
}
}
async resolvePageImage(
id: string,
page: number,
variant: 'preview' | 'thumbnail',
preferredUsername: string | null,
): Promise<string> {
const { doc } = await this.resolveDocument(id, preferredUsername);
if (!Number.isInteger(page) || page < 1 || page > doc.PageCount) {
throw new NotFoundException('Seite nicht gefunden');
}
const filePath =
variant === 'preview'
? this.pageCache.previewPath(doc.Id, page)
: this.pageCache.thumbnailPath(doc.Id, page);
try {
await fs.access(filePath);
} catch {
throw new NotFoundException('Seite nicht gefunden');
}
return filePath;
}
}
@@ -0,0 +1,25 @@
import { Controller, Get, Post, Body, Param, UseGuards } from '@nestjs/common';
import { KontonummernService } from './kontonummern.service';
import { JwtOrApiKeyGuard } from '../auth/jwt-or-apikey.guard';
export class CreateKontonummerDto {
correspondentId: number;
nummer: string;
}
@Controller('api/kontonummern')
@UseGuards(JwtOrApiKeyGuard)
export class KontonummernController {
constructor(private readonly kontonummernService: KontonummernService) {}
@Get('FromCorrespondent/:id')
async getByCorrespondent(@Param('id') id: string) {
return this.kontonummernService.findByCorrespondent(parseInt(id, 10));
}
@Post()
async create(@Body() dto: CreateKontonummerDto) {
return this.kontonummernService.create(dto.correspondentId, dto.nummer);
}
}
@@ -0,0 +1,14 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { KontonummernController } from './kontonummern.controller';
import { KontonummernService } from './kontonummern.service';
import { Kontonummer } from '../database/entities';
import { AuthModule } from '../auth/auth.module';
@Module({
imports: [TypeOrmModule.forFeature([Kontonummer]), AuthModule],
controllers: [KontonummernController],
providers: [KontonummernService],
exports: [KontonummernService],
})
export class KontonummernModule {}
@@ -0,0 +1,35 @@
import { Injectable, NotFoundException } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { Kontonummer } from '../database/entities';
@Injectable()
export class KontonummernService {
constructor(
@InjectRepository(Kontonummer)
private readonly kontonummerRepo: Repository<Kontonummer>,
) {}
async findByCorrespondent(correspondentId: number): Promise<Kontonummer[]> {
return this.kontonummerRepo.find({
where: { CorrespondentId: correspondentId },
});
}
async create(correspondentId: number, nummer: string): Promise<Kontonummer> {
const existing = await this.kontonummerRepo.findOne({
where: { CorrespondentId: correspondentId, Nummer: nummer },
});
if (existing) {
return existing;
}
const kontonummer = this.kontonummerRepo.create({
CorrespondentId: correspondentId,
Nummer: nummer,
});
return this.kontonummerRepo.save(kontonummer);
}
}
+19
View File
@@ -0,0 +1,19 @@
import { NestFactory } from '@nestjs/core';
import { AppModule } from './app.module';
import { Logger } from '@nestjs/common';
async function bootstrap(): Promise<void> {
const app = await NestFactory.create(AppModule);
const port = process.env.PORT ?? 3100;
app.enableCors({
origin: '*', // Für lokale Entwicklung zulassen, oder spezifisch: 'http://localhost:8080'
methods: 'GET,HEAD,PUT,PATCH,POST,DELETE,OPTIONS',
credentials: true,
});
await app.listen(port);
Logger.log(`Paperless-Middleware läuft auf Port ${port}`, 'Bootstrap');
}
bootstrap();
@@ -0,0 +1,14 @@
export class UploadExternalDto {
interneBelegnummer!: string;
dokumentType?: number;
Eingangsdatum?: string;
tag?: number;
betriebId?: number;
lieferant?: string;
einkaufId?: number;
externeBelegnummer?: string;
belegdatum?: string;
parentId?: string;
barcodeJson?: string;
duplikatZu?: number;
}
@@ -0,0 +1,204 @@
import { Injectable, Logger } from '@nestjs/common';
import { Cron } from '@nestjs/schedule';
import { ConfigService } from '@nestjs/config';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { DocumentField } from '../database/entities/document-field.entity';
import { DocumentType } from '../database/entities/document-type.entity';
import { PaperlessService } from './paperless.service';
import { PostprocessingService } from '../postprocessing/postprocessing.service';
@Injectable()
export class PaperlessProcessorService {
private readonly logger = new Logger(PaperlessProcessorService.name);
constructor(
private readonly configService: ConfigService,
private readonly paperlessService: PaperlessService,
private readonly postprocessingService: PostprocessingService,
@InjectRepository(DocumentType) private readonly docTypeRepo: Repository<DocumentType>,
@InjectRepository(DocumentField) private readonly docFieldRepo: Repository<DocumentField>,
) {}
@Cron(process.env.PAPERLESS_PROCESSOR_CRON || '0 * * * * *')
async processDocuments() {
try {
const response = await this.paperlessService.getDocuments({ tags__id__all: 16, page_size: 9999 });
const documents: any[] = Array.isArray(response) ? response : (response?.results ?? []);
if (documents.length === 0) return;
const customFields = await this.paperlessService.getCustomFields();
const validFieldIds = new Set(customFields.map((f: any) => f.id));
this.logger.log(`Verarbeite ${documents.length} Dokument(e) mit Tag "paperlessmanager" (ID: 16).`);
for (const doc of documents) {
try {
const updatedDoc = await this.processSingleDocument(doc, validFieldIds);
// Postprocessing nach dem Speichern evaluieren
await this.postprocessingService.evaluate(updatedDoc || doc);
} catch (innerErr: any) {
this.logger.error(`Fehler bei Dokument ID ${doc.id}: ${innerErr.message}`);
if (innerErr.response?.data) {
this.logger.error(`Paperless API Response: ${JSON.stringify(innerErr.response.data)}`);
}
}
}
} catch (err) {
this.logger.error('Fehler bei der Dokumentenverarbeitung:', err);
}
}
private async processSingleDocument(doc: any, validFieldIds: Set<number>): Promise<any> {
this.logger.log(`Verarbeite Dokument ID: ${doc.id}`);
if (!doc.document_type) {
this.logger.warn(`Dokument ${doc.id} hat keinen Dokumenten-Typen setze Tag 17.`);
const tagsSet = new Set<number>(doc.tags || []);
tagsSet.add(17);
if (!tagsSet.has(1)) {
tagsSet.add(6);
}
const updated = await this.paperlessService.updateDocument(doc.id, { tags: Array.from(tagsSet) });
return updated;
}
const docTypeConfig = await this.docTypeRepo.findOne({
where: { DocumentTypeId: doc.document_type },
});
if (!docTypeConfig) {
this.logger.warn(`Konfiguration für DocumentType ${doc.document_type} nicht in der Datenbank gefunden.`);
return null;
}
const fieldsConfig = await this.docFieldRepo.find({
where: { DocumentType: doc.document_type },
});
if (fieldsConfig.length === 0) {
this.logger.log(`Dokument ${doc.id} (Typ ${doc.document_type}) hat keine Dokument-Felder in der DB konfiguriert.`);
const newTagsNoFields = Array.from(new Set([...(doc.tags || []), 17]));
const updated = await this.paperlessService.updateDocument(doc.id, { tags: newTagsNoFields });
return updated;
}
const currentCustomFields = doc.custom_fields || [];
const newCustomFields = [...currentCustomFields];
let isAllRequiredFilled = true;
for (const fieldConf of fieldsConfig) {
if (fieldConf.Type === 4) {
const customFieldId = fieldConf.TypeIndex;
if (!customFieldId) continue;
if (!validFieldIds.has(customFieldId)) {
this.logger.warn(`Überspringe ungültiges Custom Field (TypeIndex: ${customFieldId}) für Dokument ${doc.id} - in Paperless nicht vorhanden.`);
continue;
}
const existingField = newCustomFields.find(f => f.field === customFieldId);
let isFilled = false;
if (existingField) {
isFilled = existingField.value !== null && existingField.value !== '';
} else {
newCustomFields.push({ field: customFieldId, value: null });
}
if (fieldConf.IsRequired && !isFilled) {
isAllRequiredFilled = false;
}
} else {
if (fieldConf.IsRequired) {
let isFilled = false;
switch (fieldConf.Type) {
case 1:
isFilled = doc.correspondent !== null && doc.correspondent !== undefined;
break;
case 2:
isFilled = !!doc.created || !!doc.created_date;
break;
case 3:
isFilled = doc.archive_serial_number !== null && doc.archive_serial_number !== undefined;
break;
case 5:
isFilled = !!doc.title;
break;
default:
isFilled = true;
}
if (!isFilled) {
isAllRequiredFilled = false;
}
}
}
}
const tagsSet = new Set<number>(doc.tags || []);
if (isAllRequiredFilled) {
if (docTypeConfig.TagReady) tagsSet.add(docTypeConfig.TagReady);
if (docTypeConfig.TagNotReady) tagsSet.delete(docTypeConfig.TagNotReady);
} else {
if (docTypeConfig.TagNotReady) tagsSet.add(docTypeConfig.TagNotReady);
if (docTypeConfig.TagReady) tagsSet.delete(docTypeConfig.TagReady);
}
tagsSet.add(17);
// Titel-Template auflösen
const updatePayload: any = {
custom_fields: newCustomFields,
tags: Array.from(tagsSet),
};
if (docTypeConfig.TitelTemplate) {
let title = docTypeConfig.TitelTemplate;
// Prüfen ob alle im Template referenzierten Custom Fields ausgefüllt sind
const placeholderRegex = /\{\{CUSTOM\[(\d+)\]\}\}/g;
let allFilled = true;
let match: RegExpExecArray | null;
while ((match = placeholderRegex.exec(title)) !== null) {
const fieldId = parseInt(match[1], 10);
const cf = newCustomFields.find(f => f.field === fieldId);
if (!cf || cf.value == null || cf.value === '') {
allFilled = false;
break;
}
}
if (allFilled) {
for (const cf of newCustomFields) {
const placeholder = `{{CUSTOM[${cf.field}]}}`;
if (title.includes(placeholder)) {
title = title.replaceAll(placeholder, cf.value != null ? String(cf.value) : '');
}
}
if (title.includes('{{DATE}}')) {
const created = doc.created ? new Date(doc.created) : new Date();
const dateStr = `${String(created.getDate()).padStart(2, '0')}.${String(created.getMonth() + 1).padStart(2, '0')}.${created.getFullYear()}`;
title = title.replaceAll('{{DATE}}', dateStr);
}
if (title.includes('{{ZEITSTEMPEL}}')) {
const now = new Date();
const ts = `${String(now.getDate()).padStart(2, '0')}.${String(now.getMonth() + 1).padStart(2, '0')}.${now.getFullYear()} ${String(now.getHours()).padStart(2, '0')}:${String(now.getMinutes()).padStart(2, '0')}:${String(now.getSeconds()).padStart(2, '0')}`;
title = title.replaceAll('{{ZEITSTEMPEL}}', ts);
}
updatePayload.title = title;
} else {
this.logger.log(`Dokument ${doc.id}: Titel-Template nicht angewendet nicht alle referenzierten Custom Fields ausgefüllt.`);
}
}
const updated = await this.paperlessService.updateDocument(doc.id, updatePayload);
this.logger.log(`Dokument ${doc.id} erfolgreich aktualisiert (Alle Pflichtfelder vorhanden: ${isAllRequiredFilled}).`);
return updated;
}
}
@@ -0,0 +1,351 @@
import { Injectable, Logger } from '@nestjs/common';
import { Cron, CronExpression } from '@nestjs/schedule';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository, IsNull } from 'typeorm';
import { Task } from '../database/entities/task.entity';
import { Document } from '../database/entities/document.entity';
import { Attachment } from '../database/entities/attachment.entity';
import { PaperlessService } from './paperless.service';
@Injectable()
export class PaperlessTaskProcessorService {
private readonly logger = new Logger(PaperlessTaskProcessorService.name);
constructor(
@InjectRepository(Task)
private readonly taskRepo: Repository<Task>,
@InjectRepository(Document)
private readonly documentRepo: Repository<Document>,
@InjectRepository(Attachment)
private readonly attachmentRepo: Repository<Attachment>,
private readonly paperlessService: PaperlessService,
) {}
@Cron(CronExpression.EVERY_30_SECONDS)
async handleCron() {
try {
// Fetch tasks that are not finished
const tasks = await this.taskRepo.find({
where: [
{ Fertig: IsNull() },
{ Fertig: 0 },
],
take: 10,
});
if (tasks.length === 0) {
return;
}
this.logger.log(`${tasks.length} Tasks noch zu bearbeiten`);
const toDelete: Task[] = [];
for (const t of tasks) {
let parentTask: Task | null = null;
if (t.TaskReferenceID) {
parentTask = await this.taskRepo.findOne({
where: { TaskId: t.TaskReferenceID },
});
if (!parentTask) {
this.logger.error(`ParentTask not found - Task ${t.TaskId}`);
continue;
}
if (!parentTask.Fertig || parentTask.Fertig === 0) {
this.logger.log(`ParentTask not imported - Task ${t.TaskId}`);
continue;
}
}
// Fetch task status from Paperless
const paperlessTasks = await this.paperlessService.getTask(t.TaskId);
const apiResponseTask = Array.isArray(paperlessTasks) ? paperlessTasks[0] : null;
if (apiResponseTask) {
if (apiResponseTask.status === 'SUCCESS') {
const dateDone = apiResponseTask.date_done ? new Date(apiResponseTask.date_done) : new Date();
const now = new Date();
// Add 10 seconds buffer as in C#
if (dateDone.getTime() + 10000 < now.getTime()) {
await this.processSuccessfulTask(t, apiResponseTask, parentTask);
}
} else if (apiResponseTask.status === 'FAILURE') {
this.logger.warn(`Task ${t.TaskId} failed in Paperless`);
toDelete.push(t);
}
} else {
this.logger.warn(`Task ${t.TaskId} not found in Paperless`);
toDelete.push(t);
}
}
if (toDelete.length > 0) {
await this.taskRepo.remove(toDelete);
this.logger.log(`${toDelete.length} Tasks gelöscht`);
}
} catch (error) {
this.logger.error(`Fehler bei der Task-Verarbeitung: ${error.message}`, error.stack);
}
}
private async processSuccessfulTask(t: Task, apiTask: any, parentTask: Task | null) {
const documentId = apiTask.related_document;
this.logger.log(`[Postprocessing] Task ${t.TaskId} gestartet. DocumentID: ${documentId ?? 'nicht vorhanden'}`);
if (!documentId) {
this.logger.error(`Kein Dokument für Task ${t.TaskId} gefunden.`);
return;
}
try {
this.logger.log(`[Postprocessing] Task ${t.TaskId} - Lade Dokument ${documentId} aus Paperless`);
const document = await this.paperlessService.getDocument(documentId);
if (!document) {
this.logger.warn(`Dokument mit ID ${documentId} nicht in Paperless gefunden.`);
return;
}
this.logger.log(`[Postprocessing] Task ${t.TaskId} - Dokument geladen: ID=${document.id}, Titel="${document.title}"`);
// Handle Duplicate Link
if (t.DuplikatZU) {
this.logger.log(`[Postprocessing] Task ${t.TaskId} - Verknüpfe Duplikat zu Dokument ${t.DuplikatZU}`);
const duplikatDoc = await this.paperlessService.getDocument(t.DuplikatZU);
if (duplikatDoc) {
// Update duplikatDoc metadata (Field 8 is for linked documents)
let duplikatCustomFields = Array.isArray(duplikatDoc.custom_fields) ? [...duplikatDoc.custom_fields] : [];
// Remove field 4 as in C#
duplikatCustomFields = duplikatCustomFields.filter((f: any) => f.field !== 4);
const field8 = duplikatCustomFields.find((f: any) => f.field === 8);
if (field8) {
const values = Array.isArray(field8.value) ? field8.value : [];
if (!values.includes(document.id)) {
values.push(document.id);
field8.value = values;
}
} else {
duplikatCustomFields.push({ field: 8, value: [document.id] });
}
await this.paperlessService.updateDocument(duplikatDoc.id, {
archive_serial_number: null,
document_type: 11,
custom_fields: duplikatCustomFields,
});
this.logger.log(`[Postprocessing] Task ${t.TaskId} - Duplikat-Dokument ${duplikatDoc.id} aktualisiert`);
// Update current document as well
const currentCustomFields = Array.isArray(document.custom_fields) ? [...document.custom_fields] : [];
const currentField8 = currentCustomFields.find((f: any) => f.field === 8);
if (currentField8) {
const values = Array.isArray(currentField8.value) ? currentField8.value : [];
if (!values.includes(duplikatDoc.id)) {
values.push(duplikatDoc.id);
currentField8.value = values;
}
} else {
currentCustomFields.push({ field: 8, value: [duplikatDoc.id] });
}
document.custom_fields = currentCustomFields;
} else {
this.logger.warn(`[Postprocessing] Task ${t.TaskId} - Duplikat-Dokument ${t.DuplikatZU} nicht gefunden`);
}
}
// Enrich Document
this.logger.log(`[Postprocessing] Task ${t.TaskId} - Reichere Dokument-Metadaten an`);
const updateData: any = {
custom_fields: Array.isArray(document.custom_fields) ? [...document.custom_fields] : [],
};
// CustomFieldsJson als Basis zuerst anwenden dedizierte Felder weiter unten überschreiben diese
if (t.CustomFieldsJson) {
try {
const extra = JSON.parse(t.CustomFieldsJson) as Record<string, string>;
for (const [k, v] of Object.entries(extra)) {
const fieldId = parseInt(k, 10);
if (!Number.isFinite(fieldId)) continue;
const idx = updateData.custom_fields.findIndex((f: any) => f.field === fieldId);
if (idx !== -1) updateData.custom_fields[idx].value = v;
else updateData.custom_fields.push({ field: fieldId, value: v });
}
} catch { /* JSON-Parse-Fehler ignorieren */ }
}
if (t.Asn) {
const asnNum = parseInt(t.Asn.replace(/[^0-9]/g, ''), 10);
if (!isNaN(asnNum)) {
this.logger.log(`[Postprocessing] Task ${t.TaskId} - Setze ASN (explizit): ${asnNum}`);
updateData.archive_serial_number = asnNum;
}
}
if (t.InterneBelegnummer) {
this.logger.log(`[Postprocessing] Task ${t.TaskId} - Setze InterneBelegnummer: ${t.InterneBelegnummer}`);
if (!t.Asn) {
const asnFromBelegnummer = parseInt(t.InterneBelegnummer.replace(/-/g, ''), 10);
if (!isNaN(asnFromBelegnummer)) {
updateData.archive_serial_number = asnFromBelegnummer;
} else {
this.logger.warn(`[Postprocessing] Task ${t.TaskId} - ASN aus InterneBelegnummer konnte nicht geparst werden: ${t.InterneBelegnummer}`);
}
}
const existingField7 = updateData.custom_fields.find((f: any) => f.field === 7);
if (existingField7) {
existingField7.value = t.InterneBelegnummer;
} else {
updateData.custom_fields.push({ field: 7, value: t.InterneBelegnummer });
}
}
if (t.externeBelegnummer) {
this.logger.log(`[Postprocessing] Task ${t.TaskId} - Setze externeBelegnummer: ${t.externeBelegnummer}`);
const existingField3 = updateData.custom_fields.find((f: any) => f.field === 3);
if (existingField3) {
existingField3.value = t.externeBelegnummer;
} else {
updateData.custom_fields.push({ field: 3, value: t.externeBelegnummer });
}
}
if (t.Eingangsdatum) {
const dateValue = new Date(t.Eingangsdatum).toISOString().split('T')[0];
this.logger.log(`[Postprocessing] Task ${t.TaskId} - Setze Eingangsdatum: ${dateValue}`);
const existingField9 = updateData.custom_fields.find((f: any) => f.field === 9);
if (existingField9) {
existingField9.value = dateValue;
} else {
updateData.custom_fields.push({ field: 9, value: dateValue });
}
}
if (t.DocumentType) {
this.logger.log(`[Postprocessing] Task ${t.TaskId} - Setze DocumentType: ${t.DocumentType}`);
updateData.document_type = t.DocumentType;
}
// Parent Task / Attachment logic
if (parentTask) {
this.logger.log(`[Postprocessing] Task ${t.TaskId} - Verarbeite als Anlage zu ParentTask ${parentTask.TaskId}`);
const parentPaperlessTasks = await this.paperlessService.getTask(parentTask.TaskId);
const apiParentTask = Array.isArray(parentPaperlessTasks) ? parentPaperlessTasks[0] : null;
if (apiParentTask && apiParentTask.related_document) {
const parentDoc = await this.paperlessService.getDocument(apiParentTask.related_document);
if (parentDoc) {
this.logger.log(`[Postprocessing] Task ${t.TaskId} - Elterndokument ${parentDoc.id} gefunden, setze Anlage-Typ`);
updateData.document_type = 5; // Anlage
updateData.title = `Anlage zu ${parentTask.InterneBelegnummer}`;
const field8 = updateData.custom_fields.find((f: any) => f.field === 8);
if (field8) {
const values = Array.isArray(field8.value) ? field8.value : [];
if (!values.includes(parentDoc.id)) {
values.push(parentDoc.id);
field8.value = values;
}
} else {
updateData.custom_fields.push({ field: 8, value: [parentDoc.id] });
}
} else {
this.logger.warn(`[Postprocessing] Task ${t.TaskId} - Elterndokument nicht gefunden`);
}
} else {
this.logger.warn(`[Postprocessing] Task ${t.TaskId} - ParentTask hat kein related_document`);
}
}
if (t.Belegdatum) {
this.logger.log(`[Postprocessing] Task ${t.TaskId} - Setze Belegdatum: ${t.Belegdatum.toISOString()}`);
updateData.created = t.Belegdatum.toISOString();
}
if (t.BetriebID) {
this.logger.log(`[Postprocessing] Task ${t.TaskId} - Setze Owner: ${t.BetriebID}`);
updateData.owner = t.BetriebID;
} else {
this.logger.log(`[Postprocessing] Task ${t.TaskId} - Entferne Owner (setze null)`);
updateData.owner = null;
}
// Tags
if (t.Tags) {
this.logger.log(`[Postprocessing] Task ${t.TaskId} - Setze Tags: ${t.Tags}`);
const tagIds = t.Tags.split(',').map(id => parseInt(id.trim(), 10)).filter(id => !isNaN(id));
const currentTags = document.tags || [];
const newTags = Array.from(new Set([...currentTags, ...tagIds]));
updateData.tags = newTags;
}
// Agrarmonitor Link (Skip API call for now, but save the link if needed)
if (t.EinkaufID) {
const link = `https://admin7.agrarmonitor.de/rechnungen/detail/${t.EinkaufID}`;
this.logger.log(`Skipping Agrarmonitor details for EinkaufID ${t.EinkaufID}. Link: ${link}`);
}
if (t.Lieferant) {
this.logger.log(`Skipping Correspondent lookup/creation for Lieferant ID ${t.Lieferant} (Agrarmonitor part deferred).`);
}
// Update Document in Paperless
this.logger.log(`[Postprocessing] Task ${t.TaskId} - Aktualisiere Dokument ${document.id} in Paperless. Payload: ${JSON.stringify(updateData)}`);
await this.paperlessService.updateDocument(document.id, updateData);
this.logger.log(`[Postprocessing] Task ${t.TaskId} - Dokument ${document.id} erfolgreich aktualisiert`);
// Add Notes
this.logger.log(`[Postprocessing] Task ${t.TaskId} - Füge Notizen hinzu`);
await this.paperlessService.addNote(document.id, `Task bearbeitet: ${new Date().toLocaleString('de-DE')}`);
if (t.BarcodeJson) {
await this.paperlessService.addNote(document.id, t.BarcodeJson);
this.logger.log(`[Postprocessing] Task ${t.TaskId} - BarcodeJson-Notiz hinzugefügt`);
}
// Sync local Documents table
this.logger.log(`[Postprocessing] Task ${t.TaskId} - Synchronisiere lokale Documents-Tabelle`);
const metadata = await this.paperlessService.getDocumentMetadata(document.id);
let localDoc = await this.documentRepo.findOne({ where: { documentId: document.id } });
if (!localDoc) {
this.logger.log(`[Postprocessing] Task ${t.TaskId} - Erstelle neuen lokalen Dokument-Eintrag`);
localDoc = this.documentRepo.create({
documentId: document.id,
checksum: metadata.original_checksum,
filename: metadata.original_filename,
});
await this.documentRepo.save(localDoc);
} else {
this.logger.log(`[Postprocessing] Task ${t.TaskId} - Aktualisiere bestehenden lokalen Dokument-Eintrag`);
localDoc.checksum = metadata.original_checksum;
localDoc.filename = metadata.original_filename;
await this.documentRepo.save(localDoc);
}
// Update Task status
this.logger.log(`[Postprocessing] Task ${t.TaskId} - Setze Task-Status auf Fertig`);
t.Fertig = 1;
t.PaperlessDocumentID = document.id;
await this.taskRepo.save(t);
// Update source attachment if linked
if (t.SourceAttachmentID && t.SourceAttachmentRange) {
const attachment = await this.attachmentRepo.findOne({ where: { Id: t.SourceAttachmentID } });
if (attachment) {
const ids = attachment.PaperlessDocumentIds || {};
ids[t.SourceAttachmentRange] = document.id;
attachment.PaperlessDocumentIds = ids;
await this.attachmentRepo.save(attachment);
this.logger.log(`[Postprocessing] Anhang ${attachment.Id} mit PaperlessID ${document.id} (${t.SourceAttachmentRange}) aktualisiert`);
}
}
this.logger.log(`[Postprocessing] Task ${t.TaskId} erfolgreich abgeschlossen. Dokument ID: ${document.id}`);
} catch (error) {
this.logger.error(`Fehler bei der Verarbeitung von Task ${t.TaskId}: ${error.message}`, error.stack);
}
}
}
@@ -0,0 +1,432 @@
import { Controller, Get, Param, Post, Put, Delete, UseGuards, UseInterceptors, UploadedFile, Body, Logger, HttpException, HttpStatus, Res, Query } from '@nestjs/common';
import type { Response } from 'express';
import { Public } from '../auth/public.decorator';
import { RequirePermissions } from '../auth/permissions.decorator';
import { Permission } from '../auth/permissions.enum';
import { FileInterceptor } from '@nestjs/platform-express';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { PaperlessService } from './paperless.service';
import { ApiKeyGuard } from '../auth/api-key.guard';
import { UploadExternalDto } from './dto/upload-external.dto';
import { Task } from '../database/entities/task.entity';
import { Document } from '../database/entities/document.entity';
import { DocumentField } from '../database/entities/document-field.entity';
import { DocumentType } from '../database/entities/document-type.entity';
@Controller('api/paperless')
export class PaperlessController {
private readonly logger = new Logger(PaperlessController.name);
constructor(
private readonly paperlessService: PaperlessService,
@InjectRepository(Task)
private readonly taskRepo: Repository<Task>,
@InjectRepository(Document)
private readonly documentRepo: Repository<Document>,
@InjectRepository(DocumentField)
private readonly documentFieldRepo: Repository<DocumentField>,
@InjectRepository(DocumentType)
private readonly documentTypeRepo: Repository<DocumentType>,
) {}
@Post('checksum')
async checksumExists(@Body('checksum') checksum: string) {
const exists = await this.paperlessService.checksumExists(checksum);
return { exists };
}
@Get('documents')
async getDocuments(
@Query('search') search?: string,
@Query('page') page?: string,
@Query('pageSize') pageSize?: string,
) {
const params: any = {
page: page || '1',
page_size: pageSize || '20',
};
if (search) {
params.query = search; // Global search in Paperless
}
return this.paperlessService.getDocuments(params);
}
@Get('documents/:id')
async getDocument(@Param('id') id: string) {
return this.paperlessService.getDocument(parseInt(id, 10));
}
@Get('tags')
async getTags() {
return this.paperlessService.getTags();
}
@Get('tags/:id')
async getTag(@Param('id') id: string) {
// If the service doesn't have getTag(id), I should add it or just fetch all and find
const tags = await this.paperlessService.getTags();
return tags.find(t => t.id === parseInt(id, 10));
}
@Get('document-types')
async getDocumentTypes() {
return this.paperlessService.getDocumentTypes();
}
@Get('custom-fields')
async getCustomFields() {
return this.paperlessService.getCustomFields();
}
@Get('users')
async getUsers() {
return this.paperlessService.getUsers();
}
@Get('correspondents')
async getCorrespondents(@Query('search') search?: string) {
const params: any = { page_size: 100 };
if (search) {
params.name__icontains = search;
}
const response = await this.paperlessService.getCorrespondents(params);
return response.results;
}
@Get('correspondents/:id')
async getCorrespondent(@Param('id') id: string) {
return this.paperlessService.getCorrespondent(parseInt(id, 10));
}
@Get('inbox/list')
@RequirePermissions(Permission.VIEW_INBOX)
async getInboxList() {
const documents = await this.paperlessService.getInboxDocuments();
// In old C# logic: only return docs where archive_serial_number is not null
return documents
.filter((doc: any) => doc.archive_serial_number !== null && doc.archive_serial_number !== undefined)
.map((doc: any) => ({
id: doc.id,
title: doc.title,
asn: doc.archive_serial_number,
documentType: doc.document_type,
correspondent: doc.correspondent,
created: doc.created_date,
added: doc.added,
tags: doc.tags,
customFields: doc.custom_fields,
owner: doc.owner,
}));
}
@Get('manuell/list')
@RequirePermissions(Permission.PROCESS_MANUALLY)
async getManuellList() {
const documents = await this.paperlessService.getManuellDocuments();
return documents
.filter((doc: any) => doc.archive_serial_number !== null && doc.archive_serial_number !== undefined)
.map((doc: any) => ({
id: doc.id,
title: doc.title,
asn: doc.archive_serial_number,
documentType: doc.document_type,
correspondent: doc.correspondent,
created: doc.created_date,
added: doc.added,
tags: doc.tags,
customFields: doc.custom_fields,
owner: doc.owner,
}));
}
@Public()
@Get('inbox/preview/:id')
async getInboxPreview(@Param('id') id: string, @Res() res: Response) {
try {
const stream = await this.paperlessService.getDocumentPreviewStream(parseInt(id, 10));
res.set({
'Content-Type': 'image/png',
'Content-Disposition': `inline; filename="${id}preview.png"`,
});
stream.pipe(res);
} catch (error) {
if (!res.headersSent) {
res.status(HttpStatus.INTERNAL_SERVER_ERROR).send('Error fetching preview');
}
}
}
@Public()
@Get('inbox/pdf/:id')
async getInboxPdf(@Param('id') id: string, @Res() res: Response) {
try {
const stream = await this.paperlessService.getDocumentPdfStream(parseInt(id, 10));
res.set({
'Content-Type': 'application/pdf',
'Content-Disposition': `inline; filename="${id}.pdf"`,
});
stream.pipe(res);
} catch (error) {
if (!res.headersSent) {
res.status(HttpStatus.INTERNAL_SERVER_ERROR).send('Error fetching PDF');
}
}
}
@Get('requirements/:id')
async getRequirements(@Param('id') id: string, @Query('Posteingang') posteingang: string) {
const documentTypeId = parseInt(id, 10);
const isPosteingang = posteingang === '1';
const requirements = await this.documentFieldRepo.find({
where: { DocumentType: documentTypeId },
});
// Custom fields fetching inside here could be slow, but this is the simplest translation of the old API
// Actually, getting all CFs doesn't take too long in Paperless API.
const customFields = await this.paperlessService.getCustomFields();
const retVal: any[] = [];
for (const req of requirements) {
if (isPosteingang && !req.VisiblePosteingang) {
continue;
}
const tmp: any = {
id: req.Id,
feldId: req.Type + (req.TypeIndex !== null ? '-' + req.TypeIndex : ''),
hinweis: req.Hinweis || '',
required: isPosteingang ? req.IsRequiredPosteingang : req.IsRequired,
};
if (req.Type === 4) {
// Custom Field
const cfId = Number(req.TypeIndex);
const cf = customFields.find((c: any) => c.id === cfId);
tmp.isCustomField = true;
tmp.customFieldIndex = cfId;
if (cf) {
tmp.feldName = cf.name;
tmp.feldTyp = cf.id === 12 ? 'int' : cf.data_type;
if (cf.extra_data && cf.extra_data.select_options) {
tmp.fieldOptions = cf.extra_data.select_options
.filter((o: any) => o !== null)
.map((o: any) => ({ id: o.id, label: o.label }));
}
} else {
tmp.feldName = `Feld ${req.TypeIndex}`;
tmp.feldTyp = 'string';
}
} else if (req.Type === 1) {
tmp.feldName = 'Absender';
tmp.feldTyp = 'select';
const response = await this.paperlessService.getCorrespondents({ page_size: 9999 });
const correspondents = response.results;
tmp.fieldOptions = correspondents.map((c: any) => ({ id: c.id.toString(), label: c.name }));
} else if (req.Type === 2) {
tmp.feldName = 'Belegdatum';
tmp.feldTyp = 'date';
} else if (req.Type === 3) {
tmp.feldName = 'Ablagenummer';
tmp.feldTyp = 'string';
} else if (req.Type === 5) {
tmp.feldName = 'Titel';
tmp.feldTyp = 'string';
} else {
tmp.feldName = tmp.feldId;
tmp.feldTyp = 'string';
}
retVal.push(tmp);
}
return retVal;
}
@Put('inbox/:id')
async putInboxDocument(@Param('id') id: string, @Body() body: any) {
const documentId = parseInt(id, 10);
// Fetch from paperless
const oldDocument = await this.paperlessService.getDocument(documentId);
oldDocument.document_type = body.documentType ?? oldDocument.document_type;
oldDocument.owner = body.mandant ?? oldDocument.owner;
oldDocument.correspondent = body.correspondent ?? oldDocument.correspondent;
oldDocument.title = body.title ?? oldDocument.title;
if (body.date) {
let docDate = new Date(body.date);
if (docDate.getHours() > 22) {
docDate = new Date(docDate.getTime() + 24 * 60 * 60 * 1000 - docDate.getHours() * 60 * 60 * 1000);
}
oldDocument.created_date = docDate.toISOString().split('T')[0];
}
const cfDefinitions = await this.paperlessService.getCustomFields();
// update custom fields
if (body.customFields) {
for (const [key, value] of Object.entries(body.customFields)) {
const fieldId = parseInt(key, 10);
const cfDef = cfDefinitions.find((c: any) => c.id === fieldId);
let processedValue = value;
if (cfDef?.data_type === 'documentlink' && value !== null && value !== '' && !Array.isArray(value)) {
processedValue = [value];
}
const existingFieldIndex = oldDocument.custom_fields.findIndex((f: any) => f.field === fieldId);
if (existingFieldIndex !== -1) {
if (processedValue === null || processedValue === '') {
oldDocument.custom_fields.splice(existingFieldIndex, 1);
} else {
oldDocument.custom_fields[existingFieldIndex].value = processedValue;
}
} else if (processedValue !== null && processedValue !== '') {
oldDocument.custom_fields.push({ field: fieldId, value: processedValue });
}
}
}
// Requirements check
const reqs = await this.documentFieldRepo.find({
where: { DocumentType: oldDocument.document_type },
});
let isReady = true;
let isReadyPosteingang = true;
for (const req of reqs) {
let isFieldValid = false;
if (req.Type === 1) isFieldValid = oldDocument.correspondent !== null;
if (req.Type === 2) isFieldValid = oldDocument.created_date !== null;
if (req.Type === 3) isFieldValid = oldDocument.archive_serial_number !== null;
if (req.Type === 4) isFieldValid = !!oldDocument.custom_fields.find((cf: any) => cf.field === req.TypeIndex && cf.value !== null && cf.value !== '');
if (req.Type === 5) isFieldValid = oldDocument.title !== null && oldDocument.title !== '';
if (req.IsRequired && !isFieldValid) isReady = false;
if (req.IsRequiredPosteingang && !isFieldValid) isReadyPosteingang = false;
}
const docType = await this.documentTypeRepo.findOne({
where: { DocumentTypeId: oldDocument.document_type },
});
oldDocument.tags = oldDocument.tags || [];
if (isReady) {
oldDocument.tags = oldDocument.tags.filter((t: number) => t !== 1);
if (docType?.TagNotReady) oldDocument.tags = oldDocument.tags.filter((t: number) => t !== docType.TagNotReady);
if (docType?.TagReady && !oldDocument.tags.includes(docType.TagReady)) {
oldDocument.tags.push(docType.TagReady);
}
let titleTemplate = docType?.TitelTemplate || '';
if (titleTemplate) {
for (const cf of oldDocument.custom_fields) {
const placeholder = `{{CUSTOM[${cf.field}]}}`;
if (titleTemplate.includes(placeholder)) {
titleTemplate = titleTemplate.replace(placeholder, cf.value?.toString() ?? '');
}
}
titleTemplate = titleTemplate.replace('{{DATE}}', oldDocument.created_date);
oldDocument.title = titleTemplate;
}
} else {
if (docType?.TagNotReady) {
if (isReadyPosteingang) {
oldDocument.tags = oldDocument.tags.filter((t: number) => t !== 1);
if (!oldDocument.tags.includes(docType.TagNotReady)) oldDocument.tags.push(docType.TagNotReady);
} else {
if (!oldDocument.tags.includes(1)) oldDocument.tags.push(1);
}
} else {
if (!oldDocument.tags.includes(1)) oldDocument.tags.push(1);
}
if (docType?.TagReady) {
oldDocument.tags = oldDocument.tags.filter((t: number) => t !== docType.TagReady);
}
}
await this.paperlessService.updateDocument(documentId, oldDocument);
return { success: true };
}
@Get('tasks')
@RequirePermissions(Permission.MANAGE_SETTINGS)
async getAllTasks() {
return this.taskRepo.find({ order: { Fertig: 'ASC' } });
}
@Delete('tasks/fertig')
@RequirePermissions(Permission.MANAGE_SETTINGS)
async deleteFertigeTasks() {
const result = await this.taskRepo.delete({ Fertig: 1 });
return { deleted: result.affected ?? 0 };
}
@Delete('tasks/:taskId')
@RequirePermissions(Permission.MANAGE_SETTINGS)
async deleteTask(@Param('taskId') taskId: string) {
const result = await this.taskRepo.delete({ TaskId: taskId });
if (!result.affected) {
throw new HttpException('Task nicht gefunden', HttpStatus.NOT_FOUND);
}
return { deleted: result.affected };
}
@Post('external-upload')
@UseGuards(ApiKeyGuard)
@UseInterceptors(FileInterceptor('document'))
async externalUpload(
@UploadedFile() file: Express.Multer.File,
@Body() dto: UploadExternalDto,
) {
this.logger.log(`Externer Upload gestartet: ${dto.interneBelegnummer} (${file?.originalname})`);
try {
// 0. Check if ASN already exists
await this.paperlessService.validateAsnNotExists(dto.interneBelegnummer);
// 1. Forward to Paperless
const paperlessTaskId = await this.paperlessService.uploadDocument(file.path, {
title: `Beleg ${dto.interneBelegnummer}`,
});
// 2. Create local Task
const task = this.taskRepo.create({
TaskId: paperlessTaskId.replace(/"/g, ''),
InterneBelegnummer: dto.interneBelegnummer,
DocumentType: dto.dokumentType,
Eingangsdatum: dto.Eingangsdatum ? new Date(dto.Eingangsdatum) : null,
Tags: dto.tag ? String(dto.tag) : null,
BetriebID: dto.betriebId,
Lieferant: dto.lieferant,
EinkaufID: dto.einkaufId,
externeBelegnummer: dto.externeBelegnummer,
Belegdatum: dto.belegdatum ? new Date(dto.belegdatum) : null,
TaskReferenceID: dto.parentId || '',
BarcodeJson: dto.barcodeJson || '',
DuplikatZU: dto.duplikatZu,
Fertig: 0,
});
await this.taskRepo.save(task);
this.logger.log(`Externer Upload erfolgreich: ${task.TaskId} für Beleg ${dto.interneBelegnummer}`);
return task.TaskId;
} catch (err) {
this.logger.error(`Fehler beim externen Upload für Beleg ${dto.interneBelegnummer}: ${err.message}`, err.stack);
throw new HttpException(
`Fehler beim Verarbeiten des Dokumenten-Uploads: ${err.message}`,
HttpStatus.INTERNAL_SERVER_ERROR,
);
}
}
}
@@ -0,0 +1,25 @@
import { Module, forwardRef } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { PaperlessService } from './paperless.service';
import { PaperlessController } from './paperless.controller';
import { PaperlessProcessorService } from './paperless-processor.service';
import { PaperlessTaskProcessorService } from './paperless-task-processor.service';
import { DocumentType } from '../database/entities/document-type.entity';
import { DocumentField } from '../database/entities/document-field.entity';
import { Task } from '../database/entities/task.entity';
import { Document } from '../database/entities/document.entity';
import { Attachment } from '../database/entities/attachment.entity';
import { PostprocessingModule } from '../postprocessing/postprocessing.module';
import { AuthModule } from '../auth/auth.module';
@Module({
imports: [
TypeOrmModule.forFeature([DocumentType, DocumentField, Task, Document, Attachment]),
forwardRef(() => PostprocessingModule),
AuthModule,
],
controllers: [PaperlessController],
providers: [PaperlessService, PaperlessProcessorService, PaperlessTaskProcessorService],
exports: [PaperlessService],
})
export class PaperlessModule {}
@@ -0,0 +1,287 @@
import { Injectable, Logger, HttpException, HttpStatus } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import axios, { type AxiosInstance } from 'axios';
import * as fs from 'fs';
import * as path from 'path';
import FormData = require('form-data');
@Injectable()
export class PaperlessService {
private readonly logger = new Logger(PaperlessService.name);
private readonly client: AxiosInstance;
constructor(private readonly configService: ConfigService) {
const baseURL = this.configService.get<string>('PAPERLESS_URL', 'http://localhost:8000');
const token = this.configService.get<string>('PAPERLESS_TOKEN', '');
this.client = axios.create({
baseURL: `${baseURL}/api`,
headers: {
Authorization: `Token ${token}`,
},
timeout: 30000,
});
}
async uploadDocument(
filePath: string,
options?: {
title?: string;
filename?: string;
created?: string;
documentType?: number;
correspondent?: number;
storagePath?: number;
tags?: number[];
owner?: number;
archiveSerialNumber?: number;
customFields?: Record<string | number, string>;
},
): Promise<string> {
const form = new FormData();
const uploadFilename = options?.filename
? `${options.filename}.pdf`
: path.basename(filePath);
form.append('document', fs.createReadStream(filePath), {
filename: uploadFilename,
contentType: 'application/pdf',
});
if (options?.title) form.append('title', options.title);
if (options?.created) form.append('created', options.created);
if (options?.documentType) form.append('document_type', String(options.documentType));
if (options?.correspondent) form.append('correspondent', String(options.correspondent));
if (options?.storagePath) form.append('storage_path', String(options.storagePath));
if (options?.owner !== undefined && options.owner !== null) {
form.append('owner', String(options.owner));
}
if (options?.tags) {
options.tags.forEach((tag) => form.append('tags', String(tag)));
}
if (options?.archiveSerialNumber !== undefined && !Number.isNaN(options.archiveSerialNumber)) {
form.append('archive_serial_number', String(options.archiveSerialNumber));
}
if (options?.customFields && Object.keys(options.customFields).length > 0) {
form.append('custom_fields', JSON.stringify(options.customFields));
}
const response = await this.client.post('/documents/post_document/', form, {
headers: form.getHeaders(),
timeout: 120000,
});
this.logger.log(`Dokument hochgeladen: ${response.data}`);
return response.data;
}
async getUsers(): Promise<any[]> {
const response = await this.client.get('/users/');
return response.data?.results ?? [];
}
async getDocuments(params?: Record<string, any>): Promise<any> {
const response = await this.client.get('/documents/', { params });
return response.data;
}
async getDocument(id: number): Promise<any> {
const response = await this.client.get(`/documents/${id}/`);
return response.data;
}
async getInboxDocuments(): Promise<any[]> {
// API pagination to get large amount of inbox documents (assuming max 9999 like C# app)
const response = await this.client.get('/documents/', {
params: {
page: 1,
page_size: 9999,
ordering: '-added',
truncate_content: true,
tags__id__all: 1
}
});
return response.data.results;
}
async getManuellDocuments(): Promise<any[]> {
const errorTag = this.configService.get<number>('MANUELL_BEARBEITEN_TAG', 6);
const response = await this.client.get('/documents/', {
params: {
page: 1,
page_size: 9999,
ordering: '-added',
truncate_content: true,
tags__id__all: errorTag
}
});
return response.data.results;
}
async updateDocument(id: number, data: Record<string, any>): Promise<any> {
try {
const response = await this.client.patch(`/documents/${id}/`, data);
return response.data;
} catch (err: any) {
const body = err?.response?.data;
if (body) {
this.logger.error(`Paperless updateDocument(${id}) Fehlerdetails: ${JSON.stringify(body)}`);
}
throw err;
}
}
async getDocumentTypes(): Promise<any[]> {
const response = await this.client.get('/document_types/', {
params: { page_size: 9999 }
});
return response.data.results;
}
async getTags(): Promise<any[]> {
const response = await this.client.get('/tags/', {
params: { page_size: 9999 }
});
return response.data.results;
}
async getCorrespondents(params?: Record<string, any>): Promise<any> {
const response = await this.client.get('/correspondents/', { params });
return response.data;
}
async getCorrespondent(id: number): Promise<any> {
const response = await this.client.get(`/correspondents/${id}/`);
return response.data;
}
async getCustomFields(): Promise<any[]> {
const response = await this.client.get('/custom_fields/', {
params: { page_size: 9999 }
});
return response.data.results;
}
async getTask(taskId: string): Promise<any> {
const response = await this.client.get('/tasks/', {
params: { task_id: taskId },
});
return response.data;
}
async getCorrespondentByName(name: string): Promise<any> {
const response = await this.client.get('/correspondents/', {
params: { name__icontains: name },
});
return response.data.results.find((c: any) => c.name === name);
}
async addCorrespondent(data: any): Promise<any> {
const response = await this.client.post('/correspondents/', data);
return response.data;
}
async downloadDocument(id: number, type: 'original' | 'archive' = 'archive'): Promise<Buffer> {
const endpoint = type === 'original'
? `/documents/${id}/download/`
: `/documents/${id}/download/`;
const response = await this.client.get(endpoint, {
responseType: 'arraybuffer',
params: type === 'original' ? { original: true } : {},
timeout: 120000,
});
return Buffer.from(response.data);
}
async getDocumentPreviewStream(id: number): Promise<any> {
const response = await this.client.get(`/documents/${id}/thumb/`, {
responseType: 'stream',
timeout: 30000,
});
return response.data;
}
async getDocumentPdfStream(id: number, type: 'original' | 'archive' = 'archive'): Promise<any> {
const endpoint = type === 'original'
? `/documents/${id}/download/`
: `/documents/${id}/download/`;
const response = await this.client.get(endpoint, {
responseType: 'stream',
params: type === 'original' ? { original: true } : {},
timeout: 120000,
});
return response.data;
}
async getDocumentMetadata(id: number): Promise<any> {
const response = await this.client.get(`/documents/${id}/metadata/`);
return response.data;
}
async addNote(id: number, note: string): Promise<any> {
const response = await this.client.post(`/documents/${id}/notes/`, { note });
return response.data;
}
async checksumExists(checksum: string): Promise<boolean> {
const response = await this.client.get('/documents/', {
params: { checksum__iexact: checksum },
});
return response.data.count > 0;
}
/**
* Prüft, ob eine ASN bereits vergeben ist und wirft einen Fehler, falls ja.
*/
async validateAsnNotExists(interneBelegnummer: string): Promise<void> {
if (!interneBelegnummer) return;
// Logic like in PaperlessTaskProcessorService
const asnNum = parseInt(interneBelegnummer.replace(/-/g, ''), 10);
if (isNaN(asnNum)) return;
const existingDocId = await this.findDocumentIdByAsn(asnNum);
if (existingDocId) {
throw new HttpException(
`Die ASN ${asnNum} (aus Belegnummer ${interneBelegnummer}) existiert bereits in Paperless (Dokument ID: ${existingDocId}).`,
HttpStatus.CONFLICT,
);
}
}
/**
* Liefert die Paperless-Doc-ID des passenden Dokuments oder null.
*/
async findDocumentIdByAsn(asn: number): Promise<number | null> {
if (!Number.isFinite(asn)) return null;
const response = await this.client.get('/documents/', {
params: { archive_serial_number: asn, page_size: 5 },
});
if ((response.data.count ?? 0) === 0) return null;
const match = (response.data.results as any[] ?? []).find(
(doc: any) => Number(doc.archive_serial_number) === asn,
);
return match ? Number(match.id) : null;
}
/**
* Liefert die Paperless-Doc-ID des passenden Dokuments oder null.
* Paperless kann den custom_fields-Filter ignorieren daher manuell verifizieren.
*/
async findDocumentIdByCustomField(fieldId: number, value: string): Promise<number | null> {
const response = await this.client.get('/documents/', {
params: {
[`custom_fields__${fieldId}__value__iexact`]: value,
page_size: 25,
truncate_content: true,
},
});
if ((response.data.count ?? 0) === 0) return null;
const valueLower = value.toLowerCase();
const match = (response.data.results as any[] ?? []).find((doc: any) =>
(Array.isArray(doc.custom_fields) ? doc.custom_fields as any[] : []).some(
(cf: any) => cf.field === fieldId && String(cf.value ?? '').toLowerCase() === valueLower,
),
);
return match ? Number(match.id) : null;
}
}
@@ -0,0 +1,114 @@
import { Injectable, Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { ExportTarget } from '../database/entities/export-target.entity';
import * as ftp from 'basic-ftp';
import { createClient, type WebDAVClient } from 'webdav';
@Injectable()
export class ExportService {
private readonly logger = new Logger(ExportService.name);
constructor(
@InjectRepository(ExportTarget) private readonly targetRepo: Repository<ExportTarget>,
) {}
async exportFile(targetId: number, filename: string, content: Buffer): Promise<void> {
const target = await this.targetRepo.findOneByOrFail({ Id: targetId });
if (!target.IsActive) {
throw new Error(`Export-Ziel "${target.Name}" ist deaktiviert.`);
}
switch (target.Protocol) {
case 'ftp':
await this.uploadFtp(target, filename, content);
break;
case 'webdav':
await this.uploadWebDav(target, filename, content);
break;
default:
throw new Error(`Unbekanntes Protokoll: ${target.Protocol}`);
}
}
async testConnection(targetId: number): Promise<{ success: boolean; message: string }> {
const target = await this.targetRepo.findOneByOrFail({ Id: targetId });
try {
switch (target.Protocol) {
case 'ftp':
await this.testFtp(target);
break;
case 'webdav':
await this.testWebDav(target);
break;
default:
return { success: false, message: `Unbekanntes Protokoll: ${target.Protocol}` };
}
return { success: true, message: 'Verbindung erfolgreich.' };
} catch (err: any) {
return { success: false, message: err.message };
}
}
private async uploadFtp(target: ExportTarget, filename: string, content: Buffer): Promise<void> {
const client = new ftp.Client();
try {
await client.access({
host: target.Host,
port: target.Port || 21,
user: target.Username || 'anonymous',
password: target.Password || '',
secure: false,
});
const remotePath = `${target.RemotePath || '/'}/${filename}`;
const { Readable } = await import('stream');
const stream = Readable.from(content);
await client.uploadFrom(stream, remotePath);
this.logger.log(`FTP Upload: ${remotePath}${target.Name}`);
} finally {
client.close();
}
}
private async testFtp(target: ExportTarget): Promise<void> {
const client = new ftp.Client();
try {
await client.access({
host: target.Host,
port: target.Port || 21,
user: target.Username || 'anonymous',
password: target.Password || '',
secure: false,
});
await client.list(target.RemotePath || '/');
} finally {
client.close();
}
}
private async uploadWebDav(target: ExportTarget, filename: string, content: Buffer): Promise<void> {
const client = this.createWebDavClient(target);
const remotePath = `${target.RemotePath || '/'}/${filename}`;
await client.putFileContents(remotePath, content);
this.logger.log(`WebDAV Upload: ${remotePath}${target.Name}`);
}
private async testWebDav(target: ExportTarget): Promise<void> {
const client = this.createWebDavClient(target);
await client.getDirectoryContents(target.RemotePath || '/');
}
private createWebDavClient(target: ExportTarget): WebDAVClient {
const protocol = target.Port === 443 ? 'https' : 'http';
const port = target.Port ? `:${target.Port}` : '';
const url = `${protocol}://${target.Host}${port}`;
return createClient(url, {
username: target.Username || undefined,
password: target.Password || undefined,
});
}
}
@@ -0,0 +1,43 @@
import { Injectable, Logger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import * as nodemailer from 'nodemailer';
@Injectable()
export class MailService {
private readonly logger = new Logger(MailService.name);
private transporter: nodemailer.Transporter;
constructor(private readonly configService: ConfigService) {
this.transporter = nodemailer.createTransport({
host: this.configService.get<string>('SMTP_HOST', ''),
port: this.configService.get<number>('SMTP_PORT', 587),
secure: this.configService.get<string>('SMTP_SECURE', 'false') === 'true',
auth: {
user: this.configService.get<string>('SMTP_USER', ''),
pass: this.configService.get<string>('SMTP_PASS', ''),
},
});
}
async sendMail(options: {
to: string;
subject: string;
body: string;
attachments?: { filename: string; content: Buffer }[];
}): Promise<void> {
const from = this.configService.get<string>('SMTP_FROM', 'paperless@localhost');
await this.transporter.sendMail({
from,
to: options.to,
subject: options.subject,
text: options.body,
attachments: options.attachments?.map(a => ({
filename: a.filename,
content: a.content,
})),
});
this.logger.log(`Mail gesendet an ${options.to}: "${options.subject}"`);
}
}
@@ -0,0 +1,20 @@
import { Module, forwardRef } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { Postprocessing } from '../database/entities/postprocessing.entity';
import { PostprocessingAction } from '../database/entities/postprocessing-action.entity';
import { PostprocessingLog } from '../database/entities/postprocessing-log.entity';
import { ExportTarget } from '../database/entities/export-target.entity';
import { PostprocessingService } from './postprocessing.service';
import { MailService } from './mail.service';
import { ExportService } from './export.service';
import { PaperlessModule } from '../paperless/paperless.module';
@Module({
imports: [
TypeOrmModule.forFeature([Postprocessing, PostprocessingAction, PostprocessingLog, ExportTarget]),
forwardRef(() => PaperlessModule),
],
providers: [PostprocessingService, MailService, ExportService],
exports: [PostprocessingService, ExportService, MailService],
})
export class PostprocessingModule {}
@@ -0,0 +1,84 @@
import { Test, TestingModule } from '@nestjs/testing';
import { getRepositoryToken } from '@nestjs/typeorm';
import { PostprocessingService } from './postprocessing.service';
import { Postprocessing } from '../database/entities/postprocessing.entity';
import { PostprocessingAction } from '../database/entities/postprocessing-action.entity';
import { PaperlessService } from '../paperless/paperless.service';
const mockRules: Partial<Postprocessing>[] = [
{ Id: 1, Name: 'Rule1', DocumentTypeId: 5, CorrespondentId: null, OwnerId: null, TagId: null, Order: 1, IsActive: true, NoFurther: false },
{ Id: 2, Name: 'StopRule', DocumentTypeId: null, CorrespondentId: null, OwnerId: null, TagId: null, Order: 2, IsActive: true, NoFurther: true },
];
const mockActions: Partial<PostprocessingAction>[] = [
{ Id: 1, PostprocessingId: 1, ActionType: 2, Content: '99', Order: 1, IsActive: true },
];
describe('PostprocessingService', () => {
let service: PostprocessingService;
let ppRepo: any;
let ppActionRepo: any;
let paperlessService: any;
beforeEach(async () => {
ppRepo = { find: jest.fn().mockResolvedValue(mockRules) };
ppActionRepo = { find: jest.fn().mockResolvedValue(mockActions) };
paperlessService = { updateDocument: jest.fn().mockResolvedValue({}) };
const module: TestingModule = await Test.createTestingModule({
providers: [
PostprocessingService,
{ provide: getRepositoryToken(Postprocessing), useValue: ppRepo },
{ provide: getRepositoryToken(PostprocessingAction), useValue: ppActionRepo },
{ provide: PaperlessService, useValue: paperlessService },
],
}).compile();
service = module.get<PostprocessingService>(PostprocessingService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
it('evaluate loads active rules in order', async () => {
await service.evaluate({ documentId: 100, documentTypeId: 5, tagIds: [] });
expect(ppRepo.find).toHaveBeenCalledWith({
where: { IsActive: true },
order: { Order: 'ASC' },
});
});
it('evaluate executes matching actions', async () => {
// Rule1 matches documentTypeId=5 → action adds tag
await service.evaluate({ documentId: 100, documentTypeId: 5, tagIds: [] });
expect(ppActionRepo.find).toHaveBeenCalledWith({
where: { PostprocessingId: 1, IsActive: true },
order: { Order: 'ASC' },
});
expect(paperlessService.updateDocument).toHaveBeenCalledWith(100, { tags: [99] });
});
it('evaluate stops at NoFurther rule', async () => {
// Rule1 matches, Rule2 also matches (no filters) + NoFurther → stops
await service.evaluate({ documentId: 100, documentTypeId: 5, tagIds: [] });
// Actions loaded for rule 1 and rule 2, but no rule after rule 2
expect(ppActionRepo.find).toHaveBeenCalledTimes(2);
});
it('evaluate skips non-matching rules', async () => {
// documentTypeId=999 doesn't match Rule1 (requires 5) but matches Rule2 (no filter)
ppActionRepo.find.mockResolvedValue([]);
await service.evaluate({ documentId: 100, documentTypeId: 999, tagIds: [] });
// Rule1 skipped, Rule2 matched → only 1 action lookup
expect(ppActionRepo.find).toHaveBeenCalledTimes(1);
expect(ppActionRepo.find).toHaveBeenCalledWith({
where: { PostprocessingId: 2, IsActive: true },
order: { Order: 'ASC' },
});
});
});
@@ -0,0 +1,381 @@
import { Injectable, Logger, Inject, forwardRef } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { Postprocessing, type FilterGroup, type FilterCondition } from '../database/entities/postprocessing.entity';
import { PostprocessingAction } from '../database/entities/postprocessing-action.entity';
import { PostprocessingLog } from '../database/entities/postprocessing-log.entity';
import { PaperlessService } from '../paperless/paperless.service';
import { MailService } from './mail.service';
import { ExportService } from './export.service';
import axios from 'axios';
@Injectable()
export class PostprocessingService {
private readonly logger = new Logger(PostprocessingService.name);
private readonly errorTagId: number;
constructor(
@InjectRepository(Postprocessing) private readonly ppRepo: Repository<Postprocessing>,
@InjectRepository(PostprocessingAction) private readonly actionRepo: Repository<PostprocessingAction>,
@InjectRepository(PostprocessingLog) private readonly logRepo: Repository<PostprocessingLog>,
private readonly configService: ConfigService,
@Inject(forwardRef(() => PaperlessService)) private readonly paperlessService: PaperlessService,
private readonly mailService: MailService,
private readonly exportService: ExportService,
) {
this.errorTagId = this.configService.get<number>('POSTPROCESSING_ERROR_TAG', 0);
}
async evaluate(doc: any): Promise<void> {
const rules = await this.ppRepo.find({
where: { IsActive: true },
order: { Order: 'ASC' },
});
// Enrich doc with resolved names (once per evaluation)
await this.enrichDocWithNames(doc);
this.logger.log(`[Postprocessing] Dokument ${doc.id} wird gegen ${rules.length} Regel(n) geprüft`);
for (const rule of rules) {
if (!this.hasConditions(rule.FilterJson)) {
this.logger.warn(`[Postprocessing] Regel "${rule.Name}" (ID: ${rule.Id}) hat keine Bedingungen wird übersprungen.`);
continue;
}
this.logger.debug(`[Postprocessing] Prüfe Regel "${rule.Name}" (ID: ${rule.Id}) für Dokument ${doc.id}`);
const matches = this.matchesFilter(rule.FilterJson, doc);
this.logger.log(`[Postprocessing] Regel "${rule.Name}" (ID: ${rule.Id}) → ${matches ? 'TRIFFT ZU' : 'trifft nicht zu'}`);
if (!matches) continue;
this.logger.log(`[Postprocessing] Regel "${rule.Name}" trifft zu für Dokument ${doc.id}`);
const actions = await this.actionRepo.find({
where: { PostprocessingId: rule.Id, IsActive: true },
order: { Order: 'ASC' },
});
let hasError = false;
for (const action of actions) {
try {
await this.executeAction(action, doc);
await this.log(rule.Id, action.Id, doc.id, 'success', `Aktion ${action.ActionType} erfolgreich`);
} catch (err: any) {
hasError = true;
this.logger.error(`Fehler bei Aktion ${action.Id} (Typ ${action.ActionType}) für Dokument ${doc.id}: ${err.message}`);
await this.log(rule.Id, action.Id, doc.id, 'error', err.message);
}
}
if (hasError && this.errorTagId) {
try {
const currentTags = new Set<number>(doc.tags || []);
currentTags.add(this.errorTagId);
await this.paperlessService.updateDocument(doc.id, { tags: Array.from(currentTags) });
} catch (tagErr: any) {
this.logger.error(`Konnte Error-Tag ${this.errorTagId} nicht setzen für Dokument ${doc.id}: ${tagErr.message}`);
}
}
if (rule.NoFurther) {
this.logger.debug('NoFurther keine weiteren Regeln');
break;
}
}
}
// ── Recursive Filter Evaluation ──────────────────────────────────
private hasConditions(filter: FilterGroup): boolean {
if (!filter || !filter.rules || filter.rules.length === 0) return false;
return filter.rules.some(rule => {
if ('combinator' in rule) return this.hasConditions(rule as FilterGroup);
return true;
});
}
private matchesFilter(filter: FilterGroup, doc: any): boolean {
if (!filter || !filter.rules || filter.rules.length === 0) return false;
const results = filter.rules.map(rule => {
if ('combinator' in rule) {
return this.matchesFilter(rule as FilterGroup, doc);
}
return this.evaluateCondition(rule as FilterCondition, doc);
});
return filter.combinator === 'AND'
? results.every(Boolean)
: results.some(Boolean);
}
private evaluateCondition(cond: FilterCondition, doc: any): boolean {
const actual = this.resolveFieldValue(cond.field, doc);
const expected = cond.value;
let result: boolean;
switch (cond.operator) {
case 'equals':
if (cond.field === 'tag') {
result = Array.isArray(actual) && actual.includes(Number(expected));
} else {
result = String(actual ?? '') === String(expected ?? '');
}
break;
case 'not_equals':
if (cond.field === 'tag') {
result = !Array.isArray(actual) || !actual.includes(Number(expected));
} else {
result = String(actual ?? '') !== String(expected ?? '');
}
break;
case 'contains':
if (cond.field === 'tag') {
result = Array.isArray(actual) && actual.includes(Number(expected));
} else {
result = String(actual ?? '').toLowerCase().includes(String(expected ?? '').toLowerCase());
}
break;
case 'not_contains':
if (cond.field === 'tag') {
result = !Array.isArray(actual) || !actual.includes(Number(expected));
} else {
result = !String(actual ?? '').toLowerCase().includes(String(expected ?? '').toLowerCase());
}
break;
case 'is_set':
result = actual !== null && actual !== undefined && actual !== '';
break;
case 'is_not_set':
result = actual === null || actual === undefined || actual === '';
break;
case 'gt':
result = parseFloat(actual) > parseFloat(expected);
break;
case 'lt':
result = parseFloat(actual) < parseFloat(expected);
break;
default:
this.logger.warn(`Unbekannter Operator: ${cond.operator}`);
result = false;
}
this.logger.debug(
`[Postprocessing] Bedingung: ${cond.field} ${cond.operator} "${expected}" | Istwert: "${Array.isArray(actual) ? actual.join(',') : actual}" → ${result ? 'WAHR' : 'FALSCH'}`
);
return result;
}
private resolveFieldValue(field: string, doc: any): any {
switch (field) {
case 'document_type':
return doc.document_type;
case 'correspondent':
return doc.correspondent;
case 'owner':
return doc.owner;
case 'tag':
return doc.tags || [];
case 'title':
return doc.title;
case 'archive_serial_number':
return doc.archive_serial_number;
default:
// Custom field: "custom_field_<id>"
if (field.startsWith('custom_field_')) {
const fieldId = parseInt(field.replace('custom_field_', ''), 10);
const cf = (doc.custom_fields || []).find((f: any) => f.field === fieldId);
return cf?.value ?? null;
}
return null;
}
}
// ── Action Execution ─────────────────────────────────────────────
private async executeAction(action: PostprocessingAction, doc: any): Promise<void> {
const content = action.Content;
switch (action.ActionType) {
case 1: // Export
await this.handleExport(content, doc);
break;
case 2: // Mail
await this.handleMail(content, doc);
break;
case 3: // Tag setzen/entfernen
await this.handleTags(content, doc);
break;
case 4: // Custom Field setzen
await this.handleCustomField(content, doc);
break;
case 5: // Webhook
await this.handleWebhook(content, doc);
break;
case 6: // Notiz hinzufügen
await this.handleNote(content, doc);
break;
default:
this.logger.warn(`Unbekannter ActionType: ${action.ActionType}`);
}
}
private async enrichDocWithNames(doc: any): Promise<void> {
try {
if (doc.correspondent && !doc._correspondentName) {
const response = await this.paperlessService.getCorrespondents({ page_size: 9999 });
const correspondents = response.results;
const c = correspondents.find((x: any) => x.id === doc.correspondent);
doc._correspondentName = c?.name ?? '';
}
if (doc.document_type && !doc._documentTypeName) {
const docTypes = await this.paperlessService.getDocumentTypes();
const dt = docTypes.find((x: any) => x.id === doc.document_type);
doc._documentTypeName = dt?.name ?? '';
}
} catch (err: any) {
this.logger.warn(`Konnte Namen nicht auflösen: ${err.message}`);
}
}
private resolveTemplate(template: string, doc: any): string {
const created = doc.created ? new Date(doc.created) : null;
const replacements: Record<string, string> = {
'{id}': String(doc.id ?? ''),
'{titel}': doc.title ?? '',
'{korrespondent}': String(doc.correspondent ?? ''),
'{absender}': String(doc.correspondent ?? ''),
'{korrespondent_name}': doc._correspondentName ?? String(doc.correspondent ?? ''),
'{absender_name}': doc._correspondentName ?? String(doc.correspondent ?? ''),
'{dokumenttyp}': String(doc.document_type ?? ''),
'{dokumenttyp_name}': doc._documentTypeName ?? String(doc.document_type ?? ''),
'{besitzer}': String(doc.owner ?? ''),
'{ablagenummer}': String(doc.archive_serial_number ?? ''),
'{datum}': created ? `${created.getFullYear()}-${String(created.getMonth() + 1).padStart(2, '0')}-${String(created.getDate()).padStart(2, '0')}` : '',
'{jahr}': created ? String(created.getFullYear()) : '',
'{monat}': created ? String(created.getMonth() + 1).padStart(2, '0') : '',
'{tag}': created ? String(created.getDate()).padStart(2, '0') : '',
'{zeitstempel}': (() => {
const now = new Date();
return `${String(now.getDate()).padStart(2, '0')}.${String(now.getMonth() + 1).padStart(2, '0')}.${now.getFullYear()} ${String(now.getHours()).padStart(2, '0')}:${String(now.getMinutes()).padStart(2, '0')}:${String(now.getSeconds()).padStart(2, '0')}`;
})(),
};
// Custom Fields: {custom_field_<id>}
for (const cf of (doc.custom_fields || [])) {
replacements[`{custom_field_${cf.field}}`] = String(cf.value ?? '');
}
let result = template;
for (const [placeholder, value] of Object.entries(replacements)) {
result = result.replaceAll(placeholder, value);
}
// Sanitize filename chars
return result.replace(/[<>:"/\\|?*]/g, '_');
}
private buildFilename(template: string | undefined, doc: any): string {
if (template) {
const resolved = this.resolveTemplate(template, doc);
return resolved.endsWith('.pdf') ? resolved : `${resolved}.pdf`;
}
return `${doc.title || `document_${doc.id}`}.pdf`;
}
private async handleExport(content: Record<string, any>, doc: any): Promise<void> {
const fileType = content.fileType || 'archive';
const buffer = await this.paperlessService.downloadDocument(doc.id, fileType);
const filename = this.buildFilename(content.filenameTemplate, doc);
await this.exportService.exportFile(content.exportTargetId, filename, buffer);
}
private async handleMail(content: Record<string, any>, doc: any): Promise<void> {
const fileType = content.fileType || 'archive';
const buffer = await this.paperlessService.downloadDocument(doc.id, fileType);
const filename = this.buildFilename(content.filenameTemplate, doc);
const subject = content.subject
? this.resolveTemplate(content.subject, doc)
: `Dokument: ${doc.title}`;
const body = content.body
? this.resolveTemplate(content.body, doc)
: `Anbei das Dokument "${doc.title}" (ID: ${doc.id}).`;
await this.mailService.sendMail({
to: content.to,
subject,
body,
attachments: [{ filename, content: buffer }],
});
}
private async handleTags(content: Record<string, any>, doc: any): Promise<void> {
const currentTags = new Set<number>(doc.tags || []);
const addTags: number[] = content.addTags || [];
const removeTags: number[] = content.removeTags || [];
addTags.forEach(t => currentTags.add(t));
removeTags.forEach(t => currentTags.delete(t));
await this.paperlessService.updateDocument(doc.id, { tags: Array.from(currentTags) });
}
private async handleCustomField(content: Record<string, any>, doc: any): Promise<void> {
const customFields = [...(doc.custom_fields || [])];
const existing = customFields.find((f: any) => f.field === content.fieldId);
if (existing) {
existing.value = content.value;
} else {
customFields.push({ field: content.fieldId, value: content.value });
}
await this.paperlessService.updateDocument(doc.id, { custom_fields: customFields });
}
private async handleWebhook(content: Record<string, any>, doc: any): Promise<void> {
const method = (content.method || 'POST').toUpperCase();
const headers = content.headers || {};
const body = { documentId: doc.id, title: doc.title, tags: doc.tags, ...(content.body || {}) };
await axios({
method,
url: content.url,
headers,
data: method !== 'GET' ? body : undefined,
params: method === 'GET' ? body : undefined,
timeout: 30000,
});
this.logger.log(`Webhook ${method}${content.url}`);
}
private async handleNote(content: Record<string, any>, doc: any): Promise<void> {
if (!content.note) return;
const resolvedNote = this.resolveTemplate(content.note, doc);
await this.paperlessService.addNote(doc.id, resolvedNote);
}
// ── Logging ──────────────────────────────────────────────────────
private async log(ppId: number, actionId: number | null, docId: number, status: string, message: string): Promise<void> {
const entry = this.logRepo.create({
PostprocessingId: ppId,
ActionId: actionId,
DocumentId: docId,
Status: status,
Message: message,
CreatedAt: new Date(),
});
await this.logRepo.save(entry);
}
}
@@ -0,0 +1,118 @@
import { Injectable, Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { ConfigService } from '@nestjs/config';
import * as fs from 'fs/promises';
import * as path from 'path';
import { v4 as uuidv4 } from 'uuid';
import { Task } from '../database/entities/task.entity';
import { PdfService } from './pdf.service';
import { QrCodeService } from './qr-code.service';
import { OcrService } from './ocr.service';
@Injectable()
export class DocumentPipelineService {
private readonly logger = new Logger(DocumentPipelineService.name);
private readonly archiveDir: string;
constructor(
@InjectRepository(Task) private readonly taskRepo: Repository<Task>,
private readonly pdfService: PdfService,
private readonly qrCodeService: QrCodeService,
private readonly ocrService: OcrService,
private readonly configService: ConfigService,
) {
this.archiveDir = this.configService.get<string>(
'SCANNER_ARCHIVE_DIR',
'/data/scanner/_processed_archive',
);
}
/**
* Verarbeitet ein neues Dokument:
* 1. PDF Bilder
* 2. QR-Code-Erkennung auf Seite 1
* 3. OCR via Ollama Vision auf Seite 1
* 4. Task in DB erstellen (Inbox-Eintrag)
* 5. Original in Archiv verschieben (GoBD)
*/
async processDocument(filePath: string): Promise<Task> {
const taskId = uuidv4();
const fileName = path.basename(filePath);
this.logger.log(`Pipeline startet: ${fileName} (${taskId})`);
let images: string[] = [];
try {
// 1. PDF → Bild(er)
images = await this.pdfService.pdfToImages(filePath, 200);
this.logger.log(`${images.length} Seite(n) konvertiert`);
// 2. QR-Code auf erster Seite scannen
const firstPageBuffer = await fs.readFile(images[0]);
const qrResults = await this.qrCodeService.extractFromImage(firstPageBuffer);
let barcodeData: Record<string, any> | null = null;
if (qrResults.length > 0) {
barcodeData = this.qrCodeService.parseBarcode(qrResults[0].data);
if (barcodeData) {
this.logger.log(`QR-Code erkannt und validiert: ${JSON.stringify(barcodeData)}`);
}
}
// 3. OCR auf erster Seite
const ocrMarkdown = await this.ocrService.extractTextAsMarkdown(firstPageBuffer);
// 4. Task in DB erstellen
const year = new Date().getFullYear();
const lastTask = await this.taskRepo
.createQueryBuilder('t')
.where('t.InterneBelegnummer LIKE :prefix', { prefix: `${year}-%` })
.orderBy('t.InterneBelegnummer', 'DESC')
.getOne();
const nextNum = lastTask
? parseInt(lastTask.InterneBelegnummer.split('-')[1], 10) + 1
: 1;
const belegnummer = `${year}-${String(nextNum).padStart(6, '0')}`;
const task = this.taskRepo.create({
TaskId: taskId,
InterneBelegnummer: belegnummer,
Eingangsdatum: new Date(),
Fertig: 0,
BarcodeJson: barcodeData ? JSON.stringify(barcodeData) : null,
DocumentType: barcodeData?.DocumentType ?? null,
BetriebID: barcodeData?.BetriebID ?? null,
Lieferant: barcodeData?.Lieferant ?? null,
externeBelegnummer: barcodeData?.Nummer ?? null,
});
await this.taskRepo.save(task);
this.logger.log(`Task erstellt: ${belegnummer}`);
// 5. GoBD-Archivierung
await this.archiveFile(filePath);
return task;
} finally {
await this.pdfService.cleanup(images);
}
}
/**
* Verschiebt die Originaldatei ins Archiv (GoBD-konform).
*/
private async archiveFile(filePath: string): Promise<void> {
await fs.mkdir(this.archiveDir, { recursive: true });
const datePrefix = new Date().toISOString().slice(0, 10);
const fileName = path.basename(filePath);
const archivePath = path.join(this.archiveDir, `${datePrefix}_${fileName}`);
await fs.rename(filePath, archivePath);
this.logger.log(`Archiviert: ${archivePath}`);
}
}
@@ -0,0 +1,49 @@
import { Injectable, Logger } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import axios from 'axios';
@Injectable()
export class OcrService {
private readonly logger = new Logger(OcrService.name);
private readonly ollamaUrl: string;
private readonly ollamaModel: string;
constructor(private readonly configService: ConfigService) {
this.ollamaUrl = this.configService.get<string>('OLLAMA_URL', 'http://localhost:11434');
this.ollamaModel = this.configService.get<string>('OLLAMA_MODEL', 'llava');
}
/**
* Sendet ein Bild an Ollama Vision und erhält den Inhalt als Markdown.
*/
async extractTextAsMarkdown(imageBuffer: Buffer): Promise<string> {
const base64Image = imageBuffer.toString('base64');
const prompt = `Analysiere dieses Dokument und extrahiere den gesamten Text.
Gib den Text als sauberes Markdown zurück, behalte die Struktur bei (Überschriften, Tabellen, Listen).
Antworte nur mit dem extrahierten Markdown-Text, keine Erklärungen.`;
try {
const response = await axios.post(
`${this.ollamaUrl}/api/generate`,
{
model: this.ollamaModel,
prompt,
images: [base64Image],
stream: false,
options: {
temperature: 0.1,
},
},
{ timeout: 120000 },
);
const markdown = response.data.response?.trim() ?? '';
this.logger.log(`OCR abgeschlossen: ${markdown.length} Zeichen extrahiert`);
return markdown;
} catch (error: any) {
this.logger.error(`Ollama OCR fehlgeschlagen: ${error.message}`);
throw error;
}
}
}
@@ -0,0 +1,98 @@
import { Injectable, Logger } from '@nestjs/common';
import { execFile } from 'child_process';
import { promisify } from 'util';
import * as path from 'path';
import * as fs from 'fs/promises';
import * as os from 'os';
const execFileAsync = promisify(execFile);
@Injectable()
export class PdfService {
private readonly logger = new Logger(PdfService.name);
/**
* Konvertiert eine PDF-Seite in ein PNG-Bild via Ghostscript.
* Gibt den Pfad zum temporären Bild zurück.
*/
async pdfPageToImage(pdfPath: string, page = 1, dpi = 300): Promise<string> {
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'pdf-'));
const outputPath = path.join(tmpDir, `page-${page}.png`);
await execFileAsync('gs', [
'-dNOPAUSE',
'-dBATCH',
'-dSAFER',
'-sDEVICE=png16m',
`-dFirstPage=${page}`,
`-dLastPage=${page}`,
`-r${dpi}`,
`-sOutputFile=${outputPath}`,
pdfPath,
]);
this.logger.debug(`PDF Seite ${page} konvertiert: ${outputPath}`);
return outputPath;
}
/**
* Konvertiert alle Seiten einer PDF in Bilder.
*/
async pdfToImages(pdfPath: string, dpi = 200): Promise<string[]> {
const pageCount = await this.getPageCount(pdfPath);
const images: string[] = [];
for (let i = 1; i <= pageCount; i++) {
const imgPath = await this.pdfPageToImage(pdfPath, i, dpi);
images.push(imgPath);
}
return images;
}
/**
* Ermittelt die Seitenanzahl einer PDF via Ghostscript.
*/
async getPageCount(pdfPath: string): Promise<number> {
const { stdout } = await execFileAsync('gs', [
'-q',
'-dNODISPLAY',
'-dNOSAFER',
`-c`,
`(${pdfPath.replace(/\\/g, '/')}) (r) file runpdfbegin pdfpagecount = quit`,
]);
return parseInt(stdout.trim(), 10) || 1;
}
/**
* Bereinigt eine PDF (entschlüsselt sie ggf. wenn nur Owner-Passwort gesetzt ist)
* via Ghostscript pdfwrite.
*/
async sanitizePdf(inputPath: string): Promise<string> {
const outputPath = path.join(os.tmpdir(), `sanitized-${Date.now()}.pdf`);
await execFileAsync('gs', [
'-dNOPAUSE',
'-dBATCH',
'-dSAFER',
'-sDEVICE=pdfwrite',
`-sOutputFile=${outputPath}`,
inputPath,
]);
return outputPath;
}
/**
* Räumt temporäre Bilder auf.
*/
async cleanup(imagePaths: string[]): Promise<void> {
for (const imgPath of imagePaths) {
try {
await fs.unlink(imgPath);
const dir = path.dirname(imgPath);
await fs.rmdir(dir).catch(() => {});
} catch {
// Ignorieren
}
}
}
}
@@ -0,0 +1,14 @@
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { Task } from '../database/entities/task.entity';
import { QrCodeService } from './qr-code.service';
import { OcrService } from './ocr.service';
import { PdfService } from './pdf.service';
import { DocumentPipelineService } from './document-pipeline.service';
@Module({
imports: [TypeOrmModule.forFeature([Task])],
providers: [QrCodeService, OcrService, PdfService, DocumentPipelineService],
exports: [QrCodeService, OcrService, PdfService, DocumentPipelineService],
})
export class PreprocessingModule {}
@@ -0,0 +1,102 @@
import { Injectable, Logger } from '@nestjs/common';
import sharp = require('sharp');
import jsQR from 'jsqr';
export interface QrCodeResult {
data: string;
location: {
x: number;
y: number;
width: number;
height: number;
};
}
@Injectable()
export class QrCodeService {
private readonly logger = new Logger(QrCodeService.name);
/**
* Extrahiert ALLE QR-Codes aus einem Bild-Buffer (PNG/JPEG).
* jsQR findet nur einen Code pro Aufruf daher iteratives Vorgehen:
* Code finden Bereich weiß überdecken erneut scannen, bis nichts mehr gefunden wird.
*/
async extractFromImage(imageBuffer: Buffer): Promise<QrCodeResult[]> {
const results: QrCodeResult[] = [];
const seen = new Set<string>();
let currentBuffer = imageBuffer;
const MAX_PASSES = 10;
for (let pass = 0; pass < MAX_PASSES; pass++) {
const { data, info } = await sharp(currentBuffer)
.ensureAlpha()
.raw()
.toBuffer({ resolveWithObject: true });
const imageData = new Uint8ClampedArray(data.buffer);
const code = jsQR(imageData, info.width, info.height, {
inversionAttempts: 'attemptBoth',
});
if (!code) break;
const corners = [
code.location.topLeftCorner,
code.location.topRightCorner,
code.location.bottomLeftCorner,
code.location.bottomRightCorner,
];
const xs = corners.map((c) => c.x);
const ys = corners.map((c) => c.y);
const minX = Math.floor(Math.min(...xs));
const minY = Math.floor(Math.min(...ys));
const maxX = Math.ceil(Math.max(...xs));
const maxY = Math.ceil(Math.max(...ys));
const width = Math.max(1, maxX - minX);
const height = Math.max(1, maxY - minY);
if (!seen.has(code.data)) {
seen.add(code.data);
results.push({
data: code.data,
location: { x: minX, y: minY, width, height },
});
this.logger.debug(`QR-Code erkannt (Pass ${pass + 1}): ${code.data}`);
}
// Erkannten Bereich mit weißem Rechteck (inkl. Padding) überdecken,
// damit jsQR im nächsten Pass den nächsten QR findet.
const pad = 12;
const maskX = Math.max(0, minX - pad);
const maskY = Math.max(0, minY - pad);
const maskW = Math.min(info.width - maskX, width + 2 * pad);
const maskH = Math.min(info.height - maskY, height + 2 * pad);
const svg = `<svg width="${info.width}" height="${info.height}" xmlns="http://www.w3.org/2000/svg"><rect x="${maskX}" y="${maskY}" width="${maskW}" height="${maskH}" fill="white"/></svg>`;
currentBuffer = await sharp(currentBuffer)
.composite([{ input: Buffer.from(svg), top: 0, left: 0 }])
.png()
.toBuffer();
}
return results;
}
/**
* Validiert ob der QR-Code-Inhalt dem erwarteten Schema entspricht.
* Schema: JSON mit X, Y, Jahr, Nummer, Eingangsdatum
*/
parseBarcode(qrData: string): Record<string, any> | null {
try {
const parsed = JSON.parse(qrData);
if (parsed.Jahr !== undefined && parsed.Nummer !== undefined) {
return parsed;
}
this.logger.warn(`QR-Code-Daten passen nicht zum Schema: ${qrData}`);
return null;
} catch {
this.logger.debug(`QR-Code ist kein JSON: ${qrData}`);
return null;
}
}
}
@@ -0,0 +1,249 @@
import { Injectable, Logger, OnModuleInit, OnModuleDestroy } from '@nestjs/common';
import { Cron } from '@nestjs/schedule';
import { ConfigService } from '@nestjs/config';
import { InjectRepository } from '@nestjs/typeorm';
import { IsNull, Repository } from 'typeorm';
import { randomUUID } from 'crypto';
import * as chokidar from 'chokidar';
import * as path from 'path';
import * as fs from 'fs/promises';
import { BarcodeScannerService } from '../barcode/barcode-scanner.service';
import { PageCacheService } from '../barcode/page-cache.service';
import {
InboxDocument,
type InboxSource,
} from '../database/entities/inbox-document.entity';
const STABILITY_MS = 5000;
@Injectable()
export class ScannerWatcherService implements OnModuleInit, OnModuleDestroy {
private readonly logger = new Logger(ScannerWatcherService.name);
private watcher: chokidar.FSWatcher | null = null;
private readonly sourceRoot: string;
private readonly processing = new Set<string>();
private isPeriodicScanning = false;
constructor(
private readonly configService: ConfigService,
private readonly barcodeScanner: BarcodeScannerService,
private readonly pageCache: PageCacheService,
@InjectRepository(InboxDocument)
private readonly documentRepo: Repository<InboxDocument>,
) {
this.sourceRoot = this.configService.get<string>('SCANNER_WATCH_DIR', '/mnt/scans');
}
onModuleInit(): void {
this.startWatching();
// Sequenziell, sonst greifen sich initialScan (Watcher) und Backfill
// dieselbe frisch angelegte Row und scannen doppelt. Fire-and-forget,
// damit der Modulstart nicht blockiert.
void this.bootstrap();
}
private async bootstrap(): Promise<void> {
await this.initialScan();
await this.backfillMissingScans();
}
onModuleDestroy(): void {
this.stopWatching();
}
private startWatching(): void {
this.logger.log(`Starte Überwachung: ${this.sourceRoot}`);
this.watcher = chokidar.watch(this.sourceRoot, {
ignored: /(^|[\/\\])\../,
persistent: true,
ignoreInitial: true,
awaitWriteFinish: {
stabilityThreshold: STABILITY_MS,
pollInterval: 500,
},
depth: 1,
});
this.watcher
.on('add', (filePath: string) => this.handleNewFile(filePath))
.on('error', (error: Error) => this.logger.error(`Watcher Fehler: ${error.message}`));
this.logger.log('Scanner-Watcher aktiv');
}
private stopWatching(): void {
if (this.watcher) {
this.watcher.close();
this.logger.log('Scanner-Watcher gestoppt');
}
}
private async initialScan(silent = false): Promise<void> {
let subdirs: string[];
try {
const entries = await fs.readdir(this.sourceRoot, { withFileTypes: true });
subdirs = entries.filter((e) => e.isDirectory()).map((e) => e.name);
} catch (err: any) {
if (!silent) {
this.logger.warn(`Scanner-Check: Quellverzeichnis nicht lesbar (${this.sourceRoot}): ${err.message}`);
}
return;
}
let seen = 0;
for (const subdir of subdirs) {
const dir = path.join(this.sourceRoot, subdir);
let files: string[];
try {
files = await fs.readdir(dir);
} catch (err: any) {
if (!silent) {
this.logger.warn(`Scanner-Check: ${dir} nicht lesbar: ${err.message}`);
}
continue;
}
for (const name of files) {
if (path.extname(name).toLowerCase() !== '.pdf') continue;
const full = path.join(dir, name);
if (!(await this.isStable(full))) {
if (!silent) {
this.logger.debug(`Scanner-Check: ${full} noch nicht stabil Watcher übernimmt`);
}
continue;
}
seen += 1;
await this.handleNewFile(full);
}
}
if (seen > 0) {
this.logger.log(`Scanner-Check: ${seen} Datei(en) verarbeitet`);
} else if (!silent) {
this.logger.log('Scanner-Check: keine neuen Dateien gefunden');
}
}
@Cron('*/15 * * * * *')
async periodicScan(): Promise<void> {
if (this.isPeriodicScanning) return;
this.isPeriodicScanning = true;
try {
await this.initialScan(true);
} catch (err: any) {
this.logger.error(`Periodic Scan Fehler: ${err.message}`);
} finally {
this.isPeriodicScanning = false;
}
}
private async isStable(filePath: string): Promise<boolean> {
try {
const stat = await fs.stat(filePath);
return Date.now() - stat.mtimeMs >= STABILITY_MS;
} catch {
return false;
}
}
private async handleNewFile(filePath: string): Promise<void> {
if (path.extname(filePath).toLowerCase() !== '.pdf') return;
const relative = path.relative(this.sourceRoot, filePath);
const parts = relative.split(path.sep);
if (parts.length !== 2) {
this.logger.debug(`Überspringe (falsche Tiefe): ${filePath}`);
return;
}
const subdir = parts[0];
const fileName = parts[1];
if (this.processing.has(filePath)) return;
this.processing.add(filePath);
try {
const id = randomUUID();
const source: InboxSource = subdir === 'all' ? 'all' : 'user';
const owner = source === 'all' ? null : subdir;
const targetDir = this.pageCache.documentDir(id);
const targetPdf = this.pageCache.documentPdfPath(id);
await fs.mkdir(targetDir, { recursive: true });
await this.move(filePath, targetPdf);
const doc = this.documentRepo.create({
Id: id,
OriginalName: fileName,
Source: source,
OwnerUsername: owner,
PageCount: 0,
QrCodes: [],
});
await this.documentRepo.save(doc);
this.logger.log(`Übernommen: ${relative}${id}/document.pdf`);
try {
await this.barcodeScanner.scanAndMatch(doc);
} catch (err: any) {
this.logger.warn(`Barcode-Scan nach Move fehlgeschlagen (${id}): ${err.message}`);
}
} catch (err: any) {
this.logger.error(`Übernahme fehlgeschlagen für ${filePath}: ${err.message}`);
} finally {
this.processing.delete(filePath);
}
}
private async backfillMissingScans(): Promise<void> {
let pending: InboxDocument[];
try {
pending = await this.documentRepo.find({
where: [{ PageCount: 0 }, { QrCodes: IsNull() }],
});
} catch (err: any) {
this.logger.warn(`Backfill: DB-Query fehlgeschlagen: ${err.message}`);
return;
}
let scanned = 0;
for (const doc of pending) {
try {
const didScan = await this.barcodeScanner.ensureScanned(doc);
if (didScan) scanned += 1;
} catch (err: any) {
this.logger.warn(`Backfill fehlgeschlagen (${doc.Id}): ${err.message}`);
}
}
if (scanned > 0) {
this.logger.log(`Backfill: ${scanned} Datei(en) nachträglich gescannt`);
} else {
this.logger.log('Backfill: alle Dateien bereits gescannt');
}
}
private async move(src: string, dest: string): Promise<void> {
try {
await fs.rename(src, dest);
return;
} catch (err: any) {
if (err.code !== 'EXDEV') throw err;
}
// Cross-device: copy + unlink. Wenn unlink scheitert, Kopie zurückrollen,
// damit ein kaputter Mount nicht bei jedem Neustart Duplikate produziert.
await fs.copyFile(src, dest);
try {
await fs.unlink(src);
} catch (err) {
await fs.unlink(dest).catch(() => undefined);
throw err;
}
}
}

Some files were not shown because too many files have changed in this diff Show More