Publish curated Knowledge Hub source from 2ed833f290b439a7bc91e5fd19b042c269e3bee9
This commit is contained in:
Public Source Publisher committed
1 parent
10f2edcd29
commit
59f15b2d08
218 files changed
+23978
-1832
No files matched your search
@@ -10,6 +10,8 @@
|
||||
**/node_modules
|
||||
**/dist
|
||||
artifacts
|
||||
.local-appdata
|
||||
.local-test-results
|
||||
data
|
||||
backups
|
||||
imports
|
||||
|
||||
@@ -89,6 +89,7 @@ Thumbs.db
|
||||
*.backup
|
||||
|
||||
# Local logs
|
||||
/.local-test-results/
|
||||
*.log
|
||||
logs/
|
||||
|
||||
|
||||
@@ -3,6 +3,9 @@
|
||||
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version="1.19.1" />
|
||||
<PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.19.0" />
|
||||
<PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.19.1" />
|
||||
<PackageVersion Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.10" />
|
||||
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.8.1" />
|
||||
<PackageVersion Include="xunit" Version="2.9.3" />
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
# Public source export
|
||||
|
||||
This source snapshot was generated from the private canonical repository at revision `883ef812ee04516208b84972195b5e2a46207e44`. It contains no private Git history or operational evidence. Changes are published from the canonical repository.
|
||||
This source snapshot was generated from the private canonical repository at revision `2ed833f290b439a7bc91e5fd19b042c269e3bee9`. It contains no private Git history or operational evidence. Changes are published from the canonical repository.
|
||||
+2458
-921
File diff suppressed because it is too large.
Load diff
@@ -18,6 +18,10 @@ this repository.
|
||||
- Track projects, milestones, updates, architecture decisions and runbook validation.
|
||||
- Import LeafWiki content through a read-only, preflighted and reversible workflow.
|
||||
- Create scoped, revocable service-account tokens for the versioned `/api/v1` API.
|
||||
- Synchronize personal preferences across devices and manage versioned platform policy.
|
||||
- Follow durable course-tagging and backup jobs with live progress and cancellation.
|
||||
- Search knowledge, projects, services, runbooks and decisions from one workspace.
|
||||
- Manage source freshness and inspect recorded deployment, recovery and telemetry evidence.
|
||||
- Back up the database and attachments together, then verify a clean-target restore.
|
||||
|
||||
## Screenshots
|
||||
|
||||
@@ -57,8 +57,10 @@ ENV ASPNETCORE_URLS=http://0.0.0.0:8080 \
|
||||
DOTNET_EnableDiagnostics=0 \
|
||||
Build__Commit=${BUILD_COMMIT}
|
||||
|
||||
# Explicitly refresh OpenSSL too: the pinned base may predate security
|
||||
# updates even when curl's already-satisfied dependencies would keep it.
|
||||
RUN apt-get update \
|
||||
&& apt-get install --yes --no-install-recommends curl ca-certificates \
|
||||
&& apt-get install --yes --no-install-recommends curl ca-certificates libssl3t64 openssl \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& groupadd --gid 10001 knowledgehub \
|
||||
&& useradd --uid 10001 --gid 10001 --no-create-home --shell /usr/sbin/nologin knowledgehub \
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 128
|
||||
batch:
|
||||
timeout: 5s
|
||||
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:8889
|
||||
metric_expiration: 2m
|
||||
|
||||
service:
|
||||
pipelines:
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [prometheus]
|
||||
@@ -0,0 +1,47 @@
|
||||
services:
|
||||
app:
|
||||
environment:
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
|
||||
OTEL_METRIC_EXPORT_INTERVAL: "15000"
|
||||
|
||||
otel-collector:
|
||||
image: ${KNOWLEDGE_HUB_COLLECTOR_IMAGE:-otel/opentelemetry-collector-contrib:0.162.0-amd64@sha256:340885ab6f46822374f00c58c067396b53b8d612559842e8821585567579d9e3}
|
||||
restart: unless-stopped
|
||||
command: ["--config=/etc/otelcol-contrib/config.yaml"]
|
||||
volumes:
|
||||
- ./deploy/observability/collector.yaml:/etc/otelcol-contrib/config.yaml:ro
|
||||
read_only: true
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
networks: [knowledge-hub-internal]
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.14.0@sha256:5ce7540c3c00ef4ab0c9d2c995c6a5b9c421f44b4a115d97a2c7af3b1c21cbb0
|
||||
restart: unless-stopped
|
||||
command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=7d"]
|
||||
volumes:
|
||||
- ./deploy/observability/prometheus.yaml:/etc/prometheus/prometheus.yml:ro
|
||||
- prometheus-data:/prometheus
|
||||
read_only: true
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
networks: [knowledge-hub-internal]
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:12.4.0@sha256:b0ae311af06228bcfd4a620504b653db80f5b91e94dc3dc2a5b7dab202bcde20
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: ${KNOWLEDGE_HUB_GRAFANA_USER:-admin}
|
||||
GF_SECURITY_ADMIN_PASSWORD: ${KNOWLEDGE_HUB_GRAFANA_PASSWORD:?Set a private Grafana password for observability}
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
volumes:
|
||||
- ./deploy/observability/grafana-datasource.yaml:/etc/grafana/provisioning/datasources/knowledge-hub.yaml:ro
|
||||
- grafana-data:/var/lib/grafana
|
||||
ports:
|
||||
- "127.0.0.1:${KNOWLEDGE_HUB_GRAFANA_PORT:-13000}:3000"
|
||||
read_only: true
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
networks: [knowledge-hub-internal]
|
||||
|
||||
volumes:
|
||||
prometheus-data:
|
||||
grafana-data:
|
||||
@@ -0,0 +1,8 @@
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Knowledge Hub Prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
@@ -0,0 +1,7 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
scrape_configs:
|
||||
- job_name: itworx-knowledge-hub
|
||||
static_configs:
|
||||
- targets: [otel-collector:8889]
|
||||
@@ -65,6 +65,10 @@ services:
|
||||
KnowledgeHub__Paths__Backups: /var/lib/knowledge-hub/backups
|
||||
KnowledgeHub__Paths__Keys: /var/lib/knowledge-hub/keys
|
||||
Build__Commit: ${BUILD_COMMIT:-development}
|
||||
# Explicit opt-in export to the operator's collector. Empty disables export.
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-}
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf}
|
||||
OTEL_METRIC_EXPORT_INTERVAL: ${OTEL_METRIC_EXPORT_INTERVAL:-30000}
|
||||
TZ: ${TZ:-Europe/Brussels}
|
||||
ports:
|
||||
- "${KNOWLEDGE_HUB_PORT:-1240}:8080"
|
||||
|
||||
@@ -186,6 +186,13 @@ At minimum isolate policies for:
|
||||
|
||||
Return a generic response that does not confirm account/content existence.
|
||||
|
||||
The knowledge and unified-search endpoints share a 60-request/minute budget per
|
||||
effective remote IP. Job creation and the compatibility backup-creation endpoint
|
||||
share a 10-request/minute budget. Both use zero-length queues and return 429.
|
||||
Forwarded addresses are accepted only from configured trusted proxies. Durable
|
||||
job admission additionally enforces per-actor, workspace and rolling daily budgets
|
||||
inside PostgreSQL; see [background jobs](background-jobs.md).
|
||||
|
||||
## Security verification
|
||||
|
||||
Release requires:
|
||||
|
||||
+75
-12
@@ -16,7 +16,8 @@ A full backup set contains:
|
||||
5. Product, schema and container/image versions.
|
||||
6. File inventory with sizes and SHA-256 hashes.
|
||||
7. Backup job metadata and timestamps.
|
||||
8. Encryption metadata when operator-managed encryption is enabled.
|
||||
8. A signed integrity receipt authenticating the exact manifest.
|
||||
9. Encryption metadata when operator-managed encryption is enabled.
|
||||
|
||||
Do not include runtime session secrets or plaintext API tokens.
|
||||
|
||||
@@ -42,13 +43,46 @@ mutable files. Deletions use retention, preventing a blob from disappearing duri
|
||||
5. copy/link/archive required blobs without following unsafe paths;
|
||||
6. create manifest;
|
||||
7. verify dump readability and every file hash;
|
||||
8. optionally encrypt with an operator-supplied mechanism;
|
||||
9. atomically publish completed backup;
|
||||
10. apply retention only after a verified new backup exists;
|
||||
11. record audit and administrator-visible result.
|
||||
8. authenticate the manifest hash with HMAC-SHA-256;
|
||||
9. optionally encrypt with an operator-supplied mechanism;
|
||||
10. atomically publish completed backup;
|
||||
11. apply retention only after a verified new backup exists;
|
||||
12. record audit and administrator-visible result.
|
||||
|
||||
A failed backup never replaces the last known-good backup.
|
||||
|
||||
New backup directories and files use private Unix permissions (0700 and 0600).
|
||||
Permission failures stop creation before publication. On Windows, provision and
|
||||
verify restrictive NTFS ACLs for backups, keys and restore targets before use;
|
||||
Unix mode checks do not enforce Windows ACLs. HMAC authenticates backups but
|
||||
does not encrypt their contents, so archive/share access must remain restricted.
|
||||
|
||||
## Backup authenticity key and recovery
|
||||
|
||||
The `.integrity.json` receipt beside a backup is not trusted by itself. The application
|
||||
authenticates its backup ID, exact manifest hash, completion time and verification result
|
||||
with HMAC-SHA-256. The random 32-byte key is stored at
|
||||
`KnowledgeHub:Paths:Keys/backup-integrity.key`; in the supplied Compose deployment this
|
||||
is `/var/lib/knowledge-hub/keys/backup-integrity.key`. This location is outside
|
||||
`KnowledgeHub:Paths:Backups`, so someone who can rewrite a backup and its receipt cannot
|
||||
create a valid signature. The application creates the key when integrity evidence is
|
||||
first written and restricts it to the service account on Unix hosts.
|
||||
|
||||
Treat this key as essential recovery material. Copy the exact key file into an encrypted,
|
||||
access-controlled escrow location separate from the backup storage, and include it in
|
||||
the operator's disaster-recovery procedure. To restore on a replacement host, recover
|
||||
the key to the configured Keys directory before verifying or restoring backups. Never
|
||||
put the key in a backup directory or in the same backup archive as the signed receipts.
|
||||
Protect access to the escrow copy as carefully as access to the backups.
|
||||
|
||||
If the key is missing or unreadable, backup integrity is reported as unknown and restore
|
||||
is refused. Do not replace a lost key with a newly generated one and expect existing
|
||||
backups to validate: a new key authenticates only backups created after it. Recover the
|
||||
original key from escrow first. If no copy exists, the existing backups cannot be
|
||||
cryptographically authenticated by the application and must not be treated as verified.
|
||||
Backups made before authenticated receipts were introduced also remain unverified; create
|
||||
a new backup with this version before relying on application-verified restore.
|
||||
|
||||
## Retention defaults
|
||||
|
||||
Configurable starting policy:
|
||||
@@ -65,13 +99,26 @@ source.
|
||||
|
||||
Restore is deliberately separate from normal application startup.
|
||||
|
||||
1. Select backup.
|
||||
2. Verify manifest, hashes, version compatibility and encryption.
|
||||
3. Create clean PostgreSQL database and empty target directories.
|
||||
4. Restore database.
|
||||
1. Select backup. Copy read-only media to a writable recovery directory first,
|
||||
preserving the manifest, signed receipts and file bytes. Recover the exact
|
||||
`backup-integrity.key` from protected external escrow into the configured Keys
|
||||
directory; never take it from the backup archive. The application acquires an
|
||||
exclusive storage lease there to prevent retention from removing a restore source,
|
||||
and writes its restore receipt there. Keep the original media unchanged.
|
||||
2. Use the backup's recorded application image to create a clean PostgreSQL
|
||||
database at exactly the manifest's schema version. Create an existing, empty
|
||||
target attachments directory on writable storage with room for the full payload.
|
||||
3. Copy the manifest and every listed database and attachment file into a private
|
||||
staging directory inside the target attachments volume. Verify the HMAC against
|
||||
the staged manifest bytes and verify each staged file's size and SHA-256 before
|
||||
opening or changing the target database. Restore reads only from this verified
|
||||
snapshot, so source-volume changes after verification cannot change restored bytes.
|
||||
4. Restore database. The restore service refuses a target whose latest applied
|
||||
migration differs from the authenticated manifest, before inserting any rows.
|
||||
5. Restore attachment blobs by hash.
|
||||
6. Start the recorded/current compatible application in maintenance mode.
|
||||
7. Apply supported migrations.
|
||||
6. Verify the restored data with the recorded application version.
|
||||
7. Upgrade through supported migrations using the intended newer image, then
|
||||
repeat integrity checks before promotion.
|
||||
8. Run integrity checks:
|
||||
- FK/schema;
|
||||
- user/membership;
|
||||
@@ -87,6 +134,21 @@ Restore is deliberately separate from normal application startup.
|
||||
|
||||
Never test restore by overwriting the only production instance.
|
||||
|
||||
Pass target PostgreSQL credentials through `--target-connection-file=<path>`, which
|
||||
reads the connection string from a protected file. On Unix, the file must be readable
|
||||
only by the restore account (mode `0600` or `0400`); the restore command rejects files
|
||||
with group or other access. On Windows, apply an ACL that grants access only to the
|
||||
restore operator/service account. Do not put the connection string in command-line
|
||||
arguments. Restore errors deliberately omit provider exception details because they
|
||||
can contain connection data.
|
||||
|
||||
The real-backup drill also requires the exact integrity key recovered from protected
|
||||
external escrow. Pass only its file path through
|
||||
`KNOWLEDGE_HUB_BACKUP_INTEGRITY_KEY_FILE`; the script copies the key with mode `0600`
|
||||
into its temporary working directory, uses that directory as the restore Keys path,
|
||||
and removes the temporary copy during cleanup. The key contents are never placed in an
|
||||
argument or printed by the script.
|
||||
|
||||
## Backup encryption
|
||||
|
||||
The application may integrate an operator-supplied encryption command or library after
|
||||
@@ -125,11 +187,12 @@ isolated restore proves it is necessary and the operator has assessed newer writ
|
||||
|
||||
- create backup with representative data;
|
||||
- verify manifest;
|
||||
- reject a forged manifest and matching unsigned/invalid integrity receipt;
|
||||
- corrupt dump and blob tests;
|
||||
- clean-target restore;
|
||||
- compare logical counts and hashes;
|
||||
- restore older supported schema then migrate;
|
||||
- missing key/encrypted backup failure;
|
||||
- missing integrity key or encryption key failure;
|
||||
- interrupted backup cleanup;
|
||||
- retention policy;
|
||||
- no secret inclusion;
|
||||
|
||||
+454
-1
@@ -1,7 +1,7 @@
|
||||
openapi: 3.1.0
|
||||
info:
|
||||
title: ItWorx Knowledge Hub API
|
||||
version: 0.1.0-draft
|
||||
version: 1.0.0-rc.1
|
||||
description: |
|
||||
Versioned API for the ItWorx Knowledge Hub web client and scoped automation.
|
||||
This is not an MCP transport. The future MCP service will consume this API.
|
||||
@@ -29,6 +29,9 @@ tags:
|
||||
- name: Activity
|
||||
- name: Workspace
|
||||
- name: Ingestion proposals
|
||||
- name: Preferences
|
||||
- name: DataSources
|
||||
- name: Jobs
|
||||
security:
|
||||
- cookieAuth: []
|
||||
- bearerAuth: []
|
||||
@@ -777,6 +780,59 @@ paths:
|
||||
$ref: "#/components/schemas/SearchResponse"
|
||||
"401":
|
||||
$ref: "#/components/responses/Unauthorized"
|
||||
/unified-search:
|
||||
get:
|
||||
tags: [Search]
|
||||
operationId: searchWorkspaceContent
|
||||
description: >-
|
||||
Human reader session only. Searches visible knowledge, projects, reviewed
|
||||
services, runbooks and decisions in the current workspace. Trashed content
|
||||
and hidden or archived services are excluded. Counts cover all five kinds;
|
||||
`kind` filters the returned page. A `tag` filters exact, case-insensitive
|
||||
page tags on knowledge, runbook and decision results and excludes projects
|
||||
and services. An empty query returns no results unless `tag` is supplied. The
|
||||
endpoint shares the search rate limit of 60 requests per effective remote IP
|
||||
per minute. An invalid `kind` returns 422.
|
||||
parameters:
|
||||
- name: q
|
||||
in: query
|
||||
schema:
|
||||
type: string
|
||||
maxLength: 500
|
||||
- name: kind
|
||||
in: query
|
||||
schema:
|
||||
type: string
|
||||
enum: [knowledge, projects, services, runbooks, decisions]
|
||||
- name: tag
|
||||
in: query
|
||||
description: Exact, case-insensitive page tag; applies to knowledge, runbooks and decisions.
|
||||
schema:
|
||||
type: string
|
||||
maxLength: 64
|
||||
- $ref: "#/components/parameters/Cursor"
|
||||
- name: limit
|
||||
in: query
|
||||
schema:
|
||||
type: integer
|
||||
minimum: 1
|
||||
maximum: 100
|
||||
default: 25
|
||||
responses:
|
||||
"200":
|
||||
description: Permission-filtered, ranked results and complete grouped counts
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/UnifiedSearchResponse"
|
||||
"401":
|
||||
$ref: "#/components/responses/Unauthorized"
|
||||
"403":
|
||||
$ref: "#/components/responses/Forbidden"
|
||||
"422":
|
||||
description: Invalid kind
|
||||
"429":
|
||||
description: Search request budget exhausted
|
||||
/rag/publication-manifest:
|
||||
get:
|
||||
tags: [Rag]
|
||||
@@ -3077,6 +3133,180 @@ paths:
|
||||
application/problem+json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/ProblemDetails"
|
||||
/me/preferences:
|
||||
get:
|
||||
tags: [Preferences]
|
||||
operationId: getMyPreferences
|
||||
security: [{ cookieAuth: [] }]
|
||||
responses:
|
||||
"200": { description: Persisted result, content: { application/json: { schema: { $ref: "#/components/schemas/UserPreferences" } } } }
|
||||
"401": { $ref: "#/components/responses/Unauthorized" }
|
||||
"403": { $ref: "#/components/responses/Forbidden" }
|
||||
put:
|
||||
tags: [Preferences]
|
||||
operationId: updateMyPreferences
|
||||
security: [{ cookieAuth: [] }]
|
||||
requestBody: { required: true, content: { application/json: { schema: { $ref: "#/components/schemas/UserPreferencesRequest" } } } }
|
||||
responses:
|
||||
"200": { description: Persisted result, content: { application/json: { schema: { $ref: "#/components/schemas/UserPreferences" } } } }
|
||||
"401": { $ref: "#/components/responses/Unauthorized" }
|
||||
"403": { $ref: "#/components/responses/Forbidden" }
|
||||
"412": { $ref: "#/components/responses/PreconditionFailed" }
|
||||
"428": { $ref: "#/components/responses/PreconditionRequired" }
|
||||
/admin/platform-settings:
|
||||
get:
|
||||
tags: [Admin]
|
||||
operationId: getPlatformSettings
|
||||
security: [{ cookieAuth: [] }]
|
||||
responses:
|
||||
"200": { description: Persisted result, content: { application/json: { schema: { $ref: "#/components/schemas/PlatformSettings" } } } }
|
||||
"401": { $ref: "#/components/responses/Unauthorized" }
|
||||
"403": { $ref: "#/components/responses/Forbidden" }
|
||||
put:
|
||||
tags: [Admin]
|
||||
operationId: updatePlatformSettings
|
||||
security: [{ cookieAuth: [] }]
|
||||
requestBody: { required: true, content: { application/json: { schema: { $ref: "#/components/schemas/PlatformSettingsRequest" } } } }
|
||||
responses:
|
||||
"200": { description: Persisted result, content: { application/json: { schema: { $ref: "#/components/schemas/PlatformSettings" } } } }
|
||||
"401": { $ref: "#/components/responses/Unauthorized" }
|
||||
"403": { $ref: "#/components/responses/Forbidden" }
|
||||
"412": { $ref: "#/components/responses/PreconditionFailed" }
|
||||
"428": { $ref: "#/components/responses/PreconditionRequired" }
|
||||
/admin/data-sources:
|
||||
get:
|
||||
tags: [DataSources]
|
||||
operationId: listDataSources
|
||||
security: [{ cookieAuth: [] }]
|
||||
responses:
|
||||
"200": { description: Persisted result, content: { application/json: { schema: { $ref: "#/components/schemas/DataSourceList" } } } }
|
||||
"401": { $ref: "#/components/responses/Unauthorized" }
|
||||
"403": { $ref: "#/components/responses/Forbidden" }
|
||||
post:
|
||||
tags: [DataSources]
|
||||
operationId: registerDataSource
|
||||
security: [{ cookieAuth: [] }]
|
||||
requestBody: { required: true, content: { application/json: { schema: { $ref: "#/components/schemas/DataSourceRegistration" } } } }
|
||||
responses:
|
||||
"200": { description: Persisted result, content: { application/json: { schema: { $ref: "#/components/schemas/DataSource" } } } }
|
||||
"401": { $ref: "#/components/responses/Unauthorized" }
|
||||
"403": { $ref: "#/components/responses/Forbidden" }
|
||||
/admin/data-sources/{id}:
|
||||
put:
|
||||
tags: [DataSources]
|
||||
operationId: configureDataSource
|
||||
security: [{ cookieAuth: [] }]
|
||||
parameters: [{ name: id, in: path, required: true, schema: { type: string, format: uuid } }]
|
||||
requestBody: { required: true, content: { application/json: { schema: { $ref: "#/components/schemas/DataSourceConfiguration" } } } }
|
||||
responses:
|
||||
"200": { description: Persisted result, content: { application/json: { schema: { $ref: "#/components/schemas/DataSource" } } } }
|
||||
"401": { $ref: "#/components/responses/Unauthorized" }
|
||||
"403": { $ref: "#/components/responses/Forbidden" }
|
||||
"412": { $ref: "#/components/responses/PreconditionFailed" }
|
||||
"428": { $ref: "#/components/responses/PreconditionRequired" }
|
||||
/admin/data-sources/{id}/observations:
|
||||
post:
|
||||
tags: [DataSources]
|
||||
operationId: reportDataSourceObservation
|
||||
security: [{ cookieAuth: [] }]
|
||||
parameters: [{ name: id, in: path, required: true, schema: { type: string, format: uuid } }]
|
||||
requestBody: { required: true, content: { application/json: { schema: { $ref: "#/components/schemas/DataSourceObservation" } } } }
|
||||
responses:
|
||||
"200": { description: Persisted result, content: { application/json: { schema: { $ref: "#/components/schemas/DataSource" } } } }
|
||||
"401": { $ref: "#/components/responses/Unauthorized" }
|
||||
"403": { $ref: "#/components/responses/Forbidden" }
|
||||
/admin/backups/status:
|
||||
get:
|
||||
tags: [Backups]
|
||||
operationId: getBackupStatus
|
||||
security: [{ cookieAuth: [] }]
|
||||
responses:
|
||||
"200": { description: Persisted result, content: { application/json: { schema: { $ref: "#/components/schemas/BackupStatus" } } } }
|
||||
"401": { $ref: "#/components/responses/Unauthorized" }
|
||||
"403": { $ref: "#/components/responses/Forbidden" }
|
||||
/admin/backups/{backupId}/protection:
|
||||
put:
|
||||
tags: [Backups]
|
||||
operationId: setBackupProtection
|
||||
security: [{ cookieAuth: [] }]
|
||||
parameters: [{ name: backupId, in: path, required: true, schema: { type: string } }]
|
||||
requestBody: { required: true, content: { application/json: { schema: { $ref: "#/components/schemas/BackupProtectionRequest" } } } }
|
||||
responses:
|
||||
"200": { description: Persisted result, content: { application/json: { schema: { $ref: "#/components/schemas/BackupEvidence" } } } }
|
||||
"401": { $ref: "#/components/responses/Unauthorized" }
|
||||
"403": { $ref: "#/components/responses/Forbidden" }
|
||||
"412": { $ref: "#/components/responses/PreconditionFailed" }
|
||||
"428": { $ref: "#/components/responses/PreconditionRequired" }
|
||||
/admin/audit-log:
|
||||
get:
|
||||
tags: [Admin]
|
||||
operationId: listAdminAuditLog
|
||||
security: [{ cookieAuth: [] }]
|
||||
responses:
|
||||
"200": { description: Persisted result, content: { application/json: { schema: { $ref: "#/components/schemas/AdminAuditList" } } } }
|
||||
"401": { $ref: "#/components/responses/Unauthorized" }
|
||||
"403": { $ref: "#/components/responses/Forbidden" }
|
||||
/jobs:
|
||||
get:
|
||||
tags: [Jobs]
|
||||
operationId: listBackgroundJobs
|
||||
security: [{ cookieAuth: [] }]
|
||||
responses:
|
||||
"200": { description: Persisted result, content: { application/json: { schema: { $ref: "#/components/schemas/BackgroundJobList" } } } }
|
||||
"401": { $ref: "#/components/responses/Unauthorized" }
|
||||
"403": { $ref: "#/components/responses/Forbidden" }
|
||||
/jobs/course-tagging:
|
||||
post:
|
||||
tags: [Jobs]
|
||||
operationId: queueCourseTaggingJob
|
||||
security: [{ cookieAuth: [] }]
|
||||
parameters: [{ $ref: "#/components/parameters/RequiredIdempotencyKey" }]
|
||||
requestBody: { required: true, content: { application/json: { schema: { $ref: "#/components/schemas/QueueCourseTaggingJobRequest" } } } }
|
||||
responses:
|
||||
"202": { description: Persisted result, content: { application/json: { schema: { $ref: "#/components/schemas/BackgroundJob" } } } }
|
||||
"401": { $ref: "#/components/responses/Unauthorized" }
|
||||
"403": { $ref: "#/components/responses/Forbidden" }
|
||||
/jobs/backups:
|
||||
post:
|
||||
tags: [Jobs]
|
||||
operationId: queueBackupJob
|
||||
security: [{ cookieAuth: [] }]
|
||||
parameters: [{ $ref: "#/components/parameters/RequiredIdempotencyKey" }]
|
||||
requestBody: { required: true, content: { application/json: { schema: { $ref: "#/components/schemas/QueueBackupJobRequest" } } } }
|
||||
responses:
|
||||
"202": { description: Persisted result, content: { application/json: { schema: { $ref: "#/components/schemas/BackgroundJob" } } } }
|
||||
"401": { $ref: "#/components/responses/Unauthorized" }
|
||||
"403": { $ref: "#/components/responses/Forbidden" }
|
||||
/jobs/{jobId}:
|
||||
get:
|
||||
tags: [Jobs]
|
||||
operationId: getBackgroundJob
|
||||
security: [{ cookieAuth: [] }]
|
||||
parameters: [{ name: jobId, in: path, required: true, schema: { type: string, format: uuid } }]
|
||||
responses:
|
||||
"200": { description: Persisted result, content: { application/json: { schema: { $ref: "#/components/schemas/BackgroundJob" } } } }
|
||||
"401": { $ref: "#/components/responses/Unauthorized" }
|
||||
"403": { $ref: "#/components/responses/Forbidden" }
|
||||
/jobs/{jobId}/cancel:
|
||||
post:
|
||||
tags: [Jobs]
|
||||
operationId: cancelBackgroundJob
|
||||
security: [{ cookieAuth: [] }]
|
||||
parameters: [{ name: jobId, in: path, required: true, schema: { type: string, format: uuid } }]
|
||||
responses:
|
||||
"200": { description: Persisted result, content: { application/json: { schema: { $ref: "#/components/schemas/BackgroundJob" } } } }
|
||||
"401": { $ref: "#/components/responses/Unauthorized" }
|
||||
"403": { $ref: "#/components/responses/Forbidden" }
|
||||
/jobs/{jobId}/events:
|
||||
get:
|
||||
tags: [Jobs]
|
||||
operationId: streamBackgroundJobEvents
|
||||
security: [{ cookieAuth: [] }]
|
||||
parameters: [{ name: jobId, in: path, required: true, schema: { type: string, format: uuid } }]
|
||||
responses:
|
||||
"200": { description: Job and progress Server-Sent Events with heartbeat and terminal close, content: { text/event-stream: { schema: { type: string } } } }
|
||||
"401": { $ref: "#/components/responses/Unauthorized" }
|
||||
"403": { $ref: "#/components/responses/Forbidden" }
|
||||
components:
|
||||
securitySchemes:
|
||||
cookieAuth:
|
||||
@@ -3248,6 +3478,194 @@ components:
|
||||
schema:
|
||||
$ref: "#/components/schemas/ProblemDetails"
|
||||
schemas:
|
||||
UserPreferencesRequest:
|
||||
type: object
|
||||
required: [theme, density, primaryNavigationCollapsed, contextPanelOpen, navigationOrder, dashboardWidgetOrder, hiddenDashboardWidgets, defaultLandingPage, catalogView, contextPanelWidth, navigationWidth]
|
||||
properties:
|
||||
theme: { type: string, enum: [system, light, dark] }
|
||||
density: { type: string, enum: [comfortable, compact] }
|
||||
primaryNavigationCollapsed: { type: boolean }
|
||||
contextPanelOpen: { type: boolean }
|
||||
navigationOrder: { type: array, items: { type: string } }
|
||||
dashboardWidgetOrder: { type: array, items: { type: string } }
|
||||
hiddenDashboardWidgets: { type: array, items: { type: string } }
|
||||
defaultLandingPage: { type: string }
|
||||
catalogView: { type: string, enum: [grid, list] }
|
||||
contextPanelWidth: { type: integer, minimum: 256, maximum: 440 }
|
||||
navigationWidth: { type: integer, minimum: 192, maximum: 320 }
|
||||
UserPreferences:
|
||||
allOf:
|
||||
- $ref: "#/components/schemas/UserPreferencesRequest"
|
||||
- type: object
|
||||
required: [schemaVersion, eTag, isDefault, updatedAt]
|
||||
properties:
|
||||
schemaVersion: { type: integer, enum: [1] }
|
||||
eTag: { type: string }
|
||||
isDefault: { type: boolean }
|
||||
updatedAt: { type: [string, "null"], format: date-time }
|
||||
PlatformSettingsRequest:
|
||||
type: object
|
||||
required: [displayName, primaryLocale, timeZone, ingestionProposals, backupPolicyExpectation, inventoryFreshnessPolicy]
|
||||
properties:
|
||||
displayName: { type: string, minLength: 1, maxLength: 200 }
|
||||
primaryLocale: { type: string }
|
||||
timeZone: { type: string }
|
||||
ingestionProposals: { type: object, required: [enabled, maximumSourceCount, maximumSourceBytes, failedRetryLimit], properties: { enabled: { type: boolean }, maximumSourceCount: { type: integer, minimum: 1, maximum: 20 }, maximumSourceBytes: { type: integer, format: int64 }, failedRetryLimit: { type: integer, minimum: 1, maximum: 3 } } }
|
||||
backupPolicyExpectation: { type: object, required: [expectedIntervalHours, scheduleDescription, dailyRetentionCount, weeklyRetentionCount, monthlyRetentionCount, protectedBackupCount, restoreVerificationMaxAgeDays], properties: { expectedIntervalHours: { type: integer }, scheduleDescription: { type: string }, dailyRetentionCount: { type: integer }, weeklyRetentionCount: { type: integer }, monthlyRetentionCount: { type: integer }, protectedBackupCount: { type: integer }, restoreVerificationMaxAgeDays: { type: integer } } }
|
||||
inventoryFreshnessPolicy: { type: object, required: [warningAfterHours], properties: { warningAfterHours: { type: integer } } }
|
||||
PlatformSettings:
|
||||
allOf:
|
||||
- $ref: "#/components/schemas/PlatformSettingsRequest"
|
||||
- type: object
|
||||
required: [schemaVersion, eTag, isDefault, updatedAt]
|
||||
properties:
|
||||
schemaVersion: { type: integer, enum: [1] }
|
||||
eTag: { type: string }
|
||||
isDefault: { type: boolean }
|
||||
updatedAt: { type: [string, "null"], format: date-time }
|
||||
backupPolicyExpectation: { type: object, description: Includes enforcement External; configured schedule is an expectation, not a scheduler run. }
|
||||
DataSourceRegistration:
|
||||
type: object
|
||||
required: [sourceKey, displayName, kind, expectedIntervalHours]
|
||||
properties:
|
||||
sourceKey: { type: string, maxLength: 200 }
|
||||
displayName: { type: string, maxLength: 120 }
|
||||
kind: { type: string, enum: [Inventory, Gitea, KnowledgeClient, ManualIntake, LeafWiki, CuratedPackage] }
|
||||
expectedIntervalHours: { type: integer, minimum: 1, maximum: 8760 }
|
||||
DataSourceConfiguration:
|
||||
type: object
|
||||
required: [displayName, enabled, expectedIntervalHours]
|
||||
properties:
|
||||
displayName: { type: string, maxLength: 120 }
|
||||
enabled: { type: boolean }
|
||||
expectedIntervalHours: { type: integer, minimum: 1, maximum: 8760 }
|
||||
DataSourceObservation:
|
||||
type: object
|
||||
required: [success]
|
||||
properties:
|
||||
success: { type: boolean }
|
||||
recordCount: { type: [integer, "null"], format: int64 }
|
||||
errorCode: { type: [string, "null"] }
|
||||
correlationId: { type: [string, "null"], maxLength: 100 }
|
||||
DataSource:
|
||||
type: object
|
||||
required: [id, sourceKey, displayName, kind, enabled, state, configurationState, syncAuthority, expectedIntervalHours, eTag, actions]
|
||||
properties:
|
||||
id: { type: string, format: uuid }
|
||||
sourceKey: { type: string }
|
||||
displayName: { type: string }
|
||||
kind: { type: string }
|
||||
enabled: { type: boolean }
|
||||
state: { type: string, enum: [Unknown, Current, Stale, Failed, Disabled] }
|
||||
configurationState: { type: string }
|
||||
syncAuthority: { type: string }
|
||||
expectedIntervalHours: { type: integer }
|
||||
lastAttemptAt: { type: [string, "null"], format: date-time }
|
||||
lastSuccessAt: { type: [string, "null"], format: date-time }
|
||||
lastFailureAt: { type: [string, "null"], format: date-time }
|
||||
dataObservedAt: { type: [string, "null"], format: date-time }
|
||||
recordCount: { type: [integer, "null"], format: int64 }
|
||||
lastErrorCode: { type: [string, "null"] }
|
||||
lastCorrelationId: { type: [string, "null"] }
|
||||
eTag: { type: string }
|
||||
actions: { type: array, items: { type: string } }
|
||||
DataSourceList:
|
||||
type: object
|
||||
required: [items, nextCursor]
|
||||
properties:
|
||||
items: { type: array, items: { $ref: "#/components/schemas/DataSource" } }
|
||||
nextCursor: { type: [string, "null"] }
|
||||
BackupProtectionRequest:
|
||||
type: object
|
||||
required: [protected]
|
||||
properties:
|
||||
protected: { type: boolean }
|
||||
BackupEvidence:
|
||||
type: object
|
||||
required: [protected, protectionVersion]
|
||||
properties:
|
||||
protected: { type: boolean }
|
||||
protectionVersion: { type: integer }
|
||||
integrityVerifiedAt: { type: [string, "null"], format: date-time }
|
||||
restoreVerifiedAt: { type: [string, "null"], format: date-time }
|
||||
lastRestoreSucceeded: { type: [boolean, "null"] }
|
||||
BackupStatus:
|
||||
type: object
|
||||
required: [state, scheduleAuthority, scheduleDescription, restoreConfidence, storedBackupBytes, protectedCount, expectedIntervalHours, applicationRetentionCount, dailyRetentionExpectation, weeklyRetentionExpectation, monthlyRetentionExpectation, restoreVerificationMaxAgeDays]
|
||||
properties:
|
||||
state: { type: string, enum: [Unknown, Current, Stale] }
|
||||
scheduleAuthority: { type: string, enum: [External] }
|
||||
scheduleDescription: { type: string }
|
||||
nextScheduledRun: { type: [string, "null"], format: date-time }
|
||||
nextExpectedBy: { type: [string, "null"], format: date-time }
|
||||
lastSuccessfulBackupAt: { type: [string, "null"], format: date-time }
|
||||
lastIntegrityVerifiedAt: { type: [string, "null"], format: date-time }
|
||||
lastVerifiedRestoreAt: { type: [string, "null"], format: date-time }
|
||||
restoreConfidence: { type: string, enum: [Unknown, Stale, Verified] }
|
||||
storedBackupBytes: { type: integer, format: int64, description: Sum of stored backup manifest sizes; not filesystem capacity. }
|
||||
protectedCount: { type: integer }
|
||||
expectedIntervalHours: { type: integer }
|
||||
applicationRetentionCount: { type: integer }
|
||||
dailyRetentionExpectation: { type: integer }
|
||||
weeklyRetentionExpectation: { type: integer }
|
||||
monthlyRetentionExpectation: { type: integer }
|
||||
restoreVerificationMaxAgeDays: { type: integer }
|
||||
AdminAuditList:
|
||||
type: object
|
||||
required: [items, nextCursor]
|
||||
properties:
|
||||
items: { type: array, items: { $ref: "#/components/schemas/AdminAuditItem" } }
|
||||
nextCursor: { type: [integer, "null"], format: int64 }
|
||||
AdminAuditItem:
|
||||
type: object
|
||||
required: [id, actorKind, source, action, summary, occurredAt]
|
||||
properties:
|
||||
id: { type: integer, format: int64 }
|
||||
actorKind: { type: string }
|
||||
actorId: { type: [string, "null"], format: uuid }
|
||||
source: { type: string }
|
||||
action: { type: string }
|
||||
targetType: { type: [string, "null"] }
|
||||
targetId: { type: [string, "null"] }
|
||||
summary: { type: string }
|
||||
occurredAt: { type: string, format: date-time }
|
||||
correlationId: { type: [string, "null"] }
|
||||
QueueCourseTaggingJobRequest:
|
||||
type: object
|
||||
required: [rootPageId, tag]
|
||||
properties:
|
||||
rootPageId: { type: string, format: uuid }
|
||||
tag: { type: string }
|
||||
QueueBackupJobRequest: { type: object, description: Empty object; the job snapshots the current server-side backup policy. }
|
||||
BackgroundJob:
|
||||
type: object
|
||||
required: [id, workspaceId, actorId, type, status, createdAt, cancelRequested, progress, total, processed, succeeded, skipped, failed, errors, version]
|
||||
properties:
|
||||
id: { type: string, format: uuid }
|
||||
workspaceId: { type: string, format: uuid }
|
||||
actorId: { type: string, format: uuid }
|
||||
type: { type: string }
|
||||
status: { type: string, enum: [queued, running, succeeded, partially_succeeded, failed, cancelled] }
|
||||
createdAt: { type: string, format: date-time }
|
||||
startedAt: { type: [string, "null"], format: date-time }
|
||||
completedAt: { type: [string, "null"], format: date-time }
|
||||
cancelRequestedAt: { type: [string, "null"], format: date-time }
|
||||
cancelRequested: { type: boolean }
|
||||
progress: { type: integer, minimum: 0, maximum: 100 }
|
||||
total: { type: integer }
|
||||
processed: { type: integer }
|
||||
succeeded: { type: integer }
|
||||
skipped: { type: integer }
|
||||
failed: { type: integer }
|
||||
errors: { type: array, items: { type: object, properties: { code: { type: string }, message: { type: string }, itemId: { type: [string, "null"], format: uuid } } } }
|
||||
result: { type: [object, "null"] }
|
||||
correlationId: { type: [string, "null"] }
|
||||
version: { type: integer, format: int64 }
|
||||
BackgroundJobList:
|
||||
type: object
|
||||
required: [items]
|
||||
properties:
|
||||
items: { type: array, items: { $ref: "#/components/schemas/BackgroundJob" } }
|
||||
CreateIngestionProposalRequest:
|
||||
type: object
|
||||
required: [manifest]
|
||||
@@ -5243,6 +5661,41 @@ components:
|
||||
$ref: "#/components/schemas/SearchResultItem"
|
||||
nextCursor:
|
||||
type: [string, "null"]
|
||||
UnifiedSearchResponse:
|
||||
type: object
|
||||
required: [items, counts, nextCursor]
|
||||
properties:
|
||||
items:
|
||||
type: array
|
||||
items:
|
||||
$ref: "#/components/schemas/UnifiedSearchResultItem"
|
||||
counts:
|
||||
$ref: "#/components/schemas/UnifiedSearchCounts"
|
||||
nextCursor:
|
||||
type: [string, "null"]
|
||||
UnifiedSearchCounts:
|
||||
type: object
|
||||
required: [knowledge, projects, services, runbooks, decisions, total]
|
||||
properties:
|
||||
knowledge: { type: integer, minimum: 0 }
|
||||
projects: { type: integer, minimum: 0 }
|
||||
services: { type: integer, minimum: 0 }
|
||||
runbooks: { type: integer, minimum: 0 }
|
||||
decisions: { type: integer, minimum: 0 }
|
||||
total: { type: integer, minimum: 0 }
|
||||
UnifiedSearchResultItem:
|
||||
type: object
|
||||
required: [kind, id, title, summary, snippet, updatedAt, href]
|
||||
properties:
|
||||
kind:
|
||||
type: string
|
||||
enum: [knowledge, projects, services, runbooks, decisions]
|
||||
id: { type: string, format: uuid }
|
||||
title: { type: string }
|
||||
summary: { type: string }
|
||||
snippet: { type: string }
|
||||
updatedAt: { type: string, format: date-time }
|
||||
href: { type: string, description: Same-origin relative route to the result. }
|
||||
RagPublicationManifestItem:
|
||||
type: object
|
||||
description: >-
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
# Durable background jobs
|
||||
|
||||
Course tagging and manual backup creation run as PostgreSQL-backed jobs. The admin
|
||||
workspace shows persisted progress and supports cancellation. SSE sends the same
|
||||
safe snapshot as the polling endpoint and closes on a terminal state; the browser
|
||||
falls back to polling after an interrupted stream.
|
||||
|
||||
Only an enabled workspace administrator can create or execute a job. Mutation
|
||||
endpoints require cookie authentication and CSRF validation. The worker rechecks
|
||||
the actor's membership before work and before each course page. An idempotency key
|
||||
is scoped to the workspace and actor; reusing it with a different payload returns
|
||||
409. Retrying the same payload returns the existing job, including at capacity.
|
||||
The job row and its `job.queued` audit event commit together. Cancellation requests
|
||||
and their `job.cancel_requested` audit event also commit together; repeating a
|
||||
running-job cancellation returns the current request without adding another audit
|
||||
event.
|
||||
|
||||
The database serializes admission per workspace. Defaults under `KnowledgeHub:Jobs`
|
||||
are `MaxPendingPerWorkspace=10`, `MaxPendingPerActor=3`, and
|
||||
`MaxCreatedPerDay=100` (a rolling 24-hour window). Capacity rejection returns 429
|
||||
before inserting a job or page snapshot. HTTP creation also has a shared budget
|
||||
of ten requests per effective remote IP per minute. Only configured trusted
|
||||
proxies can affect that address. These limits bound distinct-key floods across
|
||||
application restarts and replicas as well as ordinary bursts.
|
||||
|
||||
Course tagging snapshots at most `MaxCoursePages=100000` page IDs in PostgreSQL.
|
||||
The worker updates one current page at a time with its ETag, preserves content,
|
||||
skips existing tags, and records safe per-page failures. A lease is renewed while
|
||||
work is active; expired leases can be reclaimed with bounded retries. Default
|
||||
worker concurrency is two. Cancellation stops future items and preserves the
|
||||
already completed outcomes.
|
||||
|
||||
Terminal jobs retain their counters, bounded errors, result, actor and audit trail.
|
||||
Their page-ID recovery snapshots are removed in the same transaction as the final
|
||||
state. This limits large item-table growth without discarding the job history.
|
||||
The daily admission budget bounds growth of that compact history. Source data is
|
||||
never removed by this maintenance.
|
||||
|
||||
The job `Version` is a monotonic optimistic-concurrency token for persisted worker
|
||||
updates and SSE event IDs. If a cancellation update wins a race with a tracked worker
|
||||
save, the worker retries against the current version while keeping the cancellation
|
||||
request. SSE always emits a terminal event and closes when it observes a terminal
|
||||
job, including when that snapshot has the same version as the last progress event.
|
||||
@@ -65,20 +65,46 @@ af.
|
||||
|
||||
### Stap 2 — herstellen
|
||||
|
||||
Maak eerst een **beschrijfbare herstelkopie** van de gekozen back-up; herstel neemt een
|
||||
exclusieve opslaglease en schrijft achteraf een herstelbewijs naast het manifest. Geef
|
||||
de CLI de doelconnectie via een door de operator beheerd geheimenbestand, niet via een
|
||||
argument met het wachtwoord erin. Op Unix moet dat bestand alleen voor de restoregebruiker
|
||||
leesbaar zijn (`chmod 600` of `chmod 400`); op Windows beperk je de ACL tot de restore-
|
||||
operator/serviceaccount. Het bestand bevat één PostgreSQL-connectiereeks. De
|
||||
integriteitssleutel moet eveneens beschikbaar zijn in de geconfigureerde Keys-map.
|
||||
Haal de exacte `backup-integrity.key` uit beschermde externe escrow; herstel vertrouwt
|
||||
geen sleutel die alleen in de back-upkopie zit. De restore verifieert de manifest-
|
||||
handtekening en kopieert de vermelde bestanden eerst naar een privé-stagingmap binnen
|
||||
het lege doelbijlagenvolume. Reserveer daar vrije ruimte voor ten minste de volledige
|
||||
back-upinhoud. De restore gebruikt uitsluitend die geverifieerde snapshot.
|
||||
|
||||
```bash
|
||||
docker run --rm \
|
||||
--network itworx-knowledge-hub_knowledge-hub-internal \
|
||||
-v knowledge-hub-backups:/var/lib/knowledge-hub/backups:ro \
|
||||
-v <beschrijfbare-herstelkopie>:/var/lib/knowledge-hub/backups:rw \
|
||||
-e KnowledgeHub__Paths__Backups=/var/lib/knowledge-hub/backups \
|
||||
-v <map-met-backup-integriteitssleutel>:/var/lib/knowledge-hub/keys:ro \
|
||||
-e KnowledgeHub__Paths__Keys=/var/lib/knowledge-hub/keys \
|
||||
-v <beveiligd-doelconnectiebestand>:/run/secrets/restore-target-connection:ro \
|
||||
-v <doel-bijlagen-volume>:/var/lib/knowledge-hub/attachments \
|
||||
itworx/knowledge-hub:local \
|
||||
dotnet ItWorx.KnowledgeHub.Api.dll \
|
||||
--restore \
|
||||
--backup-id=<back-up-id> \
|
||||
--target-connection="Host=<doel-db-host>;Port=5432;Database=<doel-db>;Username=<gebruiker>;Password=<wachtwoord>" \
|
||||
--target-connection-file=/run/secrets/restore-target-connection \
|
||||
--target-attachments=/var/lib/knowledge-hub/attachments
|
||||
```
|
||||
|
||||
De echte herstel-drill haalt de back-up via SSH op, maar leest de integriteitssleutel
|
||||
uit een lokaal door de operator beheerd escrowbestand. Geef alleen het bestandspad mee;
|
||||
de drill maakt tijdens uitvoering een tijdelijke kopie met modus `0600` en verwijdert
|
||||
die bij opruimen:
|
||||
|
||||
```bash
|
||||
KNOWLEDGE_HUB_BACKUP_INTEGRITY_KEY_FILE=/secure/escrow/backup-integrity.key \
|
||||
bash scripts/v2-r4-real-backup-restore-drill.sh <back-up-id>
|
||||
```
|
||||
|
||||
De doeldatabase moet **leeg** zijn — herstel weigert expliciet als er al rijen in
|
||||
staan, om nooit per ongeluk bestaande gegevens te vermengen met herstelde gegevens.
|
||||
|
||||
@@ -86,7 +112,8 @@ Het proces rapporteert per tabel het aantal herstelde versus verwachte rijen, he
|
||||
aantal herstelde bijlagen, en sluit af met `GESLAAGD` of `MISLUKT`
|
||||
(afsluitcode 0 respectievelijk 1). Een `MISLUKT`-resultaat laat expliciet zien welke
|
||||
tabel niet overeenkomt — er wordt nooit stilzwijgend een onvolledig herstel als
|
||||
geslaagd gerapporteerd.
|
||||
geslaagd gerapporteerd. Providerfoutdetails worden niet afgedrukt omdat die
|
||||
connectiegegevens kunnen bevatten.
|
||||
|
||||
## Opstartgedrag en migraties
|
||||
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
# Operationele metrieken
|
||||
|
||||
De applicatie exporteert alleen naar OTLP als `OTEL_EXPORTER_OTLP_ENDPOINT` expliciet is gezet. De optionele Compose-overlay `deploy/observability/compose.yaml` schakelt een Collector, Prometheus en Grafana in op een intern Docker-netwerk. Grafana bindt uitsluitend aan de loopback-interface van de host; kies een privaat beheerderswachtwoord via `KNOWLEDGE_HUB_GRAFANA_PASSWORD` buiten Git. Start met beide Compose-bestanden in hetzelfde project nadat de configuratie en opslaglocaties zijn gecontroleerd.
|
||||
|
||||
De Collector neemt OTLP/HTTP op poort 4318 aan en biedt de metrieken aan Prometheus op 8889. Grafana leest Prometheus als geprovisioneerde gegevensbron. De applicatie heeft vaste labels voor operatie, uitkomst, jobstatus en bewijstype. SQL, paginatitels, actoren, zoektermen, tokens en bron-URL's zijn geen metrieklabels.
|
||||
|
||||
De overlay gebruikt vastgelegde image-digests. De collector-default is de
|
||||
beschikbare amd64-build van 0.162.0 voor de Unraid-host. Kies op een andere
|
||||
architectuur expliciet een geverifieerde image via `KNOWLEDGE_HUB_COLLECTOR_IMAGE`.
|
||||
De ongekwalificeerde tag `0.162.0` was tijdens de controle niet beschikbaar.
|
||||
|
||||
Gebruik in Prometheus onder meer `platform_operations_total`, `platform_operation_duration_seconds`, `platform_jobs`, `platform_queue_age_seconds` en `platform_evidence_age_seconds`. Een ontbrekende meting betekent onbekend: na een databasefout worden oudere snapshots uit de gauges verwijderd. De ouderdom van backups telt pas als het nieuwste opgeslagen manifest een geldig integriteitsbewijs heeft. De `inventory`-ouderdom is gebaseerd op de capturetijd van de nieuwste snapshot, niet op de latere importtijd. De applicatie toont het beleid voor externe backups als verwachting, zonder een planner-run te verzinnen.
|
||||
|
||||
Controleer na start een echte taak of zoekactie, vraag dan binnen een export- en scrape-interval `platform_operations_total` op via de Prometheus API en controleer de tijdreeks ook in Grafana. Een lege tijdreeks is geen bewijs van een gezonde pipeline. De geïsoleerde geautomatiseerde OTLP-test verstuurt een echte protobuf-aanvraag naar een lokale receiver en controleert dat private labels ontbreken; de volledige Collector → Prometheus → Grafana-pipeline moet in de beoogde omgeving apart worden geverifieerd.
|
||||
|
||||
Bronnen voor de gebruikte exporterconfiguratie: [OpenTelemetry Collector Prometheus exporter](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/exporter/prometheusexporter/README.md) en [OTLP exporter configuratie](https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/).
|
||||
@@ -0,0 +1,60 @@
|
||||
# Voorkeuren en platformbeheer
|
||||
|
||||
## Persoonlijke instellingen
|
||||
|
||||
Op `/settings` kies je thema, datadichtheid, panelen, landingspagina en de volgorde
|
||||
van navigatie en dashboardwidgets. Aangemelde gebruikers bewaren deze keuzes bij
|
||||
hun account. Het scherm toont laden, opslaan, gesynchroniseerd, offline of een
|
||||
versieconflict. Bij een conflict kun je de servervoorkeuren laden of je lokale
|
||||
keuze opnieuw aanbieden. Gebruik de verplaatsknoppen met toetsenbord of touch;
|
||||
slepen is een extra mogelijkheid.
|
||||
|
||||
## Beheerwerkruimten
|
||||
|
||||
`/admin` bevat het overzicht. De afzonderlijke onderdelen behandelen gebruikers,
|
||||
toegang, integraties, inventaris, kennis, classificaties, achtergrondtaken,
|
||||
back-ups, audit, platforminstellingen en systeeminformatie. Alleen bevoegde
|
||||
beheerders hebben toegang. Een verborgen navigatielink vervangt geen API-controle.
|
||||
|
||||
Op `/admin/platform` bewaar je weergavenaam, locale, tijdzone, intakebeleid,
|
||||
inventarisouderdom en verwachtingen voor externe backups. Verouderde formulieren
|
||||
kunnen geen nieuwere serverversie overschrijven. Laad bij een conflict de actuele
|
||||
waarden opnieuw voordat je wijzigingen opnieuw toepast.
|
||||
|
||||
De backupplanning en dagelijkse, wekelijkse en maandelijkse bewaaraantallen zijn
|
||||
verwachtingen over operatorbeheer. Dit scherm start geen externe planner en voert
|
||||
geen externe retentie uit. Handmatige backups verschijnen als duurzame taken op
|
||||
`/admin/jobs`; de lokale backupdienst hanteert daarnaast haar geconfigureerde
|
||||
retentie en respecteert beschermde backups. Zie [backup en herstel](18-backup-restore.md).
|
||||
|
||||
## Externe bronnen
|
||||
|
||||
Op `/admin/integrations` registreer je een unieke broncode, type, naam en verwacht
|
||||
synchronisatie-interval. Inventarisbronnen worden automatisch geregistreerd bij
|
||||
een geslaagde synchronisatie. Configuratie en waarnemingen zijn afzonderlijk:
|
||||
een ingeschakelde bron is nog geen bewijs dat de laatste run geslaagd is.
|
||||
|
||||
De broncode `inventory:default` is gereserveerd voor de inventarisimporter en kan
|
||||
niet handmatig worden geregistreerd. Uitschakelen van deze bron blokkeert volgende
|
||||
imports. De importer controleert en vergrendelt de bron binnen dezelfde transactie
|
||||
als de inventariswijziging: een eerder bevestigde uitschakeling wordt ook door een
|
||||
reeds gestarte aanvraag gerespecteerd.
|
||||
|
||||
Een operator kan een geslaagde of mislukte externe run rapporteren met een
|
||||
bekende recordtelling en correlatie-ID. Een onbekend aantal blijft leeg. Zo'n
|
||||
rapportage is geen uitgevoerde verbindingstest. De hub voert externe synchronisatie
|
||||
niet zelf uit en bewaart hier geen bronwachtwoorden. De interface toont de laatste
|
||||
poging, laatste geslaagde poging en het tijdstip waarop gegevens zijn waargenomen.
|
||||
|
||||
## Taakvoortgang en operationeel bewijs
|
||||
|
||||
Een cursustag- of back-uptaak blijft op de server bestaan als je het tabblad sluit.
|
||||
De detail-URL bevat het taak-ID. Annuleren stopt toekomstig werk zodra dat veilig
|
||||
kan; reeds afgeronde pagina-updates blijven bewaard. Fouten en aantallen komen
|
||||
uit opgeslagen serverstatus. Zie [achtergrondtaken](background-jobs.md).
|
||||
|
||||
Systeem- en deploymentinformatie tonen uitsluitend gemelde versie- en
|
||||
runtimegegevens. Ontbrekend bewijs blijft onbekend. Gebruik het broncommit, de
|
||||
image-identiteit en de toegepaste migraties samen wanneer je een release of
|
||||
herstel beoordeelt. Een gezonde applicatiecontrole bewijst op zichzelf niet dat
|
||||
de productieversie gelijk is aan de nieuwste repositoryversie.
|
||||
Generated
+19
@@ -53,6 +53,9 @@ importers:
|
||||
'@radix-ui/react-tooltip':
|
||||
specifier: ^1.2.16
|
||||
version: 1.2.16(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)
|
||||
'@tanstack/react-query':
|
||||
specifier: ^5.104.0
|
||||
version: 5.104.0(react@19.2.8)
|
||||
'@tanstack/react-table':
|
||||
specifier: ^8.21.3
|
||||
version: 8.21.3(react-dom@19.2.8(react@19.2.8))(react@19.2.8)
|
||||
@@ -854,6 +857,14 @@ packages:
|
||||
'@standard-schema/spec@1.1.0':
|
||||
resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==}
|
||||
|
||||
'@tanstack/query-core@5.104.0':
|
||||
resolution: {integrity: sha512-JrC2r/JQlXt7khBSdUpsgxNvybzOg+aITa+ARRMlP2AFo93Y8vIqz077rp+e61mJetSZ0T6AJbwW1JHer1vrPQ==}
|
||||
|
||||
'@tanstack/react-query@5.104.0':
|
||||
resolution: {integrity: sha512-e1TZmDCQnWIfiDVryIHeA6Idj+Lfx1hORLOhXS/l5wcgvtVD4yYqbTDl378sGQKIi2cSgb0TEMC9cKK8GGoJEw==}
|
||||
peerDependencies:
|
||||
react: ^18 || ^19
|
||||
|
||||
'@tanstack/react-table@8.21.3':
|
||||
resolution: {integrity: sha512-5nNMTSETP4ykGegmVkhjcS8tTLW6Vl4axfEGQN3v0zdHYbK4UfoqfPChclTrJ4EoK9QynqAu9oUf8VEmrpZ5Ww==}
|
||||
engines: {node: '>=12'}
|
||||
@@ -1259,6 +1270,7 @@ packages:
|
||||
eslint@9.39.5:
|
||||
resolution: {integrity: sha512-DgZS62aPLXKlnxILS/AYCoRvHaZeXceIzlXPkkGGzJWSow1aEk0lbTlxUSlyjC8jcaKxAdOnTDz+o1JFSBsyjw==}
|
||||
engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
|
||||
deprecated: This version is no longer supported. Please see https://eslint.org/version-support for other options.
|
||||
hasBin: true
|
||||
peerDependencies:
|
||||
jiti: '*'
|
||||
@@ -2829,6 +2841,13 @@ snapshots:
|
||||
|
||||
'@standard-schema/spec@1.1.0': {}
|
||||
|
||||
'@tanstack/query-core@5.104.0': {}
|
||||
|
||||
'@tanstack/react-query@5.104.0(react@19.2.8)':
|
||||
dependencies:
|
||||
'@tanstack/query-core': 5.104.0
|
||||
react: 19.2.8
|
||||
|
||||
'@tanstack/react-table@8.21.3(react-dom@19.2.8(react@19.2.8))(react@19.2.8)':
|
||||
dependencies:
|
||||
'@tanstack/table-core': 8.21.3
|
||||
|
||||
@@ -15,6 +15,7 @@ import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
import yaml from "js-yaml";
|
||||
import { extractRoutesFromSource } from "./lib/openapi-routes.mjs";
|
||||
|
||||
const endpointsDir = "src/backend/ItWorx.KnowledgeHub.Api/Endpoints";
|
||||
const openApiPath = "docs/api/openapi.yaml";
|
||||
@@ -32,42 +33,7 @@ function listEndpointFiles(dir) {
|
||||
}
|
||||
|
||||
function extractRoutesFromFile(filePath) {
|
||||
const source = fs.readFileSync(filePath, "utf8");
|
||||
const routes = [];
|
||||
|
||||
// MapGroup("/api/v1/xyz") establishes a prefix for every Map* call chained off the
|
||||
// returned variable within the same file — every endpoint file in this codebase uses
|
||||
// exactly this one-group-per-file pattern (verified by inspection), so a single
|
||||
// "most recent MapGroup literal in this file" tracking variable is sufficient and
|
||||
// avoids needing a real C# parser.
|
||||
const groupMatch = source.match(/MapGroup\(\s*"([^"]+)"\s*\)/);
|
||||
const groupPrefix = groupMatch ? groupMatch[1] : "";
|
||||
|
||||
const mapCallPattern = /\.Map(Get|Post|Put|Delete|Patch)\(\s*"([^"]*)"/g;
|
||||
let match;
|
||||
while ((match = mapCallPattern.exec(source)) !== null) {
|
||||
const method = match[1].toUpperCase();
|
||||
const routeLiteral = match[2];
|
||||
const fullPath = groupPrefix + routeLiteral;
|
||||
routes.push({ method, path: normalizeRoute(fullPath), file: filePath });
|
||||
}
|
||||
|
||||
return routes;
|
||||
}
|
||||
|
||||
// Normalizes an ASP.NET route template's typed route constraints
|
||||
// (`{attachmentId:guid}`) to OpenAPI's untyped `{attachmentId}` style, and strips the
|
||||
// leading `/api/v1` prefix to match how docs/api/openapi.yaml keys its paths (relative
|
||||
// to the documented `servers[0].url`).
|
||||
function normalizeRoute(route) {
|
||||
let normalized = route.replace(/\{([a-zA-Z0-9_]+)(:[a-zA-Z]+)?\}/g, "{$1}");
|
||||
if (normalized.startsWith("/api/v1")) {
|
||||
normalized = normalized.slice("/api/v1".length);
|
||||
}
|
||||
if (normalized === "") {
|
||||
normalized = "/";
|
||||
}
|
||||
return normalized;
|
||||
return extractRoutesFromSource(fs.readFileSync(filePath, "utf8"), filePath);
|
||||
}
|
||||
|
||||
function isExcluded(routePath) {
|
||||
|
||||
@@ -24,6 +24,12 @@ import { spawnSync } from "node:child_process";
|
||||
|
||||
const repoRoot = process.cwd();
|
||||
const outputPath = process.argv[2] ?? path.join(repoRoot, "release-manifest.json");
|
||||
const semverIdentifier = "(?:0|[1-9]\\d*|\\d*[A-Za-z-][0-9A-Za-z-]*)";
|
||||
const semverPattern = new RegExp(
|
||||
`^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)` +
|
||||
`(?:-(${semverIdentifier}(?:\\.${semverIdentifier})*))?` +
|
||||
`(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$`
|
||||
);
|
||||
|
||||
function sha256File(relativePath) {
|
||||
const fullPath = path.join(repoRoot, relativePath);
|
||||
@@ -88,6 +94,18 @@ function readOpenApiVersion() {
|
||||
}
|
||||
|
||||
function readSemanticVersion() {
|
||||
const releaseTag = nonEmpty(process.env.RELEASE_TAG);
|
||||
if (releaseTag) {
|
||||
const version = releaseTag.startsWith("v") ? releaseTag.slice(1) : "";
|
||||
if (!version || !semverPattern.test(version)) {
|
||||
throw new Error(`RELEASE_TAG must be a v-prefixed semantic version; received '${releaseTag}'.`);
|
||||
}
|
||||
// Docker's web-build stage intentionally has no .git directory. A validated
|
||||
// release tag is therefore the authoritative semantic version for tagged
|
||||
// images, while the full tag remains separately recorded as releaseTag.
|
||||
return version;
|
||||
}
|
||||
|
||||
const versionFile = path.join(repoRoot, "VERSION");
|
||||
if (fs.existsSync(versionFile)) {
|
||||
return fs.readFileSync(versionFile, "utf8").trim();
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
/** Extract literal Minimal API routes and their receiver-specific MapGroup prefix. */
|
||||
export function extractRoutesFromSource(source, file) {
|
||||
const prefixes = new Map([["app", ""]]);
|
||||
const groupPattern = /\b(?:var|const|RouteGroupBuilder)\s+(\w+)\s*=\s*(\w+)\.MapGroup\(\s*"([^"]+)"\s*\)/g;
|
||||
for (const match of source.matchAll(groupPattern)) {
|
||||
const parent = prefixes.get(match[2]);
|
||||
if (parent === undefined) throw new Error(`Unknown MapGroup receiver ${match[2]} in ${file}`);
|
||||
prefixes.set(match[1], parent + match[3]);
|
||||
}
|
||||
const routes = [];
|
||||
const mapPattern = /\b(\w+)\.Map(Get|Post|Put|Delete|Patch)\(\s*"([^"]*)"/g;
|
||||
for (const match of source.matchAll(mapPattern)) {
|
||||
const prefix = prefixes.get(match[1]);
|
||||
if (prefix === undefined) throw new Error(`Unknown route receiver ${match[1]} in ${file}`);
|
||||
const path = (prefix + match[3]).replace(/\{([a-zA-Z0-9_]+)(:[a-zA-Z]+)?\}/g, "{$1}");
|
||||
if (!path.startsWith("/api/v1")) {
|
||||
if (!path.startsWith("/health")) throw new Error(`Unexpected non-API route ${path} in ${file}`);
|
||||
continue;
|
||||
}
|
||||
routes.push({ method: match[2].toUpperCase(), path: path.slice("/api/v1".length) || "/", file });
|
||||
}
|
||||
return routes;
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
export function validateProductBacklog(backlog, plan) {
|
||||
const errors = [];
|
||||
if (backlog?.schemaVersion !== 2) errors.push("Product backlog schemaVersion must be 2.");
|
||||
if (!Array.isArray(backlog?.statusValues)) errors.push("Product backlog statusValues must be an array.");
|
||||
if (!Array.isArray(backlog?.items)) {
|
||||
errors.push("Product backlog items must be an array.");
|
||||
return errors;
|
||||
}
|
||||
const milestoneById = new Map((plan?.milestones ?? []).map((milestone) => [milestone.id, milestone]));
|
||||
const itemIds = new Set();
|
||||
const statuses = new Set(backlog.statusValues ?? []);
|
||||
for (const item of backlog.items) {
|
||||
if (!item || typeof item !== "object") {
|
||||
errors.push("Product backlog contains an invalid item.");
|
||||
continue;
|
||||
}
|
||||
if (typeof item.id !== "string" || !item.id || itemIds.has(item.id)) {
|
||||
errors.push(`Product backlog item has a missing or duplicate ID: ${item.id ?? "(missing)"}.`);
|
||||
}
|
||||
itemIds.add(item.id);
|
||||
if (!statuses.has(item.status)) errors.push(`${item.id} has an invalid status: ${item.status}.`);
|
||||
const milestone = milestoneById.get(item.milestone);
|
||||
if (!milestone) {
|
||||
errors.push(`${item.id} references an unknown milestone: ${item.milestone}.`);
|
||||
continue;
|
||||
}
|
||||
if (item.status === "complete" && (!Array.isArray(item.evidence) || item.evidence.length === 0)) {
|
||||
errors.push(`${item.id} is complete without evidence references.`);
|
||||
}
|
||||
if (item.status === "todo" && milestone.status === "complete" && item.postMilestone !== true) {
|
||||
errors.push(`${item.id} is todo in a completed milestone without an explicit postMilestone marker.`);
|
||||
}
|
||||
if (item.status === "partial" && (!item.notes || !Array.isArray(item.evidence) || item.evidence.length === 0)) {
|
||||
errors.push(`${item.id} is partial without notes and evidence references.`);
|
||||
}
|
||||
if (item.status === "blocked_external") {
|
||||
const acceptance = milestone.acceptance?.find((criterion) => criterion.id === item.acceptance);
|
||||
if (!item.externalOwner || !acceptance?.external || acceptance.met !== false) {
|
||||
errors.push(`${item.id} must reference an unmet external acceptance criterion and owner.`);
|
||||
}
|
||||
}
|
||||
if (item.status === "complete" && milestone.status !== "complete") {
|
||||
const acceptance = milestone.acceptance?.find((criterion) => criterion.id === item.acceptance);
|
||||
if (!acceptance || acceptance.met !== true) {
|
||||
errors.push(`${item.id} completes an unfinished milestone without a met acceptance criterion.`);
|
||||
}
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
import { commandExists } from "./exec.mjs";
|
||||
|
||||
export function requireReleaseTool(command, exists = commandExists) {
|
||||
if (!exists(command)) {
|
||||
throw new Error(`${command} is required for the release gate but was not found on PATH.`);
|
||||
}
|
||||
return command;
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
function skipQuoted(source, start) {
|
||||
const quote = source[start];
|
||||
let escaped = false;
|
||||
for (let index = start + 1; index < source.length; index += 1) {
|
||||
const character = source[index];
|
||||
if (escaped) {
|
||||
escaped = false;
|
||||
} else if (character === "\\") {
|
||||
escaped = true;
|
||||
} else if (character === quote) {
|
||||
return index + 1;
|
||||
}
|
||||
}
|
||||
return source.length;
|
||||
}
|
||||
|
||||
function readRouteTag(source, start) {
|
||||
const closing = source[start + 1] === "/";
|
||||
const nameStart = start + (closing ? 2 : 1);
|
||||
const nameMatch = /^[A-Za-z_$][\w$.-]*/u.exec(source.slice(nameStart));
|
||||
if (!nameMatch || nameMatch[0] !== "Route") return null;
|
||||
|
||||
let curlyDepth = 0;
|
||||
let quote = null;
|
||||
let escaped = false;
|
||||
for (let index = nameStart + nameMatch[0].length; index < source.length; index += 1) {
|
||||
const character = source[index];
|
||||
if (quote !== null) {
|
||||
if (escaped) escaped = false;
|
||||
else if (character === "\\") escaped = true;
|
||||
else if (character === quote) quote = null;
|
||||
continue;
|
||||
}
|
||||
if (character === "\"" || character === "'" || character === "`") {
|
||||
quote = character;
|
||||
} else if (character === "{") {
|
||||
curlyDepth += 1;
|
||||
} else if (character === "}" && curlyDepth > 0) {
|
||||
curlyDepth -= 1;
|
||||
} else if (character === ">" && curlyDepth === 0) {
|
||||
const text = source.slice(nameStart + nameMatch[0].length, index);
|
||||
return {
|
||||
closing,
|
||||
selfClosing: !closing && /\/\s*$/u.test(text),
|
||||
attributes: closing ? "" : text.replace(/\/\s*$/u, ""),
|
||||
end: index + 1
|
||||
};
|
||||
}
|
||||
}
|
||||
return { error: "AuthenticatedRoutes.tsx contains an unterminated <Route> tag." };
|
||||
}
|
||||
|
||||
function findStaticPath(attributes) {
|
||||
const pathAttribute = /(?:^|\s)path\s*=\s*(?:"([^"]*)"|'([^']*)')/u.exec(attributes);
|
||||
if (pathAttribute) return { value: pathAttribute[1] ?? pathAttribute[2] };
|
||||
if (/(?:^|\s)path\s*=\s*\{/u.test(attributes)) return { error: "dynamic" };
|
||||
return null;
|
||||
}
|
||||
|
||||
function routePath(parentPath, childPath) {
|
||||
if (childPath === "*") return parentPath ? `${parentPath.replace(/\/$/u, "")}/*` : "*";
|
||||
if (childPath.startsWith("/")) return childPath.replace(/\/{2,}/gu, "/");
|
||||
const joined = `${parentPath || ""}/${childPath}`.replace(/\/{2,}/gu, "/");
|
||||
return joined || "/";
|
||||
}
|
||||
|
||||
function routeFeatures(attributes) {
|
||||
const features = new Set();
|
||||
for (const match of attributes.matchAll(/\bfeatures\s*\?\.\s*([A-Za-z_$][\w$]*)/gu)) {
|
||||
features.add(match[1]);
|
||||
}
|
||||
return [...features].sort();
|
||||
}
|
||||
|
||||
function extractRouteStructure(source) {
|
||||
const errors = [];
|
||||
const routeStack = [];
|
||||
const records = [];
|
||||
|
||||
for (let index = 0; index < source.length;) {
|
||||
if (source.startsWith("//", index)) {
|
||||
const newline = source.indexOf("\n", index + 2);
|
||||
index = newline < 0 ? source.length : newline + 1;
|
||||
continue;
|
||||
}
|
||||
if (source.startsWith("/*", index)) {
|
||||
const commentEnd = source.indexOf("*/", index + 2);
|
||||
index = commentEnd < 0 ? source.length : commentEnd + 2;
|
||||
continue;
|
||||
}
|
||||
if (source[index] === "\"" || source[index] === "'" || source[index] === "`") {
|
||||
index = skipQuoted(source, index);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (source[index] === "<" && (source[index + 1] === "/" || /[A-Za-z_$]/u.test(source[index + 1] ?? ""))) {
|
||||
const tag = readRouteTag(source, index);
|
||||
if (tag?.error) {
|
||||
errors.push(tag.error);
|
||||
break;
|
||||
}
|
||||
if (tag) {
|
||||
if (tag.closing) {
|
||||
if (routeStack.length === 0) errors.push("AuthenticatedRoutes.tsx closes a <Route> without a matching opening tag.");
|
||||
else routeStack.pop();
|
||||
} else {
|
||||
const parent = routeStack.at(-1);
|
||||
const parentPath = parent?.path ?? "";
|
||||
const pathAttribute = findStaticPath(tag.attributes);
|
||||
const hasIndex = /(?:^|\s)index(?:\s|=|$)/u.test(tag.attributes);
|
||||
const localFeatures = routeFeatures(tag.attributes);
|
||||
const features = [...new Set([...(parent?.features ?? []), ...localFeatures])].sort();
|
||||
const administrator = Boolean(parent?.administrator)
|
||||
|| /\bsession\.role\s*===?\s*["']Administrator["']/u.test(tag.attributes);
|
||||
|
||||
if (pathAttribute?.error === "dynamic") {
|
||||
errors.push("AuthenticatedRoutes.tsx uses a dynamic Route path; route-map validation requires a static path.");
|
||||
}
|
||||
if (hasIndex && pathAttribute) errors.push("AuthenticatedRoutes.tsx declares a Route with both index and path.");
|
||||
|
||||
let fullPath = null;
|
||||
if (hasIndex) fullPath = parentPath || "/";
|
||||
else if (pathAttribute && !pathAttribute.error) fullPath = routePath(parentPath, pathAttribute.value);
|
||||
|
||||
if (fullPath !== null) {
|
||||
records.push({
|
||||
path: fullPath,
|
||||
featureGates: features,
|
||||
auth: administrator ? "administrator" : "authenticated",
|
||||
depth: routeStack.length,
|
||||
parentPath
|
||||
});
|
||||
}
|
||||
|
||||
if (!tag.selfClosing) {
|
||||
routeStack.push({
|
||||
path: fullPath ?? parentPath,
|
||||
features,
|
||||
administrator
|
||||
});
|
||||
}
|
||||
}
|
||||
index = tag.end;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
index += 1;
|
||||
}
|
||||
|
||||
if (routeStack.length > 0) errors.push("AuthenticatedRoutes.tsx contains an unclosed <Route> tag.");
|
||||
|
||||
const uniqueRecords = [];
|
||||
for (const record of records) {
|
||||
const priorIndex = uniqueRecords.findIndex((candidate) => candidate.path === record.path);
|
||||
if (priorIndex < 0) {
|
||||
uniqueRecords.push(record);
|
||||
continue;
|
||||
}
|
||||
|
||||
const prior = uniqueRecords[priorIndex];
|
||||
const isNestedSamePath = record.parentPath === record.path && record.depth > prior.depth;
|
||||
if (isNestedSamePath) {
|
||||
uniqueRecords[priorIndex] = record;
|
||||
} else {
|
||||
errors.push(`AuthenticatedRoutes.tsx declares duplicate route path ${record.path}.`);
|
||||
}
|
||||
}
|
||||
|
||||
return { records: uniqueRecords, errors };
|
||||
}
|
||||
|
||||
export function extractAuthenticatedRoutes(source) {
|
||||
return extractRouteStructure(source).records;
|
||||
}
|
||||
|
||||
export function extractAuthenticatedRoutePaths(source) {
|
||||
return extractAuthenticatedRoutes(source).map((route) => route.path);
|
||||
}
|
||||
|
||||
function mappedFeatureGates(route) {
|
||||
if (Array.isArray(route?.featureGates)) return [...route.featureGates].sort();
|
||||
return typeof route?.featureGate === "string" ? [route.featureGate] : [];
|
||||
}
|
||||
|
||||
export function validateRouteMap(routeMap, authenticatedSource, sessionSource) {
|
||||
const errors = [];
|
||||
const { records, errors: routeErrors } = extractRouteStructure(authenticatedSource);
|
||||
errors.push(...routeErrors);
|
||||
const actualPaths = records.map((route) => route.path);
|
||||
const mappedRoutes = Array.isArray(routeMap?.routes) ? routeMap.routes : [];
|
||||
const mappedPaths = mappedRoutes.map((route) => route?.path);
|
||||
|
||||
if (new Set(mappedPaths).size !== mappedPaths.length) {
|
||||
errors.push("product/route-map.json declares duplicate paths.");
|
||||
}
|
||||
const missing = actualPaths.filter((routePath) => !mappedPaths.includes(routePath));
|
||||
const stale = mappedPaths.filter((routePath) => !actualPaths.includes(routePath));
|
||||
if (missing.length > 0) errors.push(`Route map is missing: ${missing.join(", ")}.`);
|
||||
if (stale.length > 0) errors.push(`Route map has stale entries: ${stale.join(", ")}.`);
|
||||
|
||||
if (routeMap?.sourceOfTruth !== "src/frontend/src/router/AuthenticatedRoutes.tsx") {
|
||||
errors.push("Route map sourceOfTruth must identify AuthenticatedRoutes.tsx.");
|
||||
}
|
||||
const states = routeMap?.sessionStates;
|
||||
const expectedStates = ["loading", "error", "setupRequired", "unauthenticated"];
|
||||
if (!states || typeof states !== "object" || Array.isArray(states)) {
|
||||
errors.push("Route map must document the pre-route session states.");
|
||||
} else {
|
||||
const actualStates = Object.keys(states).sort();
|
||||
if (JSON.stringify(actualStates) !== JSON.stringify([...expectedStates].sort())) {
|
||||
errors.push(`Route map session states must be ${expectedStates.join(", ")}.`);
|
||||
}
|
||||
for (const [condition, screen] of Object.entries(states)) {
|
||||
if (typeof screen !== "string" || !sessionSource.includes(`<${screen}`)) {
|
||||
errors.push(`Session screen is not present in App.tsx: ${condition}.`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const actualRoute of records) {
|
||||
const mappedRoute = mappedRoutes.find((route) => route?.path === actualRoute.path);
|
||||
if (!mappedRoute) continue;
|
||||
if (mappedRoute.auth !== actualRoute.auth) {
|
||||
errors.push(`${actualRoute.path} must be marked ${actualRoute.auth}-only.`);
|
||||
}
|
||||
const expectedFeatures = actualRoute.featureGates;
|
||||
const mappedFeatures = mappedFeatureGates(mappedRoute);
|
||||
if (JSON.stringify(expectedFeatures) !== JSON.stringify(mappedFeatures)) {
|
||||
const expected = expectedFeatures.length > 0 ? expectedFeatures.join(", ") : "none";
|
||||
errors.push(`${actualRoute.path} feature-gate registry must be ${expected}.`);
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
@@ -28,4 +28,49 @@ if (!Array.isArray(manifest.files) || manifest.files.length === 0) {
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`Public source structure passed (${manifest.files.length} allowlisted files).`);
|
||||
const markdownFiles = [];
|
||||
function walkMarkdown(directory) {
|
||||
for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
|
||||
if ([".git", "node_modules", "__pycache__"].includes(entry.name)) continue;
|
||||
const fullPath = path.join(directory, entry.name);
|
||||
if (entry.isDirectory()) walkMarkdown(fullPath);
|
||||
else if (entry.isFile() && entry.name.toLowerCase().endsWith(".md")) markdownFiles.push(fullPath);
|
||||
}
|
||||
}
|
||||
|
||||
function stripCode(markdown) {
|
||||
const withoutFences = markdown.replace(/```[\s\S]*?```|~~~[\s\S]*?~~~/g, (block) =>
|
||||
block.replace(/[^\n]/g, " "));
|
||||
return withoutFences.replace(/`[^`\n]*`/g, (span) => " ".repeat(span.length));
|
||||
}
|
||||
|
||||
walkMarkdown(process.cwd());
|
||||
const errors = [];
|
||||
const linkPattern = /\[[^\]]+\]\(([^)]+)\)/g;
|
||||
for (const markdownFile of markdownFiles) {
|
||||
const content = stripCode(fs.readFileSync(markdownFile, "utf8"));
|
||||
for (const match of content.matchAll(linkPattern)) {
|
||||
let target = match[1].trim();
|
||||
if (target.startsWith("#") || /^[a-z][a-z0-9+.-]*:/i.test(target) || target.startsWith("//")) continue;
|
||||
target = target.split("#", 1)[0].split("?", 1)[0];
|
||||
if (!target) continue;
|
||||
try {
|
||||
target = decodeURIComponent(target);
|
||||
} catch {
|
||||
errors.push(`Could not URL-decode link in ${path.relative(process.cwd(), markdownFile)}: ${target}`);
|
||||
continue;
|
||||
}
|
||||
const resolved = path.resolve(path.dirname(markdownFile), target);
|
||||
if (!fs.existsSync(resolved)) {
|
||||
errors.push(`Broken local Markdown link in ${path.relative(process.cwd(), markdownFile)}: ${match[1]}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length > 0) {
|
||||
console.error("Public source validation failed:");
|
||||
for (const error of errors) console.error(`- ${error}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`Public source structure passed (${manifest.files.length} allowlisted files, ${markdownFiles.length} Markdown files, local links resolved).`);
|
||||
+88
-19
@@ -3,7 +3,8 @@
|
||||
import fs from "node:fs";
|
||||
import process from "node:process";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { commandExists, run } from "./lib/exec.mjs";
|
||||
import { capture, commandExists, run } from "./lib/exec.mjs";
|
||||
import { requireReleaseTool } from "./lib/release-gates.mjs";
|
||||
|
||||
const flags = new Set(process.argv.slice(2));
|
||||
const structureOnly = flags.has("--structure-only");
|
||||
@@ -11,6 +12,24 @@ const fast = flags.has("--fast");
|
||||
const ci = flags.has("--ci");
|
||||
const release = flags.has("--release");
|
||||
const publicSourceExport = fs.existsSync("PUBLIC_SOURCE_EXPORT.md");
|
||||
const releaseImageReference = process.env.RELEASE_IMAGE_REFERENCE ?? "itworx/knowledge-hub:local";
|
||||
let releaseImageId = null;
|
||||
// Test hosts use fixed configuration and do not need appsettings hot reload.
|
||||
// Avoid consuming the shared Linux host's inotify quota for hundreds of factories.
|
||||
const backendTestEnvironment = { DOTNET_HOSTBUILDER__RELOADCONFIGONCHANGE: "false" };
|
||||
|
||||
function splitImageReference(reference) {
|
||||
const tagSeparator = reference.lastIndexOf(":");
|
||||
if (tagSeparator <= reference.lastIndexOf("/") || tagSeparator === 0 || /\s/.test(reference)) {
|
||||
throw new Error(`Release image reference must include a repository and tag: '${reference}'.`);
|
||||
}
|
||||
const repository = reference.slice(0, tagSeparator);
|
||||
const tag = reference.slice(tagSeparator + 1);
|
||||
if (!repository || !/^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$/.test(tag)) {
|
||||
throw new Error(`Release image reference has an invalid repository or tag: '${reference}'.`);
|
||||
}
|
||||
return { repository, tag };
|
||||
}
|
||||
|
||||
function findWorkingPython() {
|
||||
// On some Windows setups `python3` resolves via PATH to the Microsoft Store
|
||||
@@ -39,12 +58,22 @@ async function section(name, action) {
|
||||
}
|
||||
|
||||
try {
|
||||
// Fail before starting a long release matrix when a mandatory scanner is
|
||||
// unavailable. CI provides these through the pinned release workflow setup;
|
||||
// local release verification must install the same tools explicitly.
|
||||
const releaseGitleaks = release ? requireReleaseTool("gitleaks") : null;
|
||||
const releaseTrivy = release ? requireReleaseTool("trivy") : null;
|
||||
|
||||
await section("Bundle structure", async () => {
|
||||
if (publicSourceExport) {
|
||||
await run(process.execPath, ["scripts/validate-public-source.mjs"]);
|
||||
} else {
|
||||
await run(process.execPath, ["scripts/validate-bundle.mjs"]);
|
||||
await run(process.execPath, ["scripts/test-safety-hook.mjs"]);
|
||||
await run(process.execPath, ["scripts/validate-product-state.mjs"]);
|
||||
await run(process.execPath, ["scripts/validate-route-map.mjs"]);
|
||||
await run(process.execPath, ["--test", "scripts/tests/route-map.test.mjs", "scripts/tests/product-backlog.test.mjs", "scripts/tests/validate-public-source.test.mjs", "scripts/tests/validate-bundle.test.mjs", "scripts/tests/v2-r4-real-backup-restore-drill.test.mjs", "scripts/tests/cleanup-temporary-resources.test.mjs"]);
|
||||
await run(python, ["-m", "unittest", "scripts.tests.test_public_source_pipeline"]);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -108,7 +137,7 @@ try {
|
||||
"trx",
|
||||
"--results-directory",
|
||||
"artifacts/test-results"
|
||||
]);
|
||||
], { env: backendTestEnvironment });
|
||||
});
|
||||
|
||||
await section("Frontend install and verification", async () => {
|
||||
@@ -149,16 +178,60 @@ try {
|
||||
|
||||
if (ci || release) {
|
||||
await section("Production image build", async () => {
|
||||
const buildCommit = process.env.GITHUB_SHA ?? process.env.CI_COMMIT_SHA ?? "verification";
|
||||
if (release) {
|
||||
const { repository, tag } = splitImageReference(releaseImageReference);
|
||||
await run("docker", [
|
||||
"build",
|
||||
"--file",
|
||||
"deploy/docker/Dockerfile",
|
||||
"--tag",
|
||||
releaseImageReference,
|
||||
"--build-arg",
|
||||
`BUILD_COMMIT=${buildCommit}`,
|
||||
"--build-arg",
|
||||
`RELEASE_TAG=${process.env.RELEASE_TAG ?? ""}`,
|
||||
"--build-arg",
|
||||
`IMAGE_REPOSITORY=${repository}`,
|
||||
"--build-arg",
|
||||
`IMAGE_TAG=${tag}`,
|
||||
"."
|
||||
]);
|
||||
releaseImageId = await capture("docker", [
|
||||
"image",
|
||||
"inspect",
|
||||
"--format",
|
||||
"{{.Id}}",
|
||||
releaseImageReference
|
||||
]);
|
||||
if (!/^sha256:[0-9a-f]{64}$/i.test(releaseImageId)) {
|
||||
throw new Error(`Docker returned an invalid image ID for '${releaseImageReference}'.`);
|
||||
}
|
||||
const imageIdPath = "artifacts/release-verification/image-id.txt";
|
||||
fs.mkdirSync("artifacts/release-verification", { recursive: true });
|
||||
fs.writeFileSync(imageIdPath, `${releaseImageId}\n`, "utf8");
|
||||
console.log(`Captured immutable release image ID in ${imageIdPath}.`);
|
||||
} else {
|
||||
await run("docker", ["compose", "build", "app"], {
|
||||
env: {
|
||||
POSTGRES_PASSWORD: "verification-only-not-for-production",
|
||||
BUILD_COMMIT: process.env.GITHUB_SHA ?? process.env.CI_COMMIT_SHA ?? "verification"
|
||||
BUILD_COMMIT: buildCommit
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (release) {
|
||||
await section("Release evidence integrity tests", async () => {
|
||||
await run(process.execPath, [
|
||||
"--test",
|
||||
"scripts/tests/release-gates.test.mjs",
|
||||
"scripts/tests/generate-release-evidence.test.mjs",
|
||||
"scripts/tests/generate-release-manifest.test.mjs"
|
||||
]);
|
||||
});
|
||||
|
||||
await section("Full-scale search/tree suite (release-gate only)", async () => {
|
||||
await run("dotnet", [
|
||||
"test",
|
||||
@@ -167,8 +240,12 @@ try {
|
||||
"Release",
|
||||
"--no-build",
|
||||
"--filter",
|
||||
"Category=ReleaseScale"
|
||||
]);
|
||||
"Category=ReleaseScale",
|
||||
"--logger",
|
||||
"trx",
|
||||
"--results-directory",
|
||||
"artifacts/test-results"
|
||||
], { env: backendTestEnvironment });
|
||||
});
|
||||
|
||||
await section(".NET dependency vulnerability scan", async () => {
|
||||
@@ -195,16 +272,14 @@ try {
|
||||
});
|
||||
|
||||
await section("Secret scan", async () => {
|
||||
if (!commandExists("gitleaks")) {
|
||||
throw new Error("gitleaks is required for the release secret scan but was not found on PATH.");
|
||||
}
|
||||
|
||||
await run("gitleaks", ["detect", "--source", ".", "--redact"]);
|
||||
await run(releaseGitleaks, ["detect", "--source", ".", "--redact"]);
|
||||
});
|
||||
|
||||
if (commandExists("trivy")) {
|
||||
await section("Container vulnerability scan", async () => {
|
||||
await run("trivy", [
|
||||
if (!releaseImageId) {
|
||||
throw new Error("The release image ID was not captured after the production image build.");
|
||||
}
|
||||
await run(releaseTrivy, [
|
||||
"image",
|
||||
"--severity",
|
||||
"CRITICAL,HIGH",
|
||||
@@ -212,15 +287,9 @@ try {
|
||||
"1",
|
||||
"--scanners",
|
||||
"vuln",
|
||||
"itworx/knowledge-hub:local"
|
||||
releaseImageId
|
||||
]);
|
||||
});
|
||||
} else {
|
||||
console.warn(
|
||||
"\ntrivy not found on PATH — container vulnerability scan skipped. Install " +
|
||||
"trivy before a real release; this is not a substitute for the scan."
|
||||
);
|
||||
}
|
||||
|
||||
console.warn(
|
||||
"\nRelease verification still requires the milestone-defined E2E (desktop + " +
|
||||
|
||||
@@ -0,0 +1,461 @@
|
||||
using System.Text.Json;
|
||||
using ItWorx.KnowledgeHub.Application.Identity;
|
||||
using ItWorx.KnowledgeHub.Application.Jobs;
|
||||
using ItWorx.KnowledgeHub.Application.Jobs.UseCases;
|
||||
using ItWorx.KnowledgeHub.Application.Knowledge;
|
||||
using ItWorx.KnowledgeHub.Application.Knowledge.UseCases;
|
||||
using ItWorx.KnowledgeHub.Application.Operations;
|
||||
using ItWorx.KnowledgeHub.Application.Operations.UseCases;
|
||||
using ItWorx.KnowledgeHub.Domain.Identity;
|
||||
using ItWorx.KnowledgeHub.Domain.Jobs;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Api.BackgroundServices;
|
||||
|
||||
/// <summary>Executes durable jobs with bounded worker concurrency. PostgreSQL leases
|
||||
/// allow another process to resume work after a crash; each course page is handled as
|
||||
/// an independent, idempotent ETag-guarded update.</summary>
|
||||
public sealed partial class JobWorker(
|
||||
IServiceScopeFactory scopeFactory,
|
||||
BackgroundJobsOptions options,
|
||||
TimeProvider timeProvider,
|
||||
PlatformMetrics metrics,
|
||||
ILogger<JobWorker> logger) : BackgroundService
|
||||
{
|
||||
private long lastMetricsRefreshMilliseconds;
|
||||
private int metricsRefreshInProgress;
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
options.Validate();
|
||||
if (!options.Enabled)
|
||||
{
|
||||
LogDisabled();
|
||||
return;
|
||||
}
|
||||
var workers = Enumerable.Range(0, options.MaxConcurrency)
|
||||
.Select(_ => RunWorkerAsync(stoppingToken));
|
||||
await Task.WhenAll(workers);
|
||||
}
|
||||
|
||||
private async Task RunWorkerAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
using var scope = scopeFactory.CreateScope();
|
||||
var repository = scope.ServiceProvider.GetRequiredService<IBackgroundJobRepository>();
|
||||
var leaseToken = Guid.NewGuid();
|
||||
Guid? jobId;
|
||||
try
|
||||
{
|
||||
var now = timeProvider.GetUtcNow();
|
||||
jobId = await repository.TryClaimNextAsync(
|
||||
leaseToken, now, now.AddSeconds(options.LeaseSeconds), stoppingToken);
|
||||
await RefreshMetricsAsync(repository, force: jobId is not null, stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
LogWorkerPollFailed(exception.GetType().Name);
|
||||
try { await Task.Delay(TimeSpan.FromMilliseconds(options.PollMilliseconds), timeProvider, stoppingToken); }
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; }
|
||||
continue;
|
||||
}
|
||||
|
||||
if (jobId is null)
|
||||
{
|
||||
try { await Task.Delay(TimeSpan.FromMilliseconds(options.PollMilliseconds), timeProvider, stoppingToken); }
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; }
|
||||
continue;
|
||||
}
|
||||
|
||||
var claimedJob = await repository.FindLeasedAsync(jobId.Value, leaseToken, stoppingToken);
|
||||
if (claimedJob is null)
|
||||
continue;
|
||||
|
||||
using var leaseLost = new CancellationTokenSource();
|
||||
using var userCancellation = new CancellationTokenSource();
|
||||
using var operationCancellation = CancellationTokenSource.CreateLinkedTokenSource(
|
||||
stoppingToken, leaseLost.Token);
|
||||
using var monitorCancellation = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken);
|
||||
var monitor = MonitorLeaseAsync(
|
||||
jobId.Value, leaseToken, leaseLost, userCancellation, monitorCancellation.Token);
|
||||
var measurement = metrics.Start("job");
|
||||
|
||||
try
|
||||
{
|
||||
await ExecuteLeasedJobAsync(
|
||||
scope.ServiceProvider, jobId.Value, leaseToken, operationCancellation.Token,
|
||||
userCancellation.Token, stoppingToken);
|
||||
var completed = await repository.FindAsync(claimedJob.WorkspaceId, jobId.Value, stoppingToken);
|
||||
if (completed?.Status == BackgroundJobStatus.Cancelled)
|
||||
measurement.Cancel();
|
||||
else if (completed?.Status == BackgroundJobStatus.Succeeded)
|
||||
measurement.Complete(completed.Processed);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
// Leave the durable running row leased. A different worker will recover
|
||||
// it when the lease expires; explicit user cancellation is checked below.
|
||||
if (userCancellation.IsCancellationRequested && !leaseLost.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
await repository.CancelLeasedJobAsync(jobId.Value, leaseToken, timeProvider.GetUtcNow(), CancellationToken.None);
|
||||
measurement.Cancel();
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
LogJobStateWriteFailed(jobId.Value, exception.GetType().Name);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (userCancellation.IsCancellationRequested && !leaseLost.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
await repository.CancelLeasedJobAsync(jobId.Value, leaseToken, timeProvider.GetUtcNow(), CancellationToken.None);
|
||||
measurement.Cancel();
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
LogJobStateWriteFailed(jobId.Value, exception.GetType().Name);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (leaseLost.IsCancellationRequested)
|
||||
{
|
||||
LogLeaseLost(jobId.Value);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
var retry = await RetryTimeAsync(repository, jobId.Value, leaseToken, CancellationToken.None);
|
||||
try
|
||||
{
|
||||
await repository.RetryLeasedJobAsync(
|
||||
jobId.Value, leaseToken, timeProvider.GetUtcNow(), retry,
|
||||
"processing_retry", "De tijdelijke verwerking is mislukt; de taak wordt opnieuw geprobeerd.",
|
||||
CancellationToken.None);
|
||||
}
|
||||
catch (Exception stateException)
|
||||
{
|
||||
LogJobStateWriteFailed(jobId.Value, stateException.GetType().Name);
|
||||
}
|
||||
LogJobExecutionFailed(jobId.Value, exception.GetType().Name);
|
||||
}
|
||||
finally
|
||||
{
|
||||
measurement.Dispose();
|
||||
monitorCancellation.Cancel();
|
||||
try { await monitor; }
|
||||
catch (OperationCanceledException) { }
|
||||
}
|
||||
|
||||
await RefreshMetricsAsync(repository, force: true, stoppingToken);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ExecuteLeasedJobAsync(
|
||||
IServiceProvider services,
|
||||
Guid jobId,
|
||||
Guid leaseToken,
|
||||
CancellationToken operationToken,
|
||||
CancellationToken userCancellationToken,
|
||||
CancellationToken hostToken)
|
||||
{
|
||||
var jobs = services.GetRequiredService<IBackgroundJobRepository>();
|
||||
var job = await jobs.FindLeasedAsync(jobId, leaseToken, operationToken);
|
||||
if (job is null)
|
||||
return;
|
||||
|
||||
if (!await IsActorAdministratorAsync(services, job, operationToken))
|
||||
{
|
||||
await jobs.FailLeasedJobAsync(jobId, leaseToken, timeProvider.GetUtcNow(),
|
||||
"actor_unavailable", "De uitvoerder is niet langer actief als beheerder.", operationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
if (await jobs.IsCancellationRequestedAsync(jobId, leaseToken, operationToken))
|
||||
{
|
||||
await jobs.CancelLeasedJobAsync(jobId, leaseToken, timeProvider.GetUtcNow(), operationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
if (job.Type == BackgroundJobType.Backup)
|
||||
{
|
||||
using var backupCancellation = CancellationTokenSource.CreateLinkedTokenSource(operationToken, userCancellationToken);
|
||||
await ExecuteBackupAsync(services, job, leaseToken, backupCancellation.Token, operationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
if (job.Type == BackgroundJobType.CourseTagging)
|
||||
await ExecuteCourseTaggingAsync(services, job, leaseToken, operationToken);
|
||||
}
|
||||
|
||||
private async Task ExecuteBackupAsync(
|
||||
IServiceProvider services,
|
||||
BackgroundJob job,
|
||||
Guid leaseToken,
|
||||
CancellationToken backupCancellationToken,
|
||||
CancellationToken stateCancellationToken)
|
||||
{
|
||||
var jobs = services.GetRequiredService<IBackgroundJobRepository>();
|
||||
var backups = services.GetRequiredService<CreateBackupUseCase>();
|
||||
var summary = await backups.ExecuteAsync(
|
||||
new CreateBackupCommand(job.ActorId, job.CorrelationId), backupCancellationToken);
|
||||
var result = JsonSerializer.Serialize(new
|
||||
{
|
||||
backupId = summary.BackupId,
|
||||
createdAt = summary.CreatedAt,
|
||||
sizeBytes = summary.TotalSizeBytes,
|
||||
tableCount = summary.TableCount,
|
||||
attachmentCount = summary.AttachmentCount,
|
||||
schemaVersion = summary.SchemaVersion
|
||||
});
|
||||
await jobs.RecordJobOutcomeAsync(
|
||||
job.Id, leaseToken, BackgroundJobItemState.Succeeded, null, null,
|
||||
stateCancellationToken);
|
||||
await jobs.CompleteLeasedJobAsync(job.Id, leaseToken, result, timeProvider.GetUtcNow(), stateCancellationToken);
|
||||
}
|
||||
|
||||
private async Task ExecuteCourseTaggingAsync(
|
||||
IServiceProvider services,
|
||||
BackgroundJob job,
|
||||
Guid leaseToken,
|
||||
CancellationToken operationToken)
|
||||
{
|
||||
using var payload = JsonDocument.Parse(job.PayloadJson);
|
||||
var rootPageId = payload.RootElement.GetProperty("rootPageId").GetGuid();
|
||||
var tag = payload.RootElement.GetProperty("tag").GetString()!;
|
||||
var jobs = services.GetRequiredService<IBackgroundJobRepository>();
|
||||
var pages = services.GetRequiredService<IPageRepository>();
|
||||
var updatePage = services.GetRequiredService<UpdatePageUseCase>();
|
||||
|
||||
while (true)
|
||||
{
|
||||
operationToken.ThrowIfCancellationRequested();
|
||||
if (await jobs.IsCancellationRequestedAsync(job.Id, leaseToken, operationToken))
|
||||
{
|
||||
await jobs.CancelLeasedJobAsync(job.Id, leaseToken, timeProvider.GetUtcNow(), operationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
var currentJob = await jobs.FindLeasedAsync(job.Id, leaseToken, operationToken);
|
||||
if (currentJob is null)
|
||||
return;
|
||||
if (currentJob.Processed >= currentJob.Total)
|
||||
{
|
||||
await jobs.CompleteLeasedJobAsync(job.Id, leaseToken, JsonSerializer.Serialize(new
|
||||
{
|
||||
rootPageId,
|
||||
tag,
|
||||
total = currentJob.Total,
|
||||
processed = currentJob.Processed,
|
||||
succeeded = currentJob.Succeeded,
|
||||
skipped = currentJob.Skipped,
|
||||
failed = currentJob.Failed
|
||||
}), timeProvider.GetUtcNow(), operationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!await IsActorAdministratorAsync(services, currentJob, operationToken))
|
||||
{
|
||||
await jobs.FailLeasedJobAsync(job.Id, leaseToken, timeProvider.GetUtcNow(),
|
||||
"actor_unavailable", "De uitvoerder is niet langer actief als beheerder.", operationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
var item = await jobs.FindNextPageItemAsync(job.Id, leaseToken, operationToken);
|
||||
if (item is null)
|
||||
throw new InvalidOperationException("The job snapshot is incomplete.");
|
||||
|
||||
await jobs.MarkPageItemProcessingAsync(job.Id, leaseToken, item.Ordinal, timeProvider.GetUtcNow(), operationToken);
|
||||
var page = await pages.FindAsync(item.PageId, operationToken);
|
||||
if (page is null || page.WorkspaceId != job.WorkspaceId || page.IsTrashed)
|
||||
{
|
||||
await jobs.RecordPageItemOutcomeAsync(
|
||||
job.Id, leaseToken, item.Ordinal, BackgroundJobItemState.Failed,
|
||||
"page_unavailable", "Pagina is niet langer beschikbaar.", item.PageId,
|
||||
timeProvider.GetUtcNow(), operationToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (page.Tags.Any(existing => string.Equals(existing, tag, StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
await jobs.RecordPageItemOutcomeAsync(
|
||||
job.Id, leaseToken, item.Ordinal,
|
||||
item.InitiallyTagged ? BackgroundJobItemState.Skipped : BackgroundJobItemState.Succeeded,
|
||||
null, null, null, timeProvider.GetUtcNow(), operationToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (page.Tags.Count >= Domain.Knowledge.Page.MaxTagCount)
|
||||
{
|
||||
await jobs.RecordPageItemOutcomeAsync(
|
||||
job.Id, leaseToken, item.Ordinal, BackgroundJobItemState.Skipped,
|
||||
null, null, null, timeProvider.GetUtcNow(), operationToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
var etag = await pages.GetETagAsync(page.Id, operationToken);
|
||||
if (etag is null)
|
||||
{
|
||||
await jobs.RecordPageItemOutcomeAsync(
|
||||
job.Id, leaseToken, item.Ordinal, BackgroundJobItemState.Failed,
|
||||
"page_unavailable", "Pagina is niet langer beschikbaar.", page.Id,
|
||||
timeProvider.GetUtcNow(), operationToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
var tags = page.Tags.Append(tag).ToArray();
|
||||
var result = await updatePage.ExecuteAsync(
|
||||
new UpdatePageCommand(
|
||||
page.Id,
|
||||
page.TitleValue,
|
||||
page.Markdown,
|
||||
page.DocumentType.ToString(),
|
||||
page.DocumentStatus.ToString(),
|
||||
$"Cursustag {tag} toegevoegd",
|
||||
tags,
|
||||
etag,
|
||||
job.ActorId,
|
||||
job.CorrelationId),
|
||||
operationToken);
|
||||
|
||||
if (result.Status == PageOperationStatus.Success)
|
||||
{
|
||||
await jobs.RecordPageItemOutcomeAsync(
|
||||
job.Id, leaseToken, item.Ordinal, BackgroundJobItemState.Succeeded,
|
||||
null, null, null, timeProvider.GetUtcNow(), operationToken);
|
||||
}
|
||||
else
|
||||
{
|
||||
var conflict = result.Status == PageOperationStatus.ConcurrencyConflict;
|
||||
await jobs.RecordPageItemOutcomeAsync(
|
||||
job.Id, leaseToken, item.Ordinal, BackgroundJobItemState.Failed,
|
||||
conflict ? "concurrency_conflict" : "page_update_failed",
|
||||
conflict ? "Pagina is tijdens de taak gewijzigd; de wijziging is overgeslagen."
|
||||
: "De pagina kon niet worden bijgewerkt.",
|
||||
page.Id, timeProvider.GetUtcNow(), operationToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<bool> IsActorAdministratorAsync(
|
||||
IServiceProvider services,
|
||||
BackgroundJob job,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var accounts = services.GetRequiredService<IUserAccountManager>();
|
||||
var memberships = services.GetRequiredService<IWorkspaceMembershipRepository>();
|
||||
var account = await accounts.FindByIdAsync(job.ActorId, cancellationToken);
|
||||
if (account is not { Enabled: true })
|
||||
return false;
|
||||
var role = await memberships.GetRoleAsync(job.WorkspaceId, job.ActorId, cancellationToken);
|
||||
return role == WorkspaceRole.Administrator;
|
||||
}
|
||||
|
||||
private async Task MonitorLeaseAsync(
|
||||
Guid jobId,
|
||||
Guid leaseToken,
|
||||
CancellationTokenSource leaseLost,
|
||||
CancellationTokenSource userCancellation,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var lastRenewal = timeProvider.GetUtcNow();
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(1), timeProvider, cancellationToken);
|
||||
using var scope = scopeFactory.CreateScope();
|
||||
var jobs = scope.ServiceProvider.GetRequiredService<IBackgroundJobRepository>();
|
||||
if (await jobs.IsCancellationRequestedAsync(jobId, leaseToken, cancellationToken))
|
||||
userCancellation.Cancel();
|
||||
|
||||
var now = timeProvider.GetUtcNow();
|
||||
if (now - lastRenewal >= TimeSpan.FromSeconds(options.LeaseRenewSeconds))
|
||||
{
|
||||
if (!await jobs.RenewLeaseAsync(jobId, leaseToken, now.AddSeconds(options.LeaseSeconds), cancellationToken))
|
||||
{
|
||||
leaseLost.Cancel();
|
||||
return;
|
||||
}
|
||||
lastRenewal = now;
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
// A transient telemetry/lease read failure does not cancel work before
|
||||
// the lease can expire. The database remains authoritative on writes.
|
||||
LogLeaseMonitorFailed(jobId, exception.GetType().Name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RefreshMetricsAsync(IBackgroundJobRepository repository, bool force, CancellationToken cancellationToken)
|
||||
{
|
||||
var nowMilliseconds = timeProvider.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
if (!force && nowMilliseconds - Interlocked.Read(ref lastMetricsRefreshMilliseconds) < 10_000)
|
||||
return;
|
||||
if (Interlocked.CompareExchange(ref metricsRefreshInProgress, 1, 0) != 0)
|
||||
return;
|
||||
try
|
||||
{
|
||||
var statistics = await repository.GetQueueStatisticsAsync(cancellationToken);
|
||||
var oldestAge = statistics.OldestQueuedAt is { } oldest
|
||||
? Math.Max(0, (timeProvider.GetUtcNow() - oldest).TotalSeconds)
|
||||
: (double?)null;
|
||||
metrics.SetQueueState(statistics.Queued, statistics.Running, statistics.Failed, oldestAge);
|
||||
Interlocked.Exchange(ref lastMetricsRefreshMilliseconds, nowMilliseconds);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
LogJobMetricsRefreshFailed(exception.GetType().Name);
|
||||
}
|
||||
finally
|
||||
{
|
||||
Volatile.Write(ref metricsRefreshInProgress, 0);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<DateTimeOffset> RetryTimeAsync(
|
||||
IBackgroundJobRepository repository, Guid jobId, Guid leaseToken, CancellationToken cancellationToken)
|
||||
{
|
||||
var attempt = (await repository.FindLeasedAsync(jobId, leaseToken, cancellationToken))?.AttemptCount ?? 1;
|
||||
var seconds = attempt switch { 1 => 2, 2 => 10, _ => 30 };
|
||||
return timeProvider.GetUtcNow().AddSeconds(seconds);
|
||||
}
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Warning, Message = "Background job poll failed ({ErrorType}).")]
|
||||
private partial void LogWorkerPollFailed(string errorType);
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Warning, Message = "Background job {JobId} failed ({ErrorType}).")]
|
||||
private partial void LogJobExecutionFailed(Guid jobId, string errorType);
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Warning, Message = "Background job {JobId} lost its processing lease.")]
|
||||
private partial void LogLeaseLost(Guid jobId);
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Warning, Message = "Background job {JobId} state write failed ({ErrorType}).")]
|
||||
private partial void LogJobStateWriteFailed(Guid jobId, string errorType);
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Warning, Message = "Background job {JobId} lease monitoring failed ({ErrorType}).")]
|
||||
private partial void LogLeaseMonitorFailed(Guid jobId, string errorType);
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Warning, Message = "Background job metrics refresh failed ({ErrorType}).")]
|
||||
private partial void LogJobMetricsRefreshFailed(string errorType);
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Information, Message = "Background job worker is disabled by configuration.")]
|
||||
private partial void LogDisabled();
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
using ItWorx.KnowledgeHub.Application.Operations;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Api.BackgroundServices;
|
||||
|
||||
public sealed partial class OperationalTelemetryBackgroundService(
|
||||
IServiceScopeFactory scopeFactory, PlatformMetrics metrics, TimeProvider clock,
|
||||
ILogger<OperationalTelemetryBackgroundService> logger) : BackgroundService
|
||||
{
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Do not race initial database migration during host startup.
|
||||
await Task.Delay(TimeSpan.FromSeconds(30), clock, stoppingToken);
|
||||
using var scope = scopeFactory.CreateScope();
|
||||
var snapshot = await scope.ServiceProvider.GetRequiredService<IOperationalTelemetryReader>()
|
||||
.ReadAsync(stoppingToken);
|
||||
var now = clock.GetUtcNow();
|
||||
metrics.SetQueueState(snapshot.QueuedJobs, snapshot.RunningJobs, snapshot.FailedJobs,
|
||||
Age(snapshot.OldestQueuedJobAt, now));
|
||||
metrics.SetOperationalAges(Age(snapshot.OldestPendingIngestionAt, now),
|
||||
Age(snapshot.LatestInventoryCaptureAt, now), Age(snapshot.LatestVerifiedBackupAt, now));
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; }
|
||||
catch (Exception)
|
||||
{
|
||||
// Omit stale gauges after a failed read, rather than exporting old success.
|
||||
metrics.ClearSnapshots();
|
||||
LogReadFailed();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static double? Age(DateTimeOffset? value, DateTimeOffset now) =>
|
||||
value is { } time && time <= now ? (now - time).TotalSeconds : null;
|
||||
|
||||
[LoggerMessage(Level = LogLevel.Warning, Message = "Operationele metriekmeting mislukt; waarden blijven onbekend tot de volgende geslaagde meting.")]
|
||||
private partial void LogReadFailed();
|
||||
}
|
||||
@@ -18,13 +18,14 @@ public static class ActivityEndpoints
|
||||
}
|
||||
|
||||
private static async Task<IResult> ListAsync(
|
||||
IAuditEventReader reader,
|
||||
IActivityEventReader reader,
|
||||
[FromQuery] int? limit,
|
||||
[FromQuery] long? before,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var items = await reader.ListRecentAsync(limit ?? 100, cancellationToken);
|
||||
return Results.Ok(new ActivityListResponse(items.Select(item => new ActivityItemResponse(
|
||||
var page = await reader.ListAsync(limit ?? 100, before, cancellationToken);
|
||||
return Results.Ok(new ActivityListResponse(page.Items.Select(item => new ActivityItemResponse(
|
||||
item.Id, item.ActorKind, item.ActorId, item.Source, item.Action, item.TargetType,
|
||||
item.TargetId, item.Summary, item.OccurredAt)).ToArray()));
|
||||
item.TargetId, item.Summary, item.OccurredAt)).ToArray(), page.NextCursor));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
using ItWorx.KnowledgeHub.Api.Authorization;
|
||||
using ItWorx.KnowledgeHub.Application.Audit;
|
||||
using ItWorx.KnowledgeHub.Contracts.Activity;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Api.Endpoints;
|
||||
|
||||
public static class AdminAuditEndpoints
|
||||
{
|
||||
public static IEndpointRouteBuilder MapAdminAuditEndpoints(this IEndpointRouteBuilder app)
|
||||
{
|
||||
app.MapGet("/api/v1/admin/audit-log", async (
|
||||
IAuditEventReader reader, [FromQuery] int? limit, [FromQuery] long? before, CancellationToken cancellationToken) =>
|
||||
{
|
||||
var page = await reader.ListPageAsync(limit ?? 50, before, cancellationToken);
|
||||
return Results.Ok(new AdminAuditListResponse(page.Items.Select(item => new AdminAuditItemResponse(
|
||||
item.Id, item.ActorKind, item.ActorId, item.Source, item.Action, item.TargetType,
|
||||
item.TargetId, item.Summary, item.OccurredAt, item.CorrelationId)).ToArray(), page.NextCursor));
|
||||
}).WithTags("Admin").RequireAuthorization(AuthorizationPolicies.RequireAdministrator);
|
||||
return app;
|
||||
}
|
||||
}
|
||||
@@ -2,12 +2,12 @@ using ItWorx.KnowledgeHub.Api.Security;
|
||||
using ItWorx.KnowledgeHub.Application.Identity;
|
||||
using ItWorx.KnowledgeHub.Application.Identity.UseCases;
|
||||
using ItWorx.KnowledgeHub.Application.Ingestion;
|
||||
using ItWorx.KnowledgeHub.Application.Platform;
|
||||
using ItWorx.KnowledgeHub.Contracts.Auth;
|
||||
using ItWorx.KnowledgeHub.Contracts.Errors;
|
||||
using Microsoft.AspNetCore.Antiforgery;
|
||||
using Microsoft.AspNetCore.Http.HttpResults;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Api.Endpoints;
|
||||
|
||||
@@ -40,18 +40,19 @@ public static class AuthEndpoints
|
||||
|
||||
private static async Task<Ok<SessionResponse>> GetSessionAsync(
|
||||
GetSessionUseCase useCase,
|
||||
IOptions<IngestionProposalOptions> ingestionOptions,
|
||||
IPlatformSettingsReader platformSettings,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var session = await useCase.ExecuteAsync(cancellationToken);
|
||||
return TypedResults.Ok(ToSessionResponse(session, ingestionOptions.Value));
|
||||
var ingestion = (await platformSettings.GetEffectiveAsync(cancellationToken)).Values.IngestionProposals;
|
||||
return TypedResults.Ok(ToSessionResponse(session, ingestion));
|
||||
}
|
||||
|
||||
private static async Task<Results<Ok<SessionResponse>, ProblemHttpResult>> LoginAsync(
|
||||
[FromBody] LoginRequest request,
|
||||
LoginUseCase loginUseCase,
|
||||
GetSessionUseCase sessionUseCase,
|
||||
IOptions<IngestionProposalOptions> ingestionOptions,
|
||||
IPlatformSettingsReader platformSettings,
|
||||
HttpContext httpContext,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -65,7 +66,8 @@ public static class AuthEndpoints
|
||||
{
|
||||
case PasswordSignInStatus.Success:
|
||||
var session = await sessionUseCase.ExecuteAsync(cancellationToken);
|
||||
return TypedResults.Ok(ToSessionResponse(session, ingestionOptions.Value));
|
||||
var ingestion = (await platformSettings.GetEffectiveAsync(cancellationToken)).Values.IngestionProposals;
|
||||
return TypedResults.Ok(ToSessionResponse(session, ingestion));
|
||||
|
||||
case PasswordSignInStatus.LockedOut:
|
||||
return TypedResults.Problem(
|
||||
@@ -124,7 +126,7 @@ public static class AuthEndpoints
|
||||
};
|
||||
}
|
||||
|
||||
private static SessionResponse ToSessionResponse(SessionInfo session, IngestionProposalOptions ingestionOptions) => new(
|
||||
private static SessionResponse ToSessionResponse(SessionInfo session, ItWorx.KnowledgeHub.Domain.Platform.IngestionProposalPolicyValues ingestionOptions) => new(
|
||||
session.Authenticated,
|
||||
session.SetupRequired,
|
||||
session.UserId,
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
using System.Globalization;
|
||||
using System.Security.Claims;
|
||||
using ItWorx.KnowledgeHub.Api.Authorization;
|
||||
using ItWorx.KnowledgeHub.Api.Security;
|
||||
using ItWorx.KnowledgeHub.Application.Audit;
|
||||
using ItWorx.KnowledgeHub.Application.Operations;
|
||||
using ItWorx.KnowledgeHub.Application.Operations.UseCases;
|
||||
using ItWorx.KnowledgeHub.Application.Platform;
|
||||
using ItWorx.KnowledgeHub.Contracts.Operations;
|
||||
using Microsoft.AspNetCore.Http.HttpResults;
|
||||
|
||||
@@ -22,31 +25,93 @@ public static class BackupsEndpoints
|
||||
.RequireAuthorization(AuthorizationPolicies.RequireAdministrator);
|
||||
|
||||
group.MapGet("", ListAsync);
|
||||
group.MapGet("/status", StatusAsync);
|
||||
group.MapPut("/{backupId}/protection", ProtectAsync)
|
||||
.AddEndpointFilter<AntiforgeryEndpointFilter>();
|
||||
|
||||
group.MapPost("", CreateAsync)
|
||||
.RequireRateLimiting(RateLimitPolicies.JobCreation)
|
||||
.AddEndpointFilter<AntiforgeryEndpointFilter>();
|
||||
|
||||
return app;
|
||||
}
|
||||
|
||||
private static async Task<Ok<BackupListResponse>> ListAsync(ListBackupsUseCase useCase, CancellationToken cancellationToken)
|
||||
private static async Task<Ok<BackupListResponse>> ListAsync(ListBackupsUseCase useCase, IBackupEvidenceStore evidence, CancellationToken cancellationToken)
|
||||
{
|
||||
var backups = await useCase.ExecuteAsync(cancellationToken);
|
||||
return TypedResults.Ok(new BackupListResponse(backups.Select(ToResponse).ToList()));
|
||||
var responses = new List<BackupSummaryResponse>(backups.Count);
|
||||
foreach (var backup in backups)
|
||||
responses.Add(ToResponse(backup, await evidence.GetAsync(backup.BackupId, cancellationToken)));
|
||||
return TypedResults.Ok(new BackupListResponse(responses));
|
||||
}
|
||||
|
||||
private static async Task<Ok<BackupSummaryResponse>> CreateAsync(
|
||||
CreateBackupUseCase useCase, ClaimsPrincipal principal, HttpContext httpContext, CancellationToken cancellationToken)
|
||||
CreateBackupUseCase useCase, IBackupEvidenceStore evidence, ClaimsPrincipal principal, HttpContext httpContext, CancellationToken cancellationToken)
|
||||
{
|
||||
var correlationId = httpContext.Items[CorrelationIdMiddleware.ItemKey] as string;
|
||||
var actorId = Guid.Parse(principal.FindFirstValue(ClaimTypes.NameIdentifier)!);
|
||||
|
||||
var summary = await useCase.ExecuteAsync(new CreateBackupCommand(actorId, correlationId), cancellationToken);
|
||||
return TypedResults.Ok(ToResponse(summary));
|
||||
return TypedResults.Ok(ToResponse(summary, await evidence.GetAsync(summary.BackupId, cancellationToken)));
|
||||
}
|
||||
|
||||
private static BackupSummaryResponse ToResponse(BackupSummary summary) =>
|
||||
private static BackupSummaryResponse ToResponse(BackupSummary summary, BackupEvidence? evidence) =>
|
||||
new(
|
||||
summary.BackupId, summary.CreatedAt, summary.TotalSizeBytes, summary.TableCount,
|
||||
summary.TotalRowCount, summary.AttachmentCount, summary.SchemaVersion, summary.ProductVersion);
|
||||
summary.TotalRowCount, summary.AttachmentCount, summary.SchemaVersion, summary.ProductVersion,
|
||||
evidence?.Protected ?? false, (evidence?.ProtectionVersion ?? 0).ToString(CultureInfo.InvariantCulture),
|
||||
evidence?.IntegrityVerifiedAt, evidence?.RestoreVerifiedAt, evidence?.LastRestoreSucceeded);
|
||||
|
||||
private static async Task<Ok<BackupStatusResponse>> StatusAsync(
|
||||
ListBackupsUseCase backups, IBackupEvidenceStore evidence, IPlatformSettingsReader settings,
|
||||
IConfiguration configuration, TimeProvider clock, CancellationToken cancellationToken)
|
||||
{
|
||||
var policy = (await settings.GetEffectiveAsync(cancellationToken)).Values.BackupPolicyExpectation;
|
||||
var items = await backups.ExecuteAsync(cancellationToken);
|
||||
var receipts = new List<BackupEvidence>();
|
||||
BackupEvidence? latestReceipt = null;
|
||||
foreach (var backup in items)
|
||||
if (await evidence.GetAsync(backup.BackupId, cancellationToken) is { } receipt)
|
||||
{
|
||||
receipts.Add(receipt);
|
||||
if (items.Count > 0 && backup.BackupId == items[0].BackupId) latestReceipt = receipt;
|
||||
}
|
||||
DateTimeOffset? last = items.Count > 0 ? items[0].CreatedAt : null;
|
||||
var integrity = receipts.Select(r => r.IntegrityVerifiedAt).Max();
|
||||
var restore = receipts.Select(r => r.RestoreVerifiedAt).Max();
|
||||
var now = clock.GetUtcNow();
|
||||
var state = last is null || latestReceipt?.IntegrityVerifiedAt is null ? "Unknown"
|
||||
: now - last > TimeSpan.FromHours(policy.ExpectedIntervalHours) ? "Stale" : "Current";
|
||||
var restoreState = restore is null ? "Unknown"
|
||||
: now - restore > TimeSpan.FromDays(policy.RestoreVerificationMaxAgeDays) ? "Stale" : "Verified";
|
||||
return TypedResults.Ok(new BackupStatusResponse(state, "External", policy.ScheduleDescription, null,
|
||||
last?.AddHours(policy.ExpectedIntervalHours), last, integrity, restore, restoreState,
|
||||
items.Sum(b => b.TotalSizeBytes), receipts.Count(r => r.Protected), policy.ExpectedIntervalHours,
|
||||
configuration.GetValue<int?>("KnowledgeHub:Backup:RetainedCount") ?? 14,
|
||||
policy.DailyRetentionCount, policy.WeeklyRetentionCount, policy.MonthlyRetentionCount,
|
||||
policy.RestoreVerificationMaxAgeDays));
|
||||
}
|
||||
|
||||
private static async Task<IResult> ProtectAsync(
|
||||
string backupId, BackupProtectionRequest request, IBackupEvidenceStore evidence,
|
||||
IAuditEventRecorder audit, HttpContext context, TimeProvider clock, CancellationToken cancellationToken)
|
||||
{
|
||||
var header = context.Request.Headers.IfMatch.ToString();
|
||||
if (header.Length < 3 || header[0] != '"' || header[^1] != '"'
|
||||
|| !int.TryParse(header[1..^1], NumberStyles.None, CultureInfo.InvariantCulture, out var version))
|
||||
return Results.Problem(statusCode: 428, title: "If-Match is verplicht.");
|
||||
var actor = Guid.Parse(context.User.FindFirstValue(ClaimTypes.NameIdentifier)!);
|
||||
BackupProtectionResult result;
|
||||
try { result = await evidence.SetProtectionAsync(backupId, request.Protected, version, actor, cancellationToken); }
|
||||
catch (IOException) { return Results.Problem(statusCode: 409, title: "Back-upopslag is in gebruik; probeer opnieuw."); }
|
||||
if (result == BackupProtectionResult.NotFound) return Results.NotFound();
|
||||
if (result == BackupProtectionResult.Conflict) return Results.Problem(statusCode: 412, title: "De bescherming is intussen gewijzigd.");
|
||||
await audit.RecordAsync(new AuditEventRecord(AuditActorKind.User, actor, "backup",
|
||||
request.Protected ? "BackupProtected" : "BackupUnprotected", "Backup", backupId,
|
||||
request.Protected ? "Back-up beschermd tegen automatische retentie." : "Back-upbescherming verwijderd.",
|
||||
context.Items[CorrelationIdMiddleware.ItemKey] as string, clock.GetUtcNow()), cancellationToken);
|
||||
var receipt = await evidence.GetAsync(backupId, cancellationToken);
|
||||
context.Response.Headers.ETag = $"\"{receipt!.ProtectionVersion}\"";
|
||||
return Results.Ok(receipt);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
using System.Globalization;
|
||||
using System.Security.Claims;
|
||||
using ItWorx.KnowledgeHub.Api.Authorization;
|
||||
using ItWorx.KnowledgeHub.Api.Security;
|
||||
using ItWorx.KnowledgeHub.Application.Common;
|
||||
using ItWorx.KnowledgeHub.Application.Identity;
|
||||
using ItWorx.KnowledgeHub.Application.Sources;
|
||||
using ItWorx.KnowledgeHub.Contracts.Sources;
|
||||
using ItWorx.KnowledgeHub.Domain.Sources;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Api.Endpoints;
|
||||
|
||||
public static class DataSourcesEndpoints
|
||||
{
|
||||
public static IEndpointRouteBuilder MapDataSourcesEndpoints(this IEndpointRouteBuilder app)
|
||||
{
|
||||
var group = app.MapGroup("/api/v1/admin/data-sources").WithTags("DataSources").RequireAuthorization(AuthorizationPolicies.RequireAdministrator);
|
||||
group.MapGet("", async (DataSourceUseCases useCases, IWorkspaceRepository workspaces, TimeProvider clock, [FromQuery] Guid? cursor, [FromQuery] int? limit, CancellationToken cancellationToken) =>
|
||||
{
|
||||
var workspace = await workspaces.GetDefaultWorkspaceIdAsync(cancellationToken);
|
||||
if (workspace is null) return Results.NotFound();
|
||||
var pageSize = Math.Clamp(limit ?? 50, 1, 100);
|
||||
var rows = await useCases.ListAsync(workspace.Value, cursor, pageSize + 1, cancellationToken);
|
||||
var items = rows.Take(pageSize).Select(source => ToResponse(source, clock.GetUtcNow())).ToArray();
|
||||
return Results.Ok(new DataSourceListResponse(items, rows.Count > pageSize ? items[^1].Id.ToString() : null));
|
||||
});
|
||||
group.MapPost("", async (RegisterDataSourceRequest request, DataSourceUseCases useCases, IWorkspaceRepository workspaces, HttpContext context, TimeProvider clock, CancellationToken cancellationToken) =>
|
||||
{
|
||||
var workspace = await workspaces.GetDefaultWorkspaceIdAsync(cancellationToken);
|
||||
if (workspace is null) return Results.NotFound();
|
||||
var result = await useCases.RegisterAsync(workspace.Value, request.SourceKey, request.DisplayName, request.Kind, request.ExpectedIntervalHours, Actor(context), Correlation(context), cancellationToken);
|
||||
return Result(result, context, clock);
|
||||
}).AddEndpointFilter<AntiforgeryEndpointFilter>();
|
||||
group.MapPut("/{id:guid}", async (Guid id, ConfigureDataSourceRequest request, DataSourceUseCases useCases, IWorkspaceRepository workspaces, HttpContext context, TimeProvider clock, CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (!Expected(context, out var expected)) return Results.Problem(statusCode: 428, title: "If-Match is verplicht.");
|
||||
var workspace = await workspaces.GetDefaultWorkspaceIdAsync(cancellationToken);
|
||||
if (workspace is null) return Results.NotFound();
|
||||
return Result(await useCases.ConfigureAsync(workspace.Value, id, expected, request.DisplayName, request.Enabled, request.ExpectedIntervalHours, Actor(context), Correlation(context), cancellationToken), context, clock);
|
||||
}).AddEndpointFilter<AntiforgeryEndpointFilter>();
|
||||
group.MapPost("/{id:guid}/observations", async (Guid id, ReportDataSourceRequest request, DataSourceUseCases useCases, IWorkspaceRepository workspaces, HttpContext context, TimeProvider clock, CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (!Expected(context, out var expected)) return Results.Problem(statusCode: 428, title: "If-Match is verplicht.");
|
||||
var workspace = await workspaces.GetDefaultWorkspaceIdAsync(cancellationToken);
|
||||
if (workspace is null) return Results.NotFound();
|
||||
return Result(await useCases.ReportAsync(workspace.Value, id, expected, request.Success, request.RecordCount, request.ErrorCode, request.CorrelationId, Actor(context), Correlation(context), cancellationToken), context, clock);
|
||||
}).AddEndpointFilter<AntiforgeryEndpointFilter>();
|
||||
return app;
|
||||
}
|
||||
|
||||
private static bool Expected(HttpContext context, out string value)
|
||||
{
|
||||
var header = context.Request.Headers.IfMatch.ToString();
|
||||
value = header.Length > 2 && header.StartsWith('"') && header.EndsWith('"') ? header[1..^1] : "";
|
||||
return int.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out var version) && version > 0;
|
||||
}
|
||||
private static Guid Actor(HttpContext context) => Guid.Parse(context.User.FindFirstValue(ClaimTypes.NameIdentifier)!);
|
||||
private static string? Correlation(HttpContext context) => context.Items[CorrelationIdMiddleware.ItemKey] as string;
|
||||
private static IResult Result(OperationResultValue<DataSource> result, HttpContext context, TimeProvider clock)
|
||||
{
|
||||
if (result.Status == OperationStatus.NotFound) return Results.NotFound();
|
||||
if (result.Status == OperationStatus.ConcurrencyConflict) return Results.Problem(statusCode: 412, title: "De bron is gewijzigd; laad de actuele versie.");
|
||||
if (result.Status != OperationStatus.Success) return Results.Problem(statusCode: 400, title: "Ongeldige bronconfiguratie.");
|
||||
var response = ToResponse(result.Value!, clock.GetUtcNow());
|
||||
context.Response.Headers.ETag = $"\"{response.ETag}\"";
|
||||
return Results.Ok(response);
|
||||
}
|
||||
private static DataSourceResponse ToResponse(DataSource source, DateTimeOffset now) => new(source.Id, source.SourceKey, source.DisplayName, source.Kind.ToString(), source.Enabled,
|
||||
source.GetState(now), source.LastAttemptAt is null ? "Unknown" : source.LastErrorCode == "authentication" ? "NeedsAuthorization" : "Reported",
|
||||
"External", source.ExpectedIntervalHours, source.LastAttemptAt, source.LastSuccessAt, source.LastFailureAt, source.DataObservedAt, source.RecordCount,
|
||||
source.LastErrorCode, source.LastCorrelationId, source.Version.ToString(CultureInfo.InvariantCulture), ["configure", "record-observation"]);
|
||||
}
|
||||
@@ -7,11 +7,11 @@ using ItWorx.KnowledgeHub.Application.Common;
|
||||
using ItWorx.KnowledgeHub.Application.Identity;
|
||||
using ItWorx.KnowledgeHub.Application.Ingestion;
|
||||
using ItWorx.KnowledgeHub.Application.Ingestion.UseCases;
|
||||
using ItWorx.KnowledgeHub.Application.Platform;
|
||||
using ItWorx.KnowledgeHub.Contracts.Ingestion;
|
||||
using ItWorx.KnowledgeHub.Domain.Identity;
|
||||
using ItWorx.KnowledgeHub.Domain.Ingestion;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Api.Endpoints;
|
||||
|
||||
@@ -48,10 +48,11 @@ public static class IngestionProposalsEndpoints
|
||||
}
|
||||
|
||||
private static async Task<IResult> CreateAsync([FromBody] CreateIngestionProposalRequest request, IWorkspaceRepository workspaces,
|
||||
CreateIngestionProposalUseCase useCase, IIngestionProposalRepository proposals, IngestionMetrics metrics, IOptions<IngestionProposalOptions> options,
|
||||
CreateIngestionProposalUseCase useCase, IIngestionProposalRepository proposals, IngestionMetrics metrics, IPlatformSettingsReader platformSettings,
|
||||
ClaimsPrincipal principal, HttpContext context, CancellationToken ct)
|
||||
{
|
||||
if (!options.Value.Enabled) return Disabled();
|
||||
var policy = (await platformSettings.GetEffectiveAsync(ct)).Values.IngestionProposals;
|
||||
if (!policy.Enabled) return Disabled();
|
||||
if (string.IsNullOrWhiteSpace(context.Request.Headers["Idempotency-Key"].ToString()))
|
||||
return Results.Problem(statusCode: 400, detail: "Idempotency-Key-header is vereist.");
|
||||
var workspace = await workspaces.GetDefaultWorkspaceIdAsync(ct);
|
||||
@@ -73,7 +74,8 @@ public static class IngestionProposalsEndpoints
|
||||
}
|
||||
}
|
||||
var result = await useCase.ExecuteAsync(new(workspace.Value, Actor(principal), request.Manifest,
|
||||
request.CorrelationId ?? Correlation(context), context.Request.Headers["Idempotency-Key"].ToString(), publication), ct);
|
||||
request.CorrelationId ?? Correlation(context), context.Request.Headers["Idempotency-Key"].ToString(), publication,
|
||||
policy.FailedRetryLimit), ct);
|
||||
metrics.RecordProposal(result.Status == OperationStatus.Success ? "success" : "failure",
|
||||
result.Status == OperationStatus.Success ? "none" : "validation");
|
||||
if (result.Status != OperationStatus.Success) return Results.Problem(statusCode: 422, detail: string.Join(' ', result.Errors));
|
||||
@@ -84,11 +86,12 @@ public static class IngestionProposalsEndpoints
|
||||
|
||||
private static async Task<IResult> ListAsync(IWorkspaceRepository workspaces, IWorkspaceMembershipRepository memberships,
|
||||
IIngestionProposalRepository proposals,
|
||||
IOptions<IngestionProposalOptions> options, HttpContext context, [FromQuery] string? status, [FromQuery] Guid? cursor,
|
||||
IPlatformSettingsReader platformSettings, HttpContext context, [FromQuery] string? status, [FromQuery] Guid? cursor,
|
||||
[FromQuery] int? limit, [FromQuery] int? page, [FromQuery] int? pageSize, [FromQuery] Guid? projectId,
|
||||
[FromQuery] Guid? submitterId, [FromQuery] int? minAgeDays, [FromQuery] string? risk, [FromQuery] string? documentType, CancellationToken ct)
|
||||
{
|
||||
if (!options.Value.Enabled) return Disabled();
|
||||
var policy = (await platformSettings.GetEffectiveAsync(ct)).Values.IngestionProposals;
|
||||
if (!policy.Enabled) return Disabled();
|
||||
var workspace = await workspaces.GetDefaultWorkspaceIdAsync(ct);
|
||||
if (workspace is null) return Results.Ok(new IngestionProposalListResponse([], null, 1, 50, 0, Correlation(context)));
|
||||
if (!TryStatus(status, out var parsed)) return Results.Problem(statusCode: 400, detail: "Ongeldige statusfilter.");
|
||||
@@ -116,9 +119,10 @@ public static class IngestionProposalsEndpoints
|
||||
|
||||
private static async Task<IResult> GetAsync(Guid proposalId, IWorkspaceRepository workspaces, IWorkspaceMembershipRepository memberships,
|
||||
IIngestionProposalRepository proposals,
|
||||
IOptions<IngestionProposalOptions> options, ClaimsPrincipal principal, HttpContext context, CancellationToken ct)
|
||||
IPlatformSettingsReader platformSettings, ClaimsPrincipal principal, HttpContext context, CancellationToken ct)
|
||||
{
|
||||
if (!options.Value.Enabled) return Disabled();
|
||||
var policy = (await platformSettings.GetEffectiveAsync(ct)).Values.IngestionProposals;
|
||||
if (!policy.Enabled) return Disabled();
|
||||
var workspace = await workspaces.GetDefaultWorkspaceIdAsync(ct);
|
||||
if (workspace is null) return NotFound();
|
||||
var proposal = await proposals.FindReadOnlyAsync(workspace.Value, proposalId, ct);
|
||||
@@ -132,10 +136,11 @@ public static class IngestionProposalsEndpoints
|
||||
|
||||
private static async Task<IResult> PatchAsync(Guid proposalId, [FromBody] PatchIngestionProposalRequest request, IWorkspaceRepository workspaces,
|
||||
PatchIngestionProposalUseCase useCase, ApplyIngestionProposalAnalysisUseCase analysisUseCase,
|
||||
IWorkspaceMembershipRepository memberships, IIngestionProposalRepository proposals, IOptions<IngestionProposalOptions> options,
|
||||
IWorkspaceMembershipRepository memberships, IIngestionProposalRepository proposals, IPlatformSettingsReader platformSettings,
|
||||
ClaimsPrincipal principal, HttpContext context, CancellationToken ct)
|
||||
{
|
||||
if (!options.Value.Enabled) return Disabled();
|
||||
var policy = (await platformSettings.GetEffectiveAsync(ct)).Values.IngestionProposals;
|
||||
if (!policy.Enabled) return Disabled();
|
||||
var workspace = await workspaces.GetDefaultWorkspaceIdAsync(ct); if (workspace is null) return NotFound();
|
||||
var proposal = await proposals.FindReadOnlyAsync(workspace.Value, proposalId, ct);
|
||||
if (proposal is null || (IsServiceAccount(principal) && proposal.IntegrationActorId != Actor(principal))) return NotFound();
|
||||
@@ -160,19 +165,20 @@ public static class IngestionProposalsEndpoints
|
||||
}
|
||||
|
||||
private static Task<IResult> RequestChangesAsync(Guid proposalId, [FromBody] IngestionProposalDecisionRequest request, IWorkspaceRepository workspaces,
|
||||
DecideIngestionProposalUseCase useCase, IIngestionProposalRepository proposals, IOptions<IngestionProposalOptions> options, ClaimsPrincipal principal, HttpContext context, CancellationToken ct) =>
|
||||
DecideAsync(IngestionDecision.RequestChanges, proposalId, request, workspaces, useCase, proposals, options, principal, context, ct);
|
||||
DecideIngestionProposalUseCase useCase, IIngestionProposalRepository proposals, IPlatformSettingsReader platformSettings, ClaimsPrincipal principal, HttpContext context, CancellationToken ct) =>
|
||||
DecideAsync(IngestionDecision.RequestChanges, proposalId, request, workspaces, useCase, proposals, platformSettings, principal, context, ct);
|
||||
private static Task<IResult> ApproveAsync(Guid proposalId, [FromBody] IngestionProposalDecisionRequest request, IWorkspaceRepository workspaces,
|
||||
DecideIngestionProposalUseCase useCase, IIngestionProposalRepository proposals, IOptions<IngestionProposalOptions> options, ClaimsPrincipal principal, HttpContext context, CancellationToken ct) =>
|
||||
DecideAsync(IngestionDecision.Approve, proposalId, request, workspaces, useCase, proposals, options, principal, context, ct);
|
||||
DecideIngestionProposalUseCase useCase, IIngestionProposalRepository proposals, IPlatformSettingsReader platformSettings, ClaimsPrincipal principal, HttpContext context, CancellationToken ct) =>
|
||||
DecideAsync(IngestionDecision.Approve, proposalId, request, workspaces, useCase, proposals, platformSettings, principal, context, ct);
|
||||
private static Task<IResult> RejectAsync(Guid proposalId, [FromBody] IngestionProposalDecisionRequest request, IWorkspaceRepository workspaces,
|
||||
DecideIngestionProposalUseCase useCase, IIngestionProposalRepository proposals, IOptions<IngestionProposalOptions> options, ClaimsPrincipal principal, HttpContext context, CancellationToken ct) =>
|
||||
DecideAsync(IngestionDecision.Reject, proposalId, request, workspaces, useCase, proposals, options, principal, context, ct);
|
||||
DecideIngestionProposalUseCase useCase, IIngestionProposalRepository proposals, IPlatformSettingsReader platformSettings, ClaimsPrincipal principal, HttpContext context, CancellationToken ct) =>
|
||||
DecideAsync(IngestionDecision.Reject, proposalId, request, workspaces, useCase, proposals, platformSettings, principal, context, ct);
|
||||
|
||||
private static async Task<IResult> DecideAsync(IngestionDecision decision, Guid proposalId, IngestionProposalDecisionRequest request, IWorkspaceRepository workspaces,
|
||||
DecideIngestionProposalUseCase useCase, IIngestionProposalRepository proposals, IOptions<IngestionProposalOptions> options, ClaimsPrincipal principal, HttpContext context, CancellationToken ct)
|
||||
DecideIngestionProposalUseCase useCase, IIngestionProposalRepository proposals, IPlatformSettingsReader platformSettings, ClaimsPrincipal principal, HttpContext context, CancellationToken ct)
|
||||
{
|
||||
if (!options.Value.Enabled) return Disabled(); var workspace = await workspaces.GetDefaultWorkspaceIdAsync(ct); if (workspace is null) return NotFound();
|
||||
var policy = (await platformSettings.GetEffectiveAsync(ct)).Values.IngestionProposals;
|
||||
if (!policy.Enabled) return Disabled(); var workspace = await workspaces.GetDefaultWorkspaceIdAsync(ct); if (workspace is null) return NotFound();
|
||||
if (ETagHelper.FromHeaderValue(context.Request.Headers.IfMatch) is null)
|
||||
return Results.Problem(statusCode: 428, detail: "If-Match-header is vereist.");
|
||||
var started = System.Diagnostics.Stopwatch.GetTimestamp();
|
||||
@@ -183,9 +189,10 @@ public static class IngestionProposalsEndpoints
|
||||
}
|
||||
|
||||
private static async Task<IResult> RetryAsync(Guid proposalId, IWorkspaceRepository workspaces, RetryIngestionProposalUseCase useCase,
|
||||
IIngestionProposalRepository proposals, IOptions<IngestionProposalOptions> options, ClaimsPrincipal principal, HttpContext context, CancellationToken ct)
|
||||
IIngestionProposalRepository proposals, IPlatformSettingsReader platformSettings, ClaimsPrincipal principal, HttpContext context, CancellationToken ct)
|
||||
{
|
||||
if (!options.Value.Enabled) return Disabled(); var workspace = await workspaces.GetDefaultWorkspaceIdAsync(ct); if (workspace is null) return NotFound();
|
||||
var policy = (await platformSettings.GetEffectiveAsync(ct)).Values.IngestionProposals;
|
||||
if (!policy.Enabled) return Disabled(); var workspace = await workspaces.GetDefaultWorkspaceIdAsync(ct); if (workspace is null) return NotFound();
|
||||
if (ETagHelper.FromHeaderValue(context.Request.Headers.IfMatch) is null)
|
||||
return Results.Problem(statusCode: 428, detail: "If-Match-header is vereist.");
|
||||
var result = await useCase.ExecuteAsync(new(workspace.Value, proposalId, Actor(principal), ETag(context)), ct);
|
||||
@@ -194,9 +201,10 @@ public static class IngestionProposalsEndpoints
|
||||
|
||||
private static async Task<IResult> PublishAsync(Guid proposalId, IWorkspaceRepository workspaces,
|
||||
PublishIngestionProposalUseCase useCase, IIngestionProposalRepository proposals, IngestionMetrics metrics,
|
||||
IOptions<IngestionProposalOptions> options, ClaimsPrincipal principal, HttpContext context, CancellationToken ct)
|
||||
IPlatformSettingsReader platformSettings, ClaimsPrincipal principal, HttpContext context, CancellationToken ct)
|
||||
{
|
||||
if (!options.Value.Enabled) return Disabled();
|
||||
var policy = (await platformSettings.GetEffectiveAsync(ct)).Values.IngestionProposals;
|
||||
if (!policy.Enabled) return Disabled();
|
||||
if (IsServiceAccount(principal)) return Results.Problem(statusCode: 403, detail: "Serviceaccounts mogen geen voorstellen publiceren.");
|
||||
var workspace = await workspaces.GetDefaultWorkspaceIdAsync(ct); if (workspace is null) return NotFound();
|
||||
if (ETagHelper.FromHeaderValue(context.Request.Headers.IfMatch) is null)
|
||||
@@ -230,16 +238,17 @@ public static class IngestionProposalsEndpoints
|
||||
|
||||
private static async Task<IResult> UploadSourceAsync(Guid proposalId, IFormFile file, IWorkspaceRepository workspaces,
|
||||
StageIngestionProposalSourceUseCase useCase, IIngestionProposalRepository proposals, IngestionMetrics metrics,
|
||||
IOptions<IngestionProposalOptions> options, ClaimsPrincipal principal, HttpContext context, CancellationToken ct)
|
||||
IPlatformSettingsReader platformSettings, ClaimsPrincipal principal, HttpContext context, CancellationToken ct)
|
||||
{
|
||||
if (!options.Value.Enabled) return Disabled(); var workspace = await workspaces.GetDefaultWorkspaceIdAsync(ct); if (workspace is null) return NotFound();
|
||||
var policy = (await platformSettings.GetEffectiveAsync(ct)).Values.IngestionProposals;
|
||||
if (!policy.Enabled) return Disabled(); var workspace = await workspaces.GetDefaultWorkspaceIdAsync(ct); if (workspace is null) return NotFound();
|
||||
if (ETagHelper.FromHeaderValue(context.Request.Headers.IfMatch) is null)
|
||||
return Results.Problem(statusCode: 428, detail: "If-Match-header is vereist.");
|
||||
if (file.Length > options.Value.MaximumSourceBytes) return Results.Problem(statusCode: 413, detail: "Bron is groter dan de toegestane limiet.");
|
||||
if (file.Length > policy.MaximumSourceBytes) return Results.Problem(statusCode: 413, detail: "Bron is groter dan de toegestane limiet.");
|
||||
await using var source = file.OpenReadStream();
|
||||
var started = System.Diagnostics.Stopwatch.GetTimestamp();
|
||||
var result = await useCase.ExecuteAsync(new(workspace.Value, proposalId, Actor(principal), file.FileName, file.ContentType,
|
||||
source, Correlation(context), options.Value.MaximumSourceCount, ETag(context), ActorKind(principal), AuditSource(principal)), ct);
|
||||
source, Correlation(context), policy.MaximumSourceCount, ETag(context), ActorKind(principal), AuditSource(principal)), ct);
|
||||
metrics.RecordStageDuration("source_staging", result.Status == OperationStatus.Success ? "success" : "failure",
|
||||
System.Diagnostics.Stopwatch.GetElapsedTime(started).TotalSeconds);
|
||||
if (result.Status != OperationStatus.Success) return await ToMutationResult(result, proposals, workspace.Value, proposalId, context, ct);
|
||||
@@ -251,9 +260,10 @@ public static class IngestionProposalsEndpoints
|
||||
|
||||
private static async Task<IResult> SubmitManualAsync(Guid proposalId, IWorkspaceRepository workspaces,
|
||||
SubmitManualIngestionProposalUseCase useCase, IIngestionProposalRepository proposals,
|
||||
IOptions<IngestionProposalOptions> options, ClaimsPrincipal principal, HttpContext context, CancellationToken ct)
|
||||
IPlatformSettingsReader platformSettings, ClaimsPrincipal principal, HttpContext context, CancellationToken ct)
|
||||
{
|
||||
if (!options.Value.Enabled) return Disabled();
|
||||
var policy = (await platformSettings.GetEffectiveAsync(ct)).Values.IngestionProposals;
|
||||
if (!policy.Enabled) return Disabled();
|
||||
if (IsServiceAccount(principal)) return Results.Forbid();
|
||||
var workspace = await workspaces.GetDefaultWorkspaceIdAsync(ct);
|
||||
if (workspace is null) return NotFound();
|
||||
|
||||
@@ -0,0 +1,277 @@
|
||||
using System.Security.Claims;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using ItWorx.KnowledgeHub.Api.Authorization;
|
||||
using ItWorx.KnowledgeHub.Api.Security;
|
||||
using ItWorx.KnowledgeHub.Application.Identity;
|
||||
using ItWorx.KnowledgeHub.Application.Jobs;
|
||||
using ItWorx.KnowledgeHub.Application.Jobs.UseCases;
|
||||
using ItWorx.KnowledgeHub.Contracts.Errors;
|
||||
using ItWorx.KnowledgeHub.Contracts.Jobs;
|
||||
using ItWorx.KnowledgeHub.Domain.Identity;
|
||||
using ItWorx.KnowledgeHub.Domain.Jobs;
|
||||
using Microsoft.AspNetCore.Http.HttpResults;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Api.Endpoints;
|
||||
|
||||
/// <summary>Cookie-admin-only control surface for durable jobs. Mutations require CSRF;
|
||||
/// event streams expose only the latest persisted safe job snapshot.</summary>
|
||||
public static class JobsEndpoints
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
Converters = { new JsonStringEnumConverter(JsonNamingPolicy.SnakeCaseLower) }
|
||||
};
|
||||
|
||||
public static IEndpointRouteBuilder MapJobsEndpoints(this IEndpointRouteBuilder app)
|
||||
{
|
||||
var group = app.MapGroup("/api/v1/jobs")
|
||||
.WithTags("Jobs")
|
||||
.RequireAuthorization(AuthorizationPolicies.RequireAdministrator);
|
||||
|
||||
group.MapPost("/course-tagging", QueueCourseTaggingAsync)
|
||||
.RequireRateLimiting(RateLimitPolicies.JobCreation)
|
||||
.AddEndpointFilter<StrictAntiforgeryEndpointFilter>();
|
||||
group.MapPost("/backups", QueueBackupAsync)
|
||||
.RequireRateLimiting(RateLimitPolicies.JobCreation)
|
||||
.AddEndpointFilter<StrictAntiforgeryEndpointFilter>();
|
||||
group.MapGet("", ListAsync);
|
||||
group.MapGet("/{jobId:guid}", GetAsync);
|
||||
group.MapPost("/{jobId:guid}/cancel", CancelAsync)
|
||||
.AddEndpointFilter<StrictAntiforgeryEndpointFilter>();
|
||||
group.MapGet("/{jobId:guid}/events", EventsAsync);
|
||||
return app;
|
||||
}
|
||||
|
||||
private static async Task<IResult> QueueCourseTaggingAsync(
|
||||
[FromBody] QueueCourseTaggingJobRequest request,
|
||||
QueueBackgroundJobUseCase useCase,
|
||||
IWorkspaceRepository workspaces,
|
||||
HttpContext context,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(context.Request.Headers["Idempotency-Key"]))
|
||||
return Results.Problem(statusCode: 400, type: ProblemTypes.Validation, title: "Idempotency-Key is verplicht.");
|
||||
var workspaceId = await workspaces.GetDefaultWorkspaceIdAsync(cancellationToken);
|
||||
if (workspaceId is null)
|
||||
return Results.Problem(statusCode: 404, type: ProblemTypes.NotFound);
|
||||
|
||||
EnqueueJobResult result;
|
||||
try
|
||||
{
|
||||
result = await useCase.QueueCourseTaggingAsync(
|
||||
new QueueCourseTaggingJobCommand(
|
||||
workspaceId.Value, CurrentUserId(context.User), request.RootPageId, request.Tag,
|
||||
context.Request.Headers["Idempotency-Key"].ToString(), CorrelationId(context)),
|
||||
cancellationToken);
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
return Results.Problem(statusCode: 400, type: ProblemTypes.Validation, title: "De cursustagtaak is ongeldig.");
|
||||
}
|
||||
return ToEnqueueResult(result);
|
||||
}
|
||||
|
||||
private static async Task<IResult> QueueBackupAsync(
|
||||
QueueBackupJobRequest _,
|
||||
QueueBackgroundJobUseCase useCase,
|
||||
IWorkspaceRepository workspaces,
|
||||
HttpContext context,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(context.Request.Headers["Idempotency-Key"]))
|
||||
return Results.Problem(statusCode: 400, type: ProblemTypes.Validation, title: "Idempotency-Key is verplicht.");
|
||||
var workspaceId = await workspaces.GetDefaultWorkspaceIdAsync(cancellationToken);
|
||||
if (workspaceId is null)
|
||||
return Results.Problem(statusCode: 404, type: ProblemTypes.NotFound);
|
||||
EnqueueJobResult result;
|
||||
try
|
||||
{
|
||||
result = await useCase.QueueBackupAsync(
|
||||
new QueueJobCommand(workspaceId.Value, CurrentUserId(context.User),
|
||||
context.Request.Headers["Idempotency-Key"].ToString(), CorrelationId(context)), cancellationToken);
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
return Results.Problem(statusCode: 400, type: ProblemTypes.Validation, title: "Een geldige Idempotency-Key is verplicht.");
|
||||
}
|
||||
return ToEnqueueResult(result);
|
||||
}
|
||||
|
||||
private static async Task<Results<Ok<BackgroundJobListResponse>, ProblemHttpResult>> ListAsync(
|
||||
BackgroundJobQueries queries,
|
||||
IWorkspaceRepository workspaces,
|
||||
[FromQuery] int? limit,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var workspaceId = await workspaces.GetDefaultWorkspaceIdAsync(cancellationToken);
|
||||
if (workspaceId is null)
|
||||
return TypedResults.Problem(statusCode: 404, type: ProblemTypes.NotFound);
|
||||
var items = await queries.ListAsync(workspaceId.Value, limit ?? 50, cancellationToken);
|
||||
return TypedResults.Ok(new BackgroundJobListResponse(items.Select(ToResponse).ToList()));
|
||||
}
|
||||
|
||||
private static async Task<Results<Ok<BackgroundJobResponse>, NotFound>> GetAsync(
|
||||
Guid jobId,
|
||||
BackgroundJobQueries queries,
|
||||
IWorkspaceRepository workspaces,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var workspaceId = await workspaces.GetDefaultWorkspaceIdAsync(cancellationToken);
|
||||
if (workspaceId is null)
|
||||
return TypedResults.NotFound();
|
||||
var job = await queries.GetAsync(workspaceId.Value, jobId, cancellationToken);
|
||||
return job is null ? TypedResults.NotFound() : TypedResults.Ok(ToResponse(job));
|
||||
}
|
||||
|
||||
private static async Task<IResult> CancelAsync(
|
||||
Guid jobId,
|
||||
CancelBackgroundJobUseCase useCase,
|
||||
IWorkspaceRepository workspaces,
|
||||
HttpContext context,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var workspaceId = await workspaces.GetDefaultWorkspaceIdAsync(cancellationToken);
|
||||
if (workspaceId is null)
|
||||
return Results.NotFound();
|
||||
var (status, job) = await useCase.ExecuteAsync(
|
||||
workspaceId.Value, jobId, CurrentUserId(context.User), CorrelationId(context), cancellationToken);
|
||||
if (status == CancelBackgroundJobStatus.NotFound || job is null)
|
||||
return Results.NotFound();
|
||||
return Results.Ok(ToResponse(job));
|
||||
}
|
||||
|
||||
private static async Task<IResult> EventsAsync(
|
||||
Guid jobId,
|
||||
BackgroundJobQueries queries,
|
||||
IWorkspaceRepository workspaces,
|
||||
IUserAccountManager users,
|
||||
IWorkspaceMembershipRepository memberships,
|
||||
BackgroundJobsOptions options,
|
||||
TimeProvider timeProvider,
|
||||
HttpContext context,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var workspaceId = await workspaces.GetDefaultWorkspaceIdAsync(cancellationToken);
|
||||
if (workspaceId is null)
|
||||
return Results.NotFound();
|
||||
var initial = await queries.GetAsync(workspaceId.Value, jobId, cancellationToken);
|
||||
if (initial is null)
|
||||
return Results.NotFound();
|
||||
|
||||
context.Response.StatusCode = StatusCodes.Status200OK;
|
||||
context.Response.ContentType = "text/event-stream; charset=utf-8";
|
||||
context.Response.Headers["Cache-Control"] = "no-cache, no-transform";
|
||||
context.Response.Headers.Append("X-Accel-Buffering", "no");
|
||||
var actorId = CurrentUserId(context.User);
|
||||
var lastVersion = -1L;
|
||||
var lastHeartbeat = timeProvider.GetUtcNow();
|
||||
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
var actor = await users.FindByIdAsync(actorId, cancellationToken);
|
||||
var role = actor is { Enabled: true }
|
||||
? await memberships.GetRoleAsync(workspaceId.Value, actorId, cancellationToken)
|
||||
: null;
|
||||
if (role != WorkspaceRole.Administrator)
|
||||
break;
|
||||
|
||||
var job = await queries.GetAsync(workspaceId.Value, jobId, cancellationToken);
|
||||
if (job is null)
|
||||
break;
|
||||
if (job.IsTerminal)
|
||||
{
|
||||
// Terminal state must close every stream, even if a concurrent
|
||||
// worker update left the terminal snapshot at the last event ID.
|
||||
await WriteEventAsync(context, "terminal", job, cancellationToken);
|
||||
break;
|
||||
}
|
||||
if (job.Version != lastVersion)
|
||||
{
|
||||
var eventName = lastVersion < 0 ? "job" : "progress";
|
||||
await WriteEventAsync(context, eventName, job, cancellationToken);
|
||||
lastVersion = job.Version;
|
||||
lastHeartbeat = timeProvider.GetUtcNow();
|
||||
}
|
||||
else if (timeProvider.GetUtcNow() - lastHeartbeat >= TimeSpan.FromSeconds(options.SseHeartbeatSeconds))
|
||||
{
|
||||
await context.Response.WriteAsync(": heartbeat\n\n", cancellationToken);
|
||||
await context.Response.Body.FlushAsync(cancellationToken);
|
||||
lastHeartbeat = timeProvider.GetUtcNow();
|
||||
}
|
||||
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(options.SsePollMilliseconds), timeProvider, cancellationToken);
|
||||
}
|
||||
return Results.Empty;
|
||||
}
|
||||
|
||||
private static async Task WriteEventAsync(HttpContext context, string eventName, BackgroundJob job, CancellationToken cancellationToken)
|
||||
{
|
||||
var data = JsonSerializer.Serialize(ToResponse(job), JsonOptions);
|
||||
await context.Response.WriteAsync($"id: {job.Version}\nevent: {eventName}\ndata: {data}\n\n", cancellationToken);
|
||||
await context.Response.Body.FlushAsync(cancellationToken);
|
||||
}
|
||||
|
||||
private static IResult ToEnqueueResult(EnqueueJobResult result) => result.Status switch
|
||||
{
|
||||
EnqueueJobStatus.Created or EnqueueJobStatus.Existing when result.Job is not null =>
|
||||
Results.Accepted($"/api/v1/jobs/{result.Job.Id:D}", ToResponse(result.Job)),
|
||||
EnqueueJobStatus.IdempotencyConflict => Results.Problem(
|
||||
statusCode: 409, type: ProblemTypes.IdempotencyKeyConflict,
|
||||
title: "Idempotency-Key is al gebruikt voor een andere aanvraag."),
|
||||
EnqueueJobStatus.RootNotFound => Results.Problem(statusCode: 404, type: ProblemTypes.PageNotFound, title: "Hoofdpagina niet gevonden."),
|
||||
EnqueueJobStatus.TooManyPages => Results.Problem(
|
||||
statusCode: 413, type: ProblemTypes.Validation,
|
||||
title: "De cursuspaginaboom overschrijdt de ingestelde taaklimiet."),
|
||||
EnqueueJobStatus.QueueFull => Results.Problem(
|
||||
statusCode: 429, title: "De taaklimiet is bereikt. Wacht tot lopende taken klaar zijn of het dagbudget is hersteld."),
|
||||
_ => Results.Problem(statusCode: 500, title: "De taak kon niet worden ingepland.")
|
||||
};
|
||||
|
||||
private static BackgroundJobResponse ToResponse(BackgroundJob job)
|
||||
{
|
||||
var errors = Array.Empty<BackgroundJobErrorResponse>();
|
||||
try
|
||||
{
|
||||
errors = (JsonSerializer.Deserialize<BackgroundJobError[]>(job.ErrorsJson) ?? [])
|
||||
.Select(error => new BackgroundJobErrorResponse(error.Code, error.Message, error.ItemId))
|
||||
.ToArray();
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
// A damaged error summary must not turn the read API into a 500 or expose raw storage.
|
||||
}
|
||||
JsonElement? result = null;
|
||||
if (!string.IsNullOrWhiteSpace(job.ResultJson))
|
||||
{
|
||||
try { result = JsonDocument.Parse(job.ResultJson).RootElement.Clone(); }
|
||||
catch (JsonException) { result = null; }
|
||||
}
|
||||
|
||||
return new BackgroundJobResponse(
|
||||
job.Id, job.WorkspaceId, job.ActorId,
|
||||
job.Type switch { BackgroundJobType.CourseTagging => "course-tagging", BackgroundJobType.Backup => "backup", _ => "unknown" },
|
||||
ToStatus(job.Status), job.CreatedAt, job.StartedAt, job.CompletedAt,
|
||||
job.CancelRequestedAt, job.CancelRequestedAt is not null,
|
||||
job.Progress, job.Total, job.Processed, job.Succeeded, job.Skipped, job.Failed,
|
||||
errors, result, job.CorrelationId, job.Version);
|
||||
}
|
||||
|
||||
private static string ToStatus(BackgroundJobStatus status) => status switch
|
||||
{
|
||||
BackgroundJobStatus.Queued => "queued",
|
||||
BackgroundJobStatus.Running => "running",
|
||||
BackgroundJobStatus.Succeeded => "succeeded",
|
||||
BackgroundJobStatus.PartiallySucceeded => "partially_succeeded",
|
||||
BackgroundJobStatus.Failed => "failed",
|
||||
BackgroundJobStatus.Cancelled => "cancelled",
|
||||
_ => "failed"
|
||||
};
|
||||
|
||||
private static Guid CurrentUserId(ClaimsPrincipal principal) =>
|
||||
Guid.Parse(principal.FindFirstValue(ClaimTypes.NameIdentifier)!);
|
||||
|
||||
private static string? CorrelationId(HttpContext context) =>
|
||||
context.Items[CorrelationIdMiddleware.ItemKey] as string;
|
||||
}
|
||||
@@ -0,0 +1,276 @@
|
||||
using System.Globalization;
|
||||
using System.Security.Claims;
|
||||
using ItWorx.KnowledgeHub.Api.Authorization;
|
||||
using ItWorx.KnowledgeHub.Api.Knowledge;
|
||||
using ItWorx.KnowledgeHub.Api.Security;
|
||||
using ItWorx.KnowledgeHub.Application.Identity;
|
||||
using ItWorx.KnowledgeHub.Application.Platform;
|
||||
using ItWorx.KnowledgeHub.Application.Platform.UseCases;
|
||||
using ItWorx.KnowledgeHub.Contracts.Errors;
|
||||
using ItWorx.KnowledgeHub.Contracts.Platform;
|
||||
using ItWorx.KnowledgeHub.Domain.Identity;
|
||||
using ItWorx.KnowledgeHub.Domain.Platform;
|
||||
using Microsoft.AspNetCore.Http.HttpResults;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Api.Endpoints;
|
||||
|
||||
public static class PlatformEndpoints
|
||||
{
|
||||
public static IEndpointRouteBuilder MapPlatformEndpoints(this IEndpointRouteBuilder app)
|
||||
{
|
||||
var me = app.MapGroup("/api/v1/me")
|
||||
.WithTags("Preferences")
|
||||
.RequireAuthorization(AuthorizationPolicies.RequireReader);
|
||||
|
||||
me.MapGet("/preferences", GetUserPreferencesAsync);
|
||||
me.MapPut("/preferences", UpdateUserPreferencesAsync)
|
||||
.AddEndpointFilter<AntiforgeryEndpointFilter>();
|
||||
|
||||
var settings = app.MapGroup("/api/v1/admin/platform-settings")
|
||||
.WithTags("Admin")
|
||||
.RequireAuthorization(AuthorizationPolicies.RequireAdministrator);
|
||||
|
||||
settings.MapGet("", GetPlatformSettingsAsync);
|
||||
settings.MapPut("", UpdatePlatformSettingsAsync)
|
||||
.AddEndpointFilter<AntiforgeryEndpointFilter>();
|
||||
|
||||
return app;
|
||||
}
|
||||
|
||||
private static async Task<Results<Ok<UserPreferencesResponse>, ProblemHttpResult>> GetUserPreferencesAsync(
|
||||
IWorkspaceRepository workspaces,
|
||||
GetUserPreferencesUseCase useCase,
|
||||
ClaimsPrincipal principal,
|
||||
HttpContext context,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var workspaceId = await workspaces.GetDefaultWorkspaceIdAsync(cancellationToken);
|
||||
if (workspaceId is null)
|
||||
{
|
||||
return TypedResults.Problem(statusCode: StatusCodes.Status404NotFound, type: ProblemTypes.NotFound);
|
||||
}
|
||||
|
||||
var snapshot = await useCase.ExecuteAsync(workspaceId.Value, CurrentUserId(principal), cancellationToken);
|
||||
var response = ToUserPreferencesResponse(snapshot);
|
||||
context.Response.Headers.ETag = ETagHelper.ToHeaderValue(response.ETag);
|
||||
return TypedResults.Ok(response);
|
||||
}
|
||||
|
||||
private static async Task<IResult> UpdateUserPreferencesAsync(
|
||||
[FromBody] UserPreferencesRequest? request,
|
||||
IWorkspaceRepository workspaces,
|
||||
UpdateUserPreferencesUseCase useCase,
|
||||
ClaimsPrincipal principal,
|
||||
HttpContext context,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!TryGetExpectedVersion(context, out var expectedVersion, out var preconditionError))
|
||||
{
|
||||
return preconditionError!;
|
||||
}
|
||||
|
||||
if (request is null)
|
||||
{
|
||||
return ValidationProblem("Voorkeuren ontbreken.");
|
||||
}
|
||||
|
||||
var workspaceId = await workspaces.GetDefaultWorkspaceIdAsync(cancellationToken);
|
||||
if (workspaceId is null)
|
||||
{
|
||||
return NotFound();
|
||||
}
|
||||
|
||||
var values = new UserPreferencesValues(
|
||||
request.Theme,
|
||||
request.Density,
|
||||
request.PrimaryNavigationCollapsed,
|
||||
request.ContextPanelOpen,
|
||||
request.NavigationOrder,
|
||||
request.DashboardWidgetOrder,
|
||||
request.HiddenDashboardWidgets,
|
||||
request.DefaultLandingPage,
|
||||
request.CatalogView,
|
||||
request.ContextPanelWidth,
|
||||
request.NavigationWidth);
|
||||
|
||||
var result = await useCase.ExecuteAsync(
|
||||
new UpdateUserPreferencesCommand(workspaceId.Value, CurrentUserId(principal), expectedVersion, values),
|
||||
cancellationToken);
|
||||
|
||||
if (result.Status == UpdateUserPreferencesStatus.PreconditionFailed)
|
||||
{
|
||||
return ConcurrencyProblem("De voorkeuren zijn sinds de laatste lezing gewijzigd.");
|
||||
}
|
||||
if (result.Status == UpdateUserPreferencesStatus.ValidationFailed)
|
||||
{
|
||||
return ValidationProblem(string.Join(" ", result.Errors));
|
||||
}
|
||||
|
||||
var response = ToUserPreferencesResponse(result.Snapshot!);
|
||||
context.Response.Headers.ETag = ETagHelper.ToHeaderValue(response.ETag);
|
||||
return TypedResults.Ok(response);
|
||||
}
|
||||
|
||||
private static async Task<Ok<PlatformSettingsResponse>> GetPlatformSettingsAsync(
|
||||
GetPlatformSettingsUseCase useCase,
|
||||
HttpContext context,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var response = ToPlatformSettingsResponse(await useCase.ExecuteAsync(cancellationToken));
|
||||
context.Response.Headers.ETag = ETagHelper.ToHeaderValue(response.ETag);
|
||||
return TypedResults.Ok(response);
|
||||
}
|
||||
|
||||
private static async Task<IResult> UpdatePlatformSettingsAsync(
|
||||
[FromBody] PlatformSettingsRequest? request,
|
||||
UpdatePlatformSettingsUseCase useCase,
|
||||
ClaimsPrincipal principal,
|
||||
HttpContext context,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!TryGetExpectedVersion(context, out var expectedVersion, out var preconditionError))
|
||||
{
|
||||
return preconditionError!;
|
||||
}
|
||||
|
||||
if (request is null || request.IngestionProposals is null
|
||||
|| request.BackupPolicyExpectation is null || request.InventoryFreshnessPolicy is null)
|
||||
{
|
||||
return ValidationProblem("Platforminstellingen ontbreken of zijn onvolledig.");
|
||||
}
|
||||
|
||||
var values = new PlatformSettingsValues(
|
||||
request.DisplayName,
|
||||
request.PrimaryLocale,
|
||||
request.TimeZone,
|
||||
new IngestionProposalPolicyValues(
|
||||
request.IngestionProposals.Enabled,
|
||||
request.IngestionProposals.MaximumSourceCount,
|
||||
request.IngestionProposals.MaximumSourceBytes,
|
||||
request.IngestionProposals.FailedRetryLimit),
|
||||
new BackupPolicyExpectationValues(
|
||||
request.BackupPolicyExpectation.ExpectedIntervalHours,
|
||||
request.BackupPolicyExpectation.ScheduleDescription,
|
||||
request.BackupPolicyExpectation.DailyRetentionCount,
|
||||
request.BackupPolicyExpectation.WeeklyRetentionCount,
|
||||
request.BackupPolicyExpectation.MonthlyRetentionCount,
|
||||
request.BackupPolicyExpectation.ProtectedBackupCount,
|
||||
request.BackupPolicyExpectation.RestoreVerificationMaxAgeDays),
|
||||
new InventoryFreshnessPolicyValues(request.InventoryFreshnessPolicy.WarningAfterHours));
|
||||
|
||||
var result = await useCase.ExecuteAsync(
|
||||
new UpdatePlatformSettingsCommand(
|
||||
expectedVersion,
|
||||
CurrentUserId(principal),
|
||||
context.Items[CorrelationIdMiddleware.ItemKey] as string,
|
||||
values),
|
||||
cancellationToken);
|
||||
|
||||
if (result.Status == UpdatePlatformSettingsStatus.PreconditionFailed)
|
||||
{
|
||||
return ConcurrencyProblem("De platforminstellingen zijn sinds de laatste lezing gewijzigd.");
|
||||
}
|
||||
if (result.Status == UpdatePlatformSettingsStatus.ValidationFailed)
|
||||
{
|
||||
return ValidationProblem(string.Join(" ", result.Errors));
|
||||
}
|
||||
|
||||
var response = ToPlatformSettingsResponse(result.Snapshot!);
|
||||
context.Response.Headers.ETag = ETagHelper.ToHeaderValue(response.ETag);
|
||||
return TypedResults.Ok(response);
|
||||
}
|
||||
|
||||
private static UserPreferencesResponse ToUserPreferencesResponse(UserPreferencesSnapshot snapshot)
|
||||
{
|
||||
var values = snapshot.Values;
|
||||
return new UserPreferencesResponse(
|
||||
1,
|
||||
snapshot.Version.ToString(CultureInfo.InvariantCulture),
|
||||
snapshot.IsDefault,
|
||||
values.Theme,
|
||||
values.Density,
|
||||
values.PrimaryNavigationCollapsed,
|
||||
values.ContextPanelOpen,
|
||||
values.NavigationOrder,
|
||||
values.DashboardWidgetOrder,
|
||||
values.HiddenDashboardWidgets,
|
||||
values.DefaultLandingPage,
|
||||
values.CatalogView,
|
||||
values.ContextPanelWidth,
|
||||
values.NavigationWidth,
|
||||
snapshot.UpdatedAt);
|
||||
}
|
||||
|
||||
private static PlatformSettingsResponse ToPlatformSettingsResponse(PlatformSettingsSnapshot snapshot)
|
||||
{
|
||||
var values = snapshot.Values;
|
||||
return new PlatformSettingsResponse(
|
||||
1,
|
||||
snapshot.Version.ToString(CultureInfo.InvariantCulture),
|
||||
snapshot.IsDefault,
|
||||
values.DisplayName,
|
||||
values.PrimaryLocale,
|
||||
values.TimeZone,
|
||||
new IngestionProposalPolicyContract(
|
||||
values.IngestionProposals.Enabled,
|
||||
values.IngestionProposals.MaximumSourceCount,
|
||||
values.IngestionProposals.MaximumSourceBytes,
|
||||
values.IngestionProposals.FailedRetryLimit),
|
||||
new BackupPolicyExpectationResponse(
|
||||
"operator-managed",
|
||||
values.BackupPolicyExpectation.ExpectedIntervalHours,
|
||||
values.BackupPolicyExpectation.ScheduleDescription,
|
||||
values.BackupPolicyExpectation.DailyRetentionCount,
|
||||
values.BackupPolicyExpectation.WeeklyRetentionCount,
|
||||
values.BackupPolicyExpectation.MonthlyRetentionCount,
|
||||
values.BackupPolicyExpectation.ProtectedBackupCount,
|
||||
values.BackupPolicyExpectation.RestoreVerificationMaxAgeDays),
|
||||
new InventoryFreshnessPolicyContract(values.InventoryFreshnessPolicy.WarningAfterHours),
|
||||
snapshot.UpdatedAt);
|
||||
}
|
||||
|
||||
private static bool TryGetExpectedVersion(HttpContext context, out int version, out IResult? error)
|
||||
{
|
||||
var raw = context.Request.Headers.IfMatch.ToString();
|
||||
if (string.IsNullOrWhiteSpace(raw))
|
||||
{
|
||||
version = 0;
|
||||
error = Results.Problem(statusCode: StatusCodes.Status428PreconditionRequired,
|
||||
title: "If-Match-header vereist.", detail: "Lees eerst de actuele ETag en stuur die mee.");
|
||||
return false;
|
||||
}
|
||||
|
||||
version = 0;
|
||||
var token = ETagHelper.FromHeaderValue(raw);
|
||||
if (token is null || !int.TryParse(token, NumberStyles.None, CultureInfo.InvariantCulture, out version) || version < 0)
|
||||
{
|
||||
error = Results.Problem(statusCode: StatusCodes.Status400BadRequest,
|
||||
type: ProblemTypes.Validation, title: "Ongeldige If-Match-header.");
|
||||
return false;
|
||||
}
|
||||
|
||||
error = null;
|
||||
return true;
|
||||
}
|
||||
|
||||
private static Guid CurrentUserId(ClaimsPrincipal principal)
|
||||
{
|
||||
var raw = principal.FindFirstValue(ClaimTypes.NameIdentifier);
|
||||
return Guid.TryParse(raw, out var id) ? id : throw new InvalidOperationException("The authenticated user ID is missing.");
|
||||
}
|
||||
|
||||
private static IResult NotFound() => Results.Problem(statusCode: StatusCodes.Status404NotFound, type: ProblemTypes.NotFound);
|
||||
|
||||
private static IResult ValidationProblem(string detail) => Results.Problem(
|
||||
statusCode: StatusCodes.Status422UnprocessableEntity,
|
||||
type: ProblemTypes.Validation,
|
||||
title: "Ongeldige instellingen.",
|
||||
detail: detail);
|
||||
|
||||
private static IResult ConcurrencyProblem(string detail) => Results.Problem(
|
||||
statusCode: StatusCodes.Status412PreconditionFailed,
|
||||
type: ProblemTypes.ConcurrencyConflict,
|
||||
title: "Versieconflict.",
|
||||
detail: detail);
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
using ItWorx.KnowledgeHub.Api.Authorization;
|
||||
using ItWorx.KnowledgeHub.Api.Security;
|
||||
using ItWorx.KnowledgeHub.Application.Identity;
|
||||
using ItWorx.KnowledgeHub.Application.Knowledge.UseCases;
|
||||
using ItWorx.KnowledgeHub.Contracts.Errors;
|
||||
@@ -16,6 +17,7 @@ public static class SearchEndpoints
|
||||
{
|
||||
app.MapGet("/api/v1/search", SearchAsync)
|
||||
.WithTags("Search")
|
||||
.RequireRateLimiting(RateLimitPolicies.Search)
|
||||
.RequireContentAccess(WorkspaceRole.Reader, Scopes.KnowledgeSearch);
|
||||
|
||||
return app;
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
using ItWorx.KnowledgeHub.Api.Authorization;
|
||||
using ItWorx.KnowledgeHub.Api.Security;
|
||||
using ItWorx.KnowledgeHub.Application.Identity;
|
||||
using ItWorx.KnowledgeHub.Application.UnifiedSearch;
|
||||
using ItWorx.KnowledgeHub.Contracts.Errors;
|
||||
using ItWorx.KnowledgeHub.Contracts.UnifiedSearch;
|
||||
using Microsoft.AspNetCore.Http.HttpResults;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Api.Endpoints;
|
||||
|
||||
public static class UnifiedSearchEndpoints
|
||||
{
|
||||
public static IEndpointRouteBuilder MapUnifiedSearchEndpoints(this IEndpointRouteBuilder app)
|
||||
{
|
||||
// Unified search includes projects and infrastructure services. Keep it on
|
||||
// the human workspace-reader policy so a bearer token with only
|
||||
// knowledge:search cannot gain either resource's scope through aggregation.
|
||||
app.MapGet("/api/v1/unified-search", SearchAsync)
|
||||
.WithTags("Search")
|
||||
.RequireRateLimiting(RateLimitPolicies.Search)
|
||||
.RequireAuthorization(AuthorizationPolicies.RequireReader);
|
||||
|
||||
return app;
|
||||
}
|
||||
|
||||
private static async Task<Results<Ok<UnifiedSearchResponse>, ProblemHttpResult>> SearchAsync(
|
||||
[FromQuery] string? q,
|
||||
[FromQuery] string? tag,
|
||||
[FromQuery] string? kind,
|
||||
[FromQuery] string? cursor,
|
||||
[FromQuery] int? limit,
|
||||
IWorkspaceRepository workspaces,
|
||||
UnifiedSearchUseCase useCase,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!TryParseKind(kind, out var searchKind))
|
||||
{
|
||||
return TypedResults.Problem(
|
||||
statusCode: StatusCodes.Status422UnprocessableEntity,
|
||||
type: ProblemTypes.Validation,
|
||||
detail: "Kind must be one of: knowledge, projects, services, runbooks, decisions.");
|
||||
}
|
||||
|
||||
var workspaceId = await workspaces.GetDefaultWorkspaceIdAsync(cancellationToken);
|
||||
if (workspaceId is null)
|
||||
{
|
||||
return TypedResults.Problem(statusCode: StatusCodes.Status404NotFound, type: ProblemTypes.NotFound);
|
||||
}
|
||||
|
||||
var result = await useCase.ExecuteAsync(workspaceId.Value, q, tag, searchKind, cursor, limit ?? 25, cancellationToken);
|
||||
IReadOnlyList<UnifiedSearchResultItem> items = result.Items
|
||||
.Select(hit => new UnifiedSearchResultItem(
|
||||
ToPublicKind(hit.Kind), hit.Id, hit.Title, hit.Summary, hit.Snippet, hit.UpdatedAt, hit.Href))
|
||||
.ToArray();
|
||||
var counts = new UnifiedSearchCounts(
|
||||
result.Counts.Knowledge, result.Counts.Projects, result.Counts.Services,
|
||||
result.Counts.Runbooks, result.Counts.Decisions);
|
||||
|
||||
return TypedResults.Ok(new UnifiedSearchResponse(items, counts, result.NextCursor));
|
||||
}
|
||||
|
||||
private static bool TryParseKind(string? value, out UnifiedSearchKind? kind)
|
||||
{
|
||||
kind = null;
|
||||
if (value is null)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
kind = value.Trim().ToLowerInvariant() switch
|
||||
{
|
||||
"knowledge" => UnifiedSearchKind.Knowledge,
|
||||
"projects" => UnifiedSearchKind.Projects,
|
||||
"services" => UnifiedSearchKind.Services,
|
||||
"runbooks" => UnifiedSearchKind.Runbooks,
|
||||
"decisions" => UnifiedSearchKind.Decisions,
|
||||
_ => null
|
||||
};
|
||||
return kind is not null;
|
||||
}
|
||||
|
||||
private static string ToPublicKind(string kind) => kind switch
|
||||
{
|
||||
"knowledge" => "knowledge",
|
||||
"project" => "projects",
|
||||
"service" => "services",
|
||||
"runbook" => "runbooks",
|
||||
"decision" => "decisions",
|
||||
_ => throw new InvalidOperationException($"Unexpected unified search kind '{kind}'.")
|
||||
};
|
||||
}
|
||||
@@ -4,6 +4,11 @@
|
||||
<RootNamespace>ItWorx.KnowledgeHub.Api</RootNamespace>
|
||||
<InvariantGlobalization>false</InvariantGlobalization>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="OpenTelemetry.Extensions.Hosting" />
|
||||
<PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" />
|
||||
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../ItWorx.KnowledgeHub.Application/ItWorx.KnowledgeHub.Application.csproj" />
|
||||
<ProjectReference Include="../ItWorx.KnowledgeHub.Contracts/ItWorx.KnowledgeHub.Contracts.csproj" />
|
||||
|
||||
@@ -21,13 +21,24 @@ public static class RestoreModeRunner
|
||||
public static async Task<int> RunAsync(string[] args, IServiceProvider services, TextWriter output, CancellationToken cancellationToken)
|
||||
{
|
||||
var backupId = GetArgValue(args, "--backup-id");
|
||||
var targetConnection = GetArgValue(args, "--target-connection");
|
||||
var targetConnectionFile = GetArgValue(args, "--target-connection-file");
|
||||
var targetAttachments = GetArgValue(args, "--target-attachments");
|
||||
|
||||
if (backupId is null || targetConnection is null || targetAttachments is null)
|
||||
if (HasArg(args, "--target-connection") || backupId is null || targetConnectionFile is null || targetAttachments is null)
|
||||
{
|
||||
await output.WriteLineAsync(
|
||||
"Gebruik: --restore --backup-id=<id> --target-connection=<connectiereeks> --target-attachments=<pad>");
|
||||
"Gebruik: --restore --backup-id=<id> --target-connection-file=<beveiligd-pad> --target-attachments=<pad>");
|
||||
return 2;
|
||||
}
|
||||
|
||||
string targetConnection;
|
||||
try
|
||||
{
|
||||
targetConnection = await ReadTargetConnectionAsync(targetConnectionFile, cancellationToken);
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
await output.WriteLineAsync("Herstel niet gestart: de doelconnectie kon niet veilig worden gelezen.");
|
||||
return 2;
|
||||
}
|
||||
|
||||
@@ -40,7 +51,9 @@ public static class RestoreModeRunner
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
await output.WriteLineAsync($"Herstel mislukt: {exception.Message}");
|
||||
// Provider messages may include the target connection string. Do not
|
||||
// print exception text in this maintenance command's output.
|
||||
await output.WriteLineAsync("Herstel mislukt; gevoelige foutdetails zijn niet weergegeven.");
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -63,7 +76,47 @@ public static class RestoreModeRunner
|
||||
private static string? GetArgValue(string[] args, string name)
|
||||
{
|
||||
var prefix = name + "=";
|
||||
var match = args.FirstOrDefault(argument => argument.StartsWith(prefix, StringComparison.Ordinal));
|
||||
return match?[prefix.Length..];
|
||||
var matches = args.Where(argument => argument.StartsWith(prefix, StringComparison.Ordinal)).ToArray();
|
||||
return matches.Length == 1 && matches[0].Length > prefix.Length
|
||||
? matches[0][prefix.Length..]
|
||||
: null;
|
||||
}
|
||||
|
||||
private static bool HasArg(string[] args, string name) =>
|
||||
args.Any(argument => argument.Equals(name, StringComparison.Ordinal)
|
||||
|| argument.StartsWith(name + "=", StringComparison.Ordinal));
|
||||
|
||||
private static async Task<string> ReadTargetConnectionAsync(string path, CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(path)) throw new IOException("Missing connection file.");
|
||||
var fullPath = Path.GetFullPath(path);
|
||||
var attributes = File.GetAttributes(fullPath);
|
||||
if ((attributes & (FileAttributes.Directory | FileAttributes.ReparsePoint)) != 0)
|
||||
throw new IOException("Connection file must be a regular file.");
|
||||
|
||||
if (!OperatingSystem.IsWindows())
|
||||
{
|
||||
var mode = File.GetUnixFileMode(fullPath);
|
||||
const UnixFileMode groupOrOtherAccess = UnixFileMode.GroupRead | UnixFileMode.GroupWrite
|
||||
| UnixFileMode.GroupExecute | UnixFileMode.OtherRead | UnixFileMode.OtherWrite | UnixFileMode.OtherExecute;
|
||||
if ((mode & groupOrOtherAccess) != 0 || (mode & UnixFileMode.UserRead) == 0)
|
||||
throw new IOException("Connection file permissions are too broad.");
|
||||
}
|
||||
|
||||
await using var stream = new FileStream(fullPath, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
if (stream.Length is <= 0 or > 16_384) throw new IOException("Connection file is empty or too large.");
|
||||
var bytes = new byte[checked((int)stream.Length)];
|
||||
await stream.ReadExactlyAsync(bytes, cancellationToken);
|
||||
var extra = new byte[1];
|
||||
if (await stream.ReadAsync(extra, cancellationToken) != 0)
|
||||
throw new IOException("Connection file changed while being read.");
|
||||
|
||||
var connectionString = new System.Text.UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true)
|
||||
.GetString(bytes)
|
||||
.TrimStart('\uFEFF')
|
||||
.TrimEnd('\r', '\n');
|
||||
if (string.IsNullOrWhiteSpace(connectionString) || connectionString.Contains('\r') || connectionString.Contains('\n'))
|
||||
throw new IOException("Connection file contents are invalid.");
|
||||
return connectionString;
|
||||
}
|
||||
}
|
||||
@@ -13,23 +13,31 @@ using ItWorx.KnowledgeHub.Application.Identity.UseCases;
|
||||
using ItWorx.KnowledgeHub.Application.Infrastructure.UseCases;
|
||||
using ItWorx.KnowledgeHub.Application.Ingestion;
|
||||
using ItWorx.KnowledgeHub.Application.Ingestion.UseCases;
|
||||
using ItWorx.KnowledgeHub.Application.Jobs;
|
||||
using ItWorx.KnowledgeHub.Application.Jobs.UseCases;
|
||||
using ItWorx.KnowledgeHub.Application.Knowledge.UseCases;
|
||||
using ItWorx.KnowledgeHub.Application.Migration.UseCases;
|
||||
using ItWorx.KnowledgeHub.Application.Operations.UseCases;
|
||||
using ItWorx.KnowledgeHub.Application.Platform;
|
||||
using ItWorx.KnowledgeHub.Application.Platform.UseCases;
|
||||
using ItWorx.KnowledgeHub.Application.Projects.UseCases;
|
||||
using ItWorx.KnowledgeHub.Application.Runbooks.UseCases;
|
||||
using ItWorx.KnowledgeHub.Application.UnifiedSearch;
|
||||
using ItWorx.KnowledgeHub.Contracts.SystemInfo;
|
||||
using ItWorx.KnowledgeHub.Domain.Automation;
|
||||
using ItWorx.KnowledgeHub.Domain.Identity;
|
||||
using ItWorx.KnowledgeHub.Infrastructure;
|
||||
using ItWorx.KnowledgeHub.Infrastructure.Operations;
|
||||
using ItWorx.KnowledgeHub.Infrastructure.Persistence;
|
||||
using ItWorx.KnowledgeHub.Infrastructure.Platform;
|
||||
using Microsoft.AspNetCore.Authentication.Cookies;
|
||||
using Microsoft.AspNetCore.DataProtection;
|
||||
using Microsoft.AspNetCore.Diagnostics.HealthChecks;
|
||||
using Microsoft.AspNetCore.Identity;
|
||||
using Microsoft.AspNetCore.RateLimiting;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using OpenTelemetry.Metrics;
|
||||
using OpenTelemetry.Resources;
|
||||
|
||||
var timeProvider = TimeProvider.System;
|
||||
var startedAt = timeProvider.GetUtcNow();
|
||||
@@ -58,6 +66,38 @@ if (builder.Environment.IsProduction())
|
||||
}
|
||||
|
||||
builder.Services.AddSingleton(timeProvider);
|
||||
builder.Services.AddScoped<IUserPreferencesRepository, UserPreferencesRepository>();
|
||||
builder.Services.AddScoped<PlatformSettingsRepository>();
|
||||
builder.Services.AddScoped<IPlatformSettingsRepository>(provider => provider.GetRequiredService<PlatformSettingsRepository>());
|
||||
builder.Services.AddScoped<IPlatformSettingsReader>(provider => provider.GetRequiredService<PlatformSettingsRepository>());
|
||||
builder.Services.AddScoped<GetUserPreferencesUseCase>();
|
||||
builder.Services.AddScoped<UpdateUserPreferencesUseCase>();
|
||||
builder.Services.AddScoped<GetPlatformSettingsUseCase>();
|
||||
builder.Services.AddScoped<UpdatePlatformSettingsUseCase>();
|
||||
builder.Services.AddScoped<QueueBackgroundJobUseCase>();
|
||||
builder.Services.AddScoped<CancelBackgroundJobUseCase>();
|
||||
builder.Services.AddScoped<BackgroundJobQueries>();
|
||||
builder.Services.AddSingleton(provider =>
|
||||
{
|
||||
var options = new BackgroundJobsOptions();
|
||||
provider.GetRequiredService<IConfiguration>().GetSection("KnowledgeHub:Jobs").Bind(options);
|
||||
options.Validate();
|
||||
return options;
|
||||
});
|
||||
builder.Services.AddScoped<ItWorx.KnowledgeHub.Application.Sources.IDataSourceRepository, ItWorx.KnowledgeHub.Infrastructure.Sources.DataSourceRepository>();
|
||||
builder.Services.AddScoped<ItWorx.KnowledgeHub.Application.Sources.DataSourceUseCases>();
|
||||
builder.Services.AddOpenTelemetry()
|
||||
.ConfigureResource(resource => resource.AddService("itworx-knowledge-hub"))
|
||||
.WithMetrics(metrics =>
|
||||
{
|
||||
metrics.AddAspNetCoreInstrumentation()
|
||||
.AddMeter("ItWorx.KnowledgeHub.Ingestion", "ItWorx.KnowledgeHub.Platform", "System.Runtime");
|
||||
// OTLP is enabled only by an explicit operator endpoint. No default cloud destination.
|
||||
if (!string.IsNullOrWhiteSpace(builder.Configuration["OTEL_EXPORTER_OTLP_ENDPOINT"]))
|
||||
{
|
||||
metrics.AddOtlpExporter();
|
||||
}
|
||||
});
|
||||
builder.Services.AddSingleton(_ => new Meter("ItWorx.KnowledgeHub.Ingestion"));
|
||||
builder.Services.AddSingleton<IngestionMetrics>();
|
||||
builder.Services.AddProblemDetails(options =>
|
||||
@@ -208,6 +248,10 @@ builder.Services.AddRateLimiter(options =>
|
||||
CreateRemoteIpPartition(context, permitLimit: 10));
|
||||
options.AddPolicy(RateLimitPolicies.SourceUpload, context =>
|
||||
CreateRemoteIpPartition(context, permitLimit: 5));
|
||||
options.AddPolicy(RateLimitPolicies.Search, context =>
|
||||
CreateRemoteIpPartition(context, permitLimit: 60));
|
||||
options.AddPolicy(RateLimitPolicies.JobCreation, context =>
|
||||
CreateRemoteIpPartition(context, permitLimit: 10));
|
||||
});
|
||||
|
||||
// --- Use cases (composition root) -------------------------------------------------------
|
||||
@@ -228,6 +272,8 @@ builder.Services.AddScoped<CommitPageCreationSessionUseCase>();
|
||||
builder.Services.AddScoped<DiscardPageCreationSessionUseCase>();
|
||||
builder.Services.AddScoped<CleanupExpiredPageCreationSessionsUseCase>();
|
||||
builder.Services.AddHostedService<ItWorx.KnowledgeHub.Api.BackgroundServices.StagingSessionCleanupBackgroundService>();
|
||||
builder.Services.AddHostedService<ItWorx.KnowledgeHub.Api.BackgroundServices.OperationalTelemetryBackgroundService>();
|
||||
builder.Services.AddHostedService<ItWorx.KnowledgeHub.Api.BackgroundServices.JobWorker>();
|
||||
builder.Services.AddScoped<GetPageUseCase>();
|
||||
builder.Services.AddScoped<GetPageChildrenUseCase>();
|
||||
builder.Services.AddScoped<UpdatePageUseCase>();
|
||||
@@ -311,6 +357,7 @@ builder.Services.AddScoped<ListInfrastructureVolumesUseCase>();
|
||||
builder.Services.AddScoped<GetInfrastructureServiceUseCase>();
|
||||
builder.Services.AddScoped<ListInfrastructureChangesUseCase>();
|
||||
builder.Services.AddScoped<SynchronizeInfrastructureInventoryUseCase>();
|
||||
builder.Services.AddScoped<UnifiedSearchUseCase>();
|
||||
builder.Services.AddScoped<ItWorx.KnowledgeHub.Application.Infrastructure.Validation.EditorialProfileValidator>();
|
||||
builder.Services.AddScoped<UpdateInfrastructureEditorialProfileUseCase>();
|
||||
builder.Services.AddScoped<SetManualOverrideUseCase>();
|
||||
@@ -495,6 +542,7 @@ app.MapGet(
|
||||
async (
|
||||
IHostEnvironment environment,
|
||||
IConfiguration configuration,
|
||||
IPlatformSettingsReader platformSettings,
|
||||
ReleaseManifestReader releaseManifestReader,
|
||||
KnowledgeHubDbContext db,
|
||||
CancellationToken cancellationToken) =>
|
||||
@@ -502,8 +550,9 @@ app.MapGet(
|
||||
var assembly = Assembly.GetExecutingAssembly();
|
||||
var version = assembly.GetName().Version?.ToString() ?? "0.0.0";
|
||||
var commit = configuration["Build:Commit"] ?? "development";
|
||||
var locale = configuration["KnowledgeHub:PrimaryLocale"] ?? "nl-BE";
|
||||
var timeZone = configuration["KnowledgeHub:TimeZone"] ?? "Europe/Brussels";
|
||||
var settings = (await platformSettings.GetEffectiveAsync(cancellationToken)).Values;
|
||||
var locale = settings.PrimaryLocale;
|
||||
var timeZone = settings.TimeZone;
|
||||
|
||||
var uptimeSeconds = Math.Max(
|
||||
0,
|
||||
@@ -549,10 +598,10 @@ app.MapGet(
|
||||
|
||||
return Results.Ok(
|
||||
new SystemInfoResponse(
|
||||
"ItWorx Knowledge Hub",
|
||||
version,
|
||||
settings.DisplayName,
|
||||
manifest?.SemanticVersion ?? version,
|
||||
commit,
|
||||
"m7",
|
||||
migrationRange?.Split("..").Last() ?? "Unknown",
|
||||
environment.EnvironmentName,
|
||||
locale,
|
||||
timeZone,
|
||||
@@ -568,6 +617,7 @@ app.MapPagesEndpoints();
|
||||
app.MapDraftsEndpoints();
|
||||
app.MapTemplatesEndpoints();
|
||||
app.MapSearchEndpoints();
|
||||
app.MapUnifiedSearchEndpoints();
|
||||
app.MapRagPublicationEndpoints();
|
||||
app.MapAttachmentsEndpoints();
|
||||
app.MapIngestionProposalsEndpoints();
|
||||
@@ -578,6 +628,10 @@ app.MapDecisionsEndpoints();
|
||||
app.MapRunbooksEndpoints();
|
||||
app.MapServiceAccountsEndpoints();
|
||||
app.MapBackupsEndpoints();
|
||||
app.MapAdminAuditEndpoints();
|
||||
app.MapDataSourcesEndpoints();
|
||||
app.MapPlatformEndpoints();
|
||||
app.MapJobsEndpoints();
|
||||
app.MapInfrastructureEndpoints();
|
||||
app.MapActivityEndpoints();
|
||||
app.MapWorkspaceEndpoints();
|
||||
|
||||
@@ -8,4 +8,6 @@ public static class RateLimitPolicies
|
||||
public const string Login = "login";
|
||||
public const string Recovery = "recovery";
|
||||
public const string SourceUpload = "source-upload";
|
||||
public const string Search = "search";
|
||||
public const string JobCreation = "job-creation";
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
using ItWorx.KnowledgeHub.Contracts.Errors;
|
||||
using Microsoft.AspNetCore.Antiforgery;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Api.Security;
|
||||
|
||||
/// <summary>CSRF validation that cannot be bypassed by a caller-supplied Authorization
|
||||
/// header. Used by routes that are specifically cookie-authenticated.</summary>
|
||||
public sealed class StrictAntiforgeryEndpointFilter(IAntiforgery antiforgery) : IEndpointFilter
|
||||
{
|
||||
public async ValueTask<object?> InvokeAsync(
|
||||
EndpointFilterInvocationContext context,
|
||||
EndpointFilterDelegate next)
|
||||
{
|
||||
try
|
||||
{
|
||||
await antiforgery.ValidateRequestAsync(context.HttpContext);
|
||||
}
|
||||
catch (AntiforgeryValidationException)
|
||||
{
|
||||
return Results.Problem(
|
||||
statusCode: StatusCodes.Status400BadRequest,
|
||||
type: ProblemTypes.CsrfValidationFailed,
|
||||
title: "CSRF-validatie mislukt.",
|
||||
detail: "De aanvraag ontbreekt een geldig CSRF-token.");
|
||||
}
|
||||
|
||||
return await next(context);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
namespace ItWorx.KnowledgeHub.Application.Audit;
|
||||
|
||||
public sealed record OperationalActivityPage(IReadOnlyList<AuditEventView> Items, long? NextCursor);
|
||||
|
||||
public interface IActivityEventReader
|
||||
{
|
||||
Task<OperationalActivityPage> ListAsync(int limit, long? before, CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -2,9 +2,10 @@ namespace ItWorx.KnowledgeHub.Application.Audit;
|
||||
|
||||
public sealed record AuditEventView(
|
||||
long Id, string ActorKind, Guid? ActorId, string Source, string Action,
|
||||
string? TargetType, string? TargetId, string Summary, DateTimeOffset OccurredAt);
|
||||
string? TargetType, string? TargetId, string Summary, DateTimeOffset OccurredAt, string? CorrelationId = null);
|
||||
|
||||
public interface IAuditEventReader
|
||||
{
|
||||
Task<IReadOnlyList<AuditEventView>> ListRecentAsync(int limit, CancellationToken cancellationToken);
|
||||
Task<OperationalActivityPage> ListPageAsync(int limit, long? before, CancellationToken cancellationToken);
|
||||
}
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
namespace ItWorx.KnowledgeHub.Application.Infrastructure;
|
||||
|
||||
/// <summary>The registered inventory source was disabled or has an incompatible kind
|
||||
/// when the import transaction acquired its source-row lock.</summary>
|
||||
public sealed class InactiveInventorySourceException : Exception
|
||||
{
|
||||
}
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
namespace ItWorx.KnowledgeHub.Application.Infrastructure;
|
||||
|
||||
public sealed class StaleInventorySnapshotException : Exception
|
||||
{
|
||||
public StaleInventorySnapshotException() : base("De snapshot is ouder dan de laatst verwerkte inventaris.") { }
|
||||
}
|
||||
+68
-3
@@ -1,5 +1,9 @@
|
||||
using ItWorx.KnowledgeHub.Application.Audit;
|
||||
using ItWorx.KnowledgeHub.Application.Common;
|
||||
using ItWorx.KnowledgeHub.Application.Operations;
|
||||
using ItWorx.KnowledgeHub.Application.Platform;
|
||||
using ItWorx.KnowledgeHub.Application.Sources;
|
||||
using ItWorx.KnowledgeHub.Domain.Sources;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Application.Infrastructure.UseCases;
|
||||
|
||||
@@ -14,12 +18,19 @@ public sealed record SynchronizeInfrastructureInventoryCommand(
|
||||
public sealed class SynchronizeInfrastructureInventoryUseCase(
|
||||
IInfrastructureRepository infrastructure,
|
||||
IAuditEventRecorder auditEvents,
|
||||
TimeProvider timeProvider)
|
||||
TimeProvider timeProvider,
|
||||
PlatformMetrics? metrics = null,
|
||||
IDataSourceRepository? sources = null,
|
||||
IPlatformSettingsReader? settings = null)
|
||||
{
|
||||
public async Task<OperationResultValue<InventorySyncResult>> ExecuteAsync(
|
||||
SynchronizeInfrastructureInventoryCommand command,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
using var measurement = metrics?.Start("inventory");
|
||||
var now = timeProvider.GetUtcNow();
|
||||
if (command.Snapshot.CapturedAt > now || command.CorrelationId?.Length > 100 || command.CorrelationId?.Any(char.IsControl) == true)
|
||||
return OperationResultValue.Failure<InventorySyncResult>(OperationStatus.ValidationFailed, ["Ongeldige observatietijd of correlatie."]);
|
||||
if (string.IsNullOrWhiteSpace(command.Snapshot.SourceKey)
|
||||
|| string.IsNullOrWhiteSpace(command.Snapshot.SourceHash))
|
||||
{
|
||||
@@ -34,8 +45,61 @@ public sealed class SynchronizeInfrastructureInventoryUseCase(
|
||||
OperationStatus.ValidationFailed, ["De inventaris bevat dubbele bron-ID's."]);
|
||||
}
|
||||
|
||||
var result = await infrastructure.ApplySnapshotAsync(
|
||||
command.WorkspaceId, command.Snapshot, timeProvider.GetUtcNow(), cancellationToken);
|
||||
var registeredSource = sources is null ? null : await sources.FindByKeyAsync(command.WorkspaceId, "inventory:default", cancellationToken);
|
||||
if (registeredSource is { Enabled: false } or { Kind: not DataSourceKind.Inventory })
|
||||
return OperationResultValue.Failure<InventorySyncResult>(OperationStatus.InvalidTransition, ["Deze inventarisbron is uitgeschakeld."]);
|
||||
InventorySyncResult result;
|
||||
try
|
||||
{
|
||||
result = await infrastructure.ApplySnapshotAsync(command.WorkspaceId, command.Snapshot, now, cancellationToken);
|
||||
}
|
||||
catch (StaleInventorySnapshotException)
|
||||
{
|
||||
return OperationResultValue.Failure<InventorySyncResult>(OperationStatus.InvalidTransition, ["De snapshot is ouder dan de actuele inventaris."]);
|
||||
}
|
||||
catch (InactiveInventorySourceException)
|
||||
{
|
||||
return OperationResultValue.Failure<InventorySyncResult>(OperationStatus.InvalidTransition, ["Deze inventarisbron is uitgeschakeld of ongeldig."]);
|
||||
}
|
||||
|
||||
if (sources is not null)
|
||||
{
|
||||
// The snapshot lock has been released. A concurrent identical import may
|
||||
// have registered the source while this request waited for that lock.
|
||||
var source = registeredSource ?? await sources.FindByKeyAsync(command.WorkspaceId, "inventory:default", cancellationToken);
|
||||
if (source is null)
|
||||
{
|
||||
var cadence = settings is null ? 24 : (await settings.GetEffectiveAsync(cancellationToken)).Values.InventoryFreshnessPolicy.WarningAfterHours;
|
||||
source = DataSource.Create(command.WorkspaceId, "inventory:default", "Infrastructuurinventaris", DataSourceKind.Inventory, cadence, now);
|
||||
sources.Add(source);
|
||||
}
|
||||
if (source is { Enabled: false } or { Kind: not DataSourceKind.Inventory })
|
||||
throw new InvalidOperationException("Inventaris verwerkt; de bronregistratie is inmiddels gewijzigd.");
|
||||
if (!result.AlreadyImported || source.DataObservedAt is null)
|
||||
{
|
||||
source.Observe(true, command.Snapshot.Services.Count, null, command.CorrelationId, now, command.Snapshot.CapturedAt);
|
||||
if (!await sources.SaveAsync(null, cancellationToken))
|
||||
{
|
||||
// A competing import can still win the registration/update race
|
||||
// after the read above. Accept it only if the persisted enabled
|
||||
// source already carries at least this capture's evidence.
|
||||
var persisted = await sources.FindByKeyAsync(command.WorkspaceId, "inventory:default", cancellationToken);
|
||||
if (persisted is not { Enabled: true, Kind: DataSourceKind.Inventory })
|
||||
throw new InvalidOperationException("Inventaris verwerkt; bronregistratie gewijzigd. Dezelfde snapshot kan veilig opnieuw worden aangeboden.");
|
||||
if (persisted.DataObservedAt is null || persisted.DataObservedAt < command.Snapshot.CapturedAt)
|
||||
{
|
||||
persisted.Observe(true, command.Snapshot.Services.Count, null, command.CorrelationId,
|
||||
timeProvider.GetUtcNow(), command.Snapshot.CapturedAt);
|
||||
if (!await sources.SaveAsync(null, cancellationToken))
|
||||
{
|
||||
persisted = await sources.FindByKeyAsync(command.WorkspaceId, "inventory:default", cancellationToken);
|
||||
if (persisted is not { Enabled: true, Kind: DataSourceKind.Inventory } || persisted.DataObservedAt is null || persisted.DataObservedAt < command.Snapshot.CapturedAt)
|
||||
throw new InvalidOperationException("Inventaris verwerkt; bronregistratie gewijzigd. Dezelfde snapshot kan veilig opnieuw worden aangeboden.");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await auditEvents.RecordAsync(
|
||||
new AuditEventRecord(
|
||||
@@ -47,6 +111,7 @@ public sealed class SynchronizeInfrastructureInventoryUseCase(
|
||||
command.CorrelationId, timeProvider.GetUtcNow()),
|
||||
cancellationToken);
|
||||
|
||||
measurement?.Complete(command.Snapshot.Services.Count);
|
||||
return OperationResultValue.Success(result);
|
||||
}
|
||||
|
||||
|
||||
+5
-3
@@ -4,13 +4,15 @@ using ItWorx.KnowledgeHub.Domain.Ingestion;
|
||||
namespace ItWorx.KnowledgeHub.Application.Ingestion.UseCases;
|
||||
|
||||
public sealed record CreateIngestionProposalCommand(Guid WorkspaceId, Guid ActorId, string Manifest, string CorrelationId,
|
||||
string? IdempotencyKey, IngestionPublication? Publication = null);
|
||||
string? IdempotencyKey, IngestionPublication? Publication = null,
|
||||
int MaxPublishRetries = IngestionProposal.DefaultMaxPublishRetries);
|
||||
|
||||
public sealed class CreateIngestionProposalUseCase(IIngestionProposalRepository proposals, TimeProvider timeProvider)
|
||||
{
|
||||
public async Task<OperationResultValue<Guid>> ExecuteAsync(CreateIngestionProposalCommand command, CancellationToken cancellationToken)
|
||||
{
|
||||
if (command.WorkspaceId == Guid.Empty || command.ActorId == Guid.Empty || string.IsNullOrWhiteSpace(command.Manifest) || command.Manifest.Length > 100_000)
|
||||
if (command.WorkspaceId == Guid.Empty || command.ActorId == Guid.Empty || string.IsNullOrWhiteSpace(command.Manifest)
|
||||
|| command.Manifest.Length > 100_000 || command.MaxPublishRetries is < 1 or > 3)
|
||||
return OperationResultValue.Failure<Guid>(OperationStatus.ValidationFailed, ["Ongeldig voorstel."]);
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(command.IdempotencyKey))
|
||||
@@ -22,7 +24,7 @@ public sealed class CreateIngestionProposalUseCase(IIngestionProposalRepository
|
||||
|
||||
var proposal = IngestionProposal.Create(command.WorkspaceId, command.ActorId,
|
||||
string.IsNullOrWhiteSpace(command.CorrelationId) ? "ingestion" : command.CorrelationId,
|
||||
command.Manifest, IngestionProposal.DefaultMaxPublishRetries);
|
||||
command.Manifest, command.MaxPublishRetries);
|
||||
if (!string.IsNullOrWhiteSpace(command.IdempotencyKey)) proposal.AssignIdempotencyKey(command.IdempotencyKey);
|
||||
if (command.Publication is not null)
|
||||
{
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
using ItWorx.KnowledgeHub.Domain.Jobs;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Application.Jobs;
|
||||
|
||||
public enum EnqueueJobStatus
|
||||
{
|
||||
Created,
|
||||
Existing,
|
||||
IdempotencyConflict,
|
||||
RootNotFound,
|
||||
TooManyPages,
|
||||
QueueFull
|
||||
}
|
||||
|
||||
public sealed record EnqueueJobResult(EnqueueJobStatus Status, BackgroundJob? Job);
|
||||
|
||||
public sealed record BackgroundJobQueueLimits(int PendingPerWorkspace, int PendingPerActor, int CreatedPerDay);
|
||||
|
||||
public sealed record BackgroundJobPageWorkItem(
|
||||
long Ordinal,
|
||||
Guid PageId,
|
||||
bool InitiallyTagged,
|
||||
BackgroundJobItemState State);
|
||||
|
||||
public sealed record BackgroundJobQueueStatistics(long Queued, long Running, long Failed, DateTimeOffset? OldestQueuedAt);
|
||||
@@ -0,0 +1,59 @@
|
||||
using ItWorx.KnowledgeHub.Application.Audit;
|
||||
using ItWorx.KnowledgeHub.Domain.Jobs;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Application.Jobs;
|
||||
|
||||
/// <summary>Persistence and lease port for durable operator jobs. Implementations claim
|
||||
/// with PostgreSQL SKIP LOCKED and keep the subtree snapshot in rows, never application
|
||||
/// memory.</summary>
|
||||
public interface IBackgroundJobRepository
|
||||
{
|
||||
Task<EnqueueJobResult> EnqueueAsync(
|
||||
BackgroundJob job,
|
||||
Guid? rootPageId,
|
||||
string? tag,
|
||||
int maxCoursePages,
|
||||
BackgroundJobQueueLimits limits,
|
||||
AuditEventRecord auditEvent,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
Task<BackgroundJob?> FindAsync(Guid workspaceId, Guid jobId, CancellationToken cancellationToken);
|
||||
Task<IReadOnlyList<BackgroundJob>> ListAsync(Guid workspaceId, int limit, CancellationToken cancellationToken);
|
||||
Task<BackgroundJob?> FindByIdempotencyAsync(Guid workspaceId, Guid actorId, string keyHash, CancellationToken cancellationToken);
|
||||
|
||||
Task<bool> RequestCancellationAsync(
|
||||
Guid workspaceId,
|
||||
Guid jobId,
|
||||
DateTimeOffset now,
|
||||
AuditEventRecord auditEvent,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
Task<Guid?> TryClaimNextAsync(Guid leaseToken, DateTimeOffset now, DateTimeOffset leaseUntil, CancellationToken cancellationToken);
|
||||
Task<bool> RenewLeaseAsync(Guid jobId, Guid leaseToken, DateTimeOffset leaseUntil, CancellationToken cancellationToken);
|
||||
Task<BackgroundJob?> FindLeasedAsync(Guid jobId, Guid leaseToken, CancellationToken cancellationToken);
|
||||
Task<BackgroundJobPageWorkItem?> FindNextPageItemAsync(Guid jobId, Guid leaseToken, CancellationToken cancellationToken);
|
||||
Task MarkPageItemProcessingAsync(Guid jobId, Guid leaseToken, long ordinal, DateTimeOffset now, CancellationToken cancellationToken);
|
||||
Task RecordPageItemOutcomeAsync(
|
||||
Guid jobId,
|
||||
Guid leaseToken,
|
||||
long ordinal,
|
||||
BackgroundJobItemState outcome,
|
||||
string? errorCode,
|
||||
string? errorMessage,
|
||||
Guid? pageId,
|
||||
DateTimeOffset now,
|
||||
CancellationToken cancellationToken);
|
||||
Task RecordJobOutcomeAsync(
|
||||
Guid jobId,
|
||||
Guid leaseToken,
|
||||
BackgroundJobItemState outcome,
|
||||
string? errorCode,
|
||||
string? errorMessage,
|
||||
CancellationToken cancellationToken);
|
||||
Task CompleteLeasedJobAsync(Guid jobId, Guid leaseToken, string? resultJson, DateTimeOffset now, CancellationToken cancellationToken);
|
||||
Task CancelLeasedJobAsync(Guid jobId, Guid leaseToken, DateTimeOffset now, CancellationToken cancellationToken);
|
||||
Task RetryLeasedJobAsync(Guid jobId, Guid leaseToken, DateTimeOffset now, DateTimeOffset retryAt, string code, string safeMessage, CancellationToken cancellationToken);
|
||||
Task FailLeasedJobAsync(Guid jobId, Guid leaseToken, DateTimeOffset now, string code, string safeMessage, CancellationToken cancellationToken);
|
||||
Task<bool> IsCancellationRequestedAsync(Guid jobId, Guid leaseToken, CancellationToken cancellationToken);
|
||||
Task<BackgroundJobQueueStatistics> GetQueueStatisticsAsync(CancellationToken cancellationToken);
|
||||
}
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
using ItWorx.KnowledgeHub.Application.Audit;
|
||||
using ItWorx.KnowledgeHub.Domain.Jobs;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Application.Jobs.UseCases;
|
||||
|
||||
public enum CancelBackgroundJobStatus
|
||||
{
|
||||
NotFound,
|
||||
Requested,
|
||||
AlreadyTerminal
|
||||
}
|
||||
|
||||
public sealed class CancelBackgroundJobUseCase(
|
||||
IBackgroundJobRepository jobs,
|
||||
TimeProvider timeProvider)
|
||||
{
|
||||
public async Task<(CancelBackgroundJobStatus Status, BackgroundJob? Job)> ExecuteAsync(
|
||||
Guid workspaceId,
|
||||
Guid jobId,
|
||||
Guid actorId,
|
||||
string? correlationId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var job = await jobs.FindAsync(workspaceId, jobId, cancellationToken);
|
||||
if (job is null)
|
||||
return (CancelBackgroundJobStatus.NotFound, null);
|
||||
if (job.IsTerminal)
|
||||
return (CancelBackgroundJobStatus.AlreadyTerminal, job);
|
||||
|
||||
var requestedAt = timeProvider.GetUtcNow();
|
||||
var requested = await jobs.RequestCancellationAsync(
|
||||
workspaceId,
|
||||
jobId,
|
||||
requestedAt,
|
||||
new AuditEventRecord(
|
||||
AuditActorKind.User, actorId, Source: "web", Action: "job.cancel_requested",
|
||||
TargetType: "background_job", TargetId: jobId.ToString("D"),
|
||||
Summary: "Annuleringsverzoek voor achtergrondtaak geregistreerd.",
|
||||
correlationId, requestedAt),
|
||||
cancellationToken);
|
||||
if (!requested)
|
||||
{
|
||||
job = await jobs.FindAsync(workspaceId, jobId, cancellationToken);
|
||||
if (job is null)
|
||||
return (CancelBackgroundJobStatus.NotFound, null);
|
||||
if (job.IsTerminal)
|
||||
return (CancelBackgroundJobStatus.AlreadyTerminal, job);
|
||||
return job.CancelRequestedAt is not null
|
||||
? (CancelBackgroundJobStatus.Requested, job)
|
||||
: (CancelBackgroundJobStatus.NotFound, null);
|
||||
}
|
||||
job = await jobs.FindAsync(workspaceId, jobId, cancellationToken);
|
||||
if (job is null)
|
||||
return (CancelBackgroundJobStatus.NotFound, null);
|
||||
return (CancelBackgroundJobStatus.Requested, job);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class BackgroundJobQueries(IBackgroundJobRepository jobs)
|
||||
{
|
||||
public Task<BackgroundJob?> GetAsync(Guid workspaceId, Guid jobId, CancellationToken cancellationToken) =>
|
||||
jobs.FindAsync(workspaceId, jobId, cancellationToken);
|
||||
|
||||
public Task<IReadOnlyList<BackgroundJob>> ListAsync(Guid workspaceId, int limit, CancellationToken cancellationToken) =>
|
||||
jobs.ListAsync(workspaceId, Math.Clamp(limit, 1, 100), cancellationToken);
|
||||
}
|
||||
+140
@@ -0,0 +1,140 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using ItWorx.KnowledgeHub.Application.Audit;
|
||||
using ItWorx.KnowledgeHub.Domain.Jobs;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Application.Jobs.UseCases;
|
||||
|
||||
public sealed record QueueJobCommand(
|
||||
Guid WorkspaceId,
|
||||
Guid ActorId,
|
||||
string IdempotencyKey,
|
||||
string? CorrelationId);
|
||||
|
||||
public sealed record QueueCourseTaggingJobCommand(
|
||||
Guid WorkspaceId,
|
||||
Guid ActorId,
|
||||
Guid RootPageId,
|
||||
string Tag,
|
||||
string IdempotencyKey,
|
||||
string? CorrelationId);
|
||||
|
||||
public sealed class QueueBackgroundJobUseCase(
|
||||
IBackgroundJobRepository jobs,
|
||||
TimeProvider timeProvider,
|
||||
BackgroundJobsOptions options)
|
||||
{
|
||||
private static readonly JsonSerializerOptions PayloadOptions = new(JsonSerializerDefaults.Web);
|
||||
|
||||
public async Task<EnqueueJobResult> QueueCourseTaggingAsync(
|
||||
QueueCourseTaggingJobCommand command,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var tag = NormalizeTag(command.Tag);
|
||||
if (command.RootPageId == Guid.Empty || tag is null || !TryValidateKey(command.IdempotencyKey))
|
||||
throw new ArgumentException("Ongeldige cursustagtaak.");
|
||||
|
||||
var payload = JsonSerializer.Serialize(new { rootPageId = command.RootPageId, tag }, PayloadOptions);
|
||||
var job = Create(
|
||||
command.WorkspaceId, command.ActorId, BackgroundJobType.CourseTagging, payload,
|
||||
command.IdempotencyKey, command.CorrelationId, total: 0);
|
||||
return await EnqueueAsync(job, command.RootPageId, tag, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<EnqueueJobResult> QueueBackupAsync(QueueJobCommand command, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!TryValidateKey(command.IdempotencyKey))
|
||||
throw new ArgumentException("Een geldige Idempotency-Key is verplicht.");
|
||||
|
||||
var payload = "{}";
|
||||
var job = Create(
|
||||
command.WorkspaceId, command.ActorId, BackgroundJobType.Backup, payload,
|
||||
command.IdempotencyKey, command.CorrelationId, total: 1);
|
||||
return await EnqueueAsync(job, null, null, cancellationToken);
|
||||
}
|
||||
|
||||
private Task<EnqueueJobResult> EnqueueAsync(
|
||||
BackgroundJob job,
|
||||
Guid? rootPageId,
|
||||
string? tag,
|
||||
CancellationToken cancellationToken) =>
|
||||
jobs.EnqueueAsync(
|
||||
job, rootPageId, tag, options.MaxCoursePages,
|
||||
new BackgroundJobQueueLimits(options.MaxPendingPerWorkspace, options.MaxPendingPerActor, options.MaxCreatedPerDay),
|
||||
new AuditEventRecord(
|
||||
AuditActorKind.User,
|
||||
job.ActorId,
|
||||
Source: "web",
|
||||
Action: "job.queued",
|
||||
TargetType: "background_job",
|
||||
TargetId: job.Id.ToString("D"),
|
||||
Summary: $"Achtergrondtaak '{job.Type}' toegevoegd aan de wachtrij.",
|
||||
job.CorrelationId,
|
||||
job.CreatedAt),
|
||||
cancellationToken);
|
||||
|
||||
private BackgroundJob Create(
|
||||
Guid workspaceId,
|
||||
Guid actorId,
|
||||
BackgroundJobType type,
|
||||
string payload,
|
||||
string idempotencyKey,
|
||||
string? correlationId,
|
||||
int total)
|
||||
{
|
||||
var now = timeProvider.GetUtcNow();
|
||||
var keyHash = Hash(idempotencyKey.Trim());
|
||||
var payloadHash = Hash($"{type}:{payload}");
|
||||
return BackgroundJob.Create(
|
||||
Guid.NewGuid(), workspaceId, actorId, type, payload, keyHash, payloadHash,
|
||||
correlationId, now, total);
|
||||
}
|
||||
|
||||
private static string? NormalizeTag(string? raw)
|
||||
{
|
||||
var tag = raw?.Trim();
|
||||
if (string.IsNullOrEmpty(tag) || tag.Length > 64 || tag.Any(char.IsControl))
|
||||
return null;
|
||||
return tag.ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static bool TryValidateKey(string? key) =>
|
||||
!string.IsNullOrWhiteSpace(key)
|
||||
&& key.Length <= 128
|
||||
&& key.Trim().Length > 0
|
||||
&& !key.Any(char.IsControl);
|
||||
|
||||
private static string Hash(string value) =>
|
||||
Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(value)));
|
||||
}
|
||||
|
||||
public sealed class BackgroundJobsOptions
|
||||
{
|
||||
public const string SectionName = "KnowledgeHub:Jobs";
|
||||
public bool Enabled { get; set; } = true;
|
||||
public int MaxCoursePages { get; set; } = 100_000;
|
||||
public int MaxPendingPerWorkspace { get; set; } = 10;
|
||||
public int MaxPendingPerActor { get; set; } = 3;
|
||||
public int MaxCreatedPerDay { get; set; } = 100;
|
||||
public int MaxConcurrency { get; set; } = 2;
|
||||
public int LeaseSeconds { get; set; } = 120;
|
||||
public int LeaseRenewSeconds { get; set; } = 30;
|
||||
public int PollMilliseconds { get; set; } = 1000;
|
||||
public int SsePollMilliseconds { get; set; } = 1000;
|
||||
public int SseHeartbeatSeconds { get; set; } = 15;
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
MaxCoursePages = Math.Clamp(MaxCoursePages, 1, 100_000);
|
||||
MaxPendingPerWorkspace = Math.Clamp(MaxPendingPerWorkspace, 1, 100);
|
||||
MaxPendingPerActor = Math.Clamp(MaxPendingPerActor, 1, MaxPendingPerWorkspace);
|
||||
MaxCreatedPerDay = Math.Clamp(MaxCreatedPerDay, 1, 1000);
|
||||
MaxConcurrency = Math.Clamp(MaxConcurrency, 1, 8);
|
||||
LeaseSeconds = Math.Clamp(LeaseSeconds, 30, 600);
|
||||
LeaseRenewSeconds = Math.Clamp(LeaseRenewSeconds, 10, Math.Max(10, LeaseSeconds / 2));
|
||||
PollMilliseconds = Math.Clamp(PollMilliseconds, 100, 30_000);
|
||||
SsePollMilliseconds = Math.Clamp(SsePollMilliseconds, 250, 10_000);
|
||||
SseHeartbeatSeconds = Math.Clamp(SseHeartbeatSeconds, 5, 60);
|
||||
}
|
||||
}
|
||||
+10
-3
@@ -1,3 +1,5 @@
|
||||
using ItWorx.KnowledgeHub.Application.Operations;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Application.Knowledge.UseCases;
|
||||
|
||||
public sealed record SearchPagesQuery(
|
||||
@@ -18,8 +20,13 @@ public sealed record SearchPagesQuery(
|
||||
/// reachable by an unauthenticated caller" (the endpoint itself requires Reader),
|
||||
/// not per-page grants that do not exist in this product yet.
|
||||
/// </summary>
|
||||
public sealed class SearchPagesUseCase(ISearchRepository search)
|
||||
public sealed class SearchPagesUseCase(ISearchRepository search, PlatformMetrics? metrics = null)
|
||||
{
|
||||
public Task<SearchPage> ExecuteAsync(SearchPagesQuery query, CancellationToken cancellationToken) =>
|
||||
search.SearchAsync(query.WorkspaceId, query.Query, query.Tag, query.DocumentType, query.Cursor, query.Limit, cancellationToken);
|
||||
public async Task<SearchPage> ExecuteAsync(SearchPagesQuery query, CancellationToken cancellationToken)
|
||||
{
|
||||
using var measurement = metrics?.Start("search");
|
||||
var result = await search.SearchAsync(query.WorkspaceId, query.Query, query.Tag, query.DocumentType, query.Cursor, query.Limit, cancellationToken);
|
||||
measurement?.Complete(result.Items.Count);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
namespace ItWorx.KnowledgeHub.Application.Operations;
|
||||
|
||||
public sealed record BackupEvidence(
|
||||
bool Protected,
|
||||
int ProtectionVersion,
|
||||
DateTimeOffset? IntegrityVerifiedAt,
|
||||
DateTimeOffset? RestoreVerifiedAt,
|
||||
bool? LastRestoreSucceeded);
|
||||
|
||||
public enum BackupProtectionResult { Updated, NotFound, Conflict }
|
||||
|
||||
/// <summary>Recovery evidence lives beside the backup, independently of the source database.</summary>
|
||||
public interface IBackupEvidenceStore
|
||||
{
|
||||
Task<BackupEvidence?> GetAsync(
|
||||
string backupId, CancellationToken cancellationToken, string? expectedManifestSha256 = null);
|
||||
Task RecordRestoreAsync(string backupId, RestoreReport report, CancellationToken cancellationToken);
|
||||
Task<BackupProtectionResult> SetProtectionAsync(
|
||||
string backupId, bool protect, int expectedVersion, Guid actorId, CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
namespace ItWorx.KnowledgeHub.Application.Operations;
|
||||
|
||||
public sealed record OperationalTelemetrySnapshot(
|
||||
long QueuedJobs, long RunningJobs, long FailedJobs,
|
||||
DateTimeOffset? OldestQueuedJobAt, DateTimeOffset? OldestPendingIngestionAt,
|
||||
DateTimeOffset? LatestInventoryCaptureAt, DateTimeOffset? LatestVerifiedBackupAt);
|
||||
|
||||
public interface IOperationalTelemetryReader
|
||||
{
|
||||
Task<OperationalTelemetrySnapshot> ReadAsync(CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -16,7 +16,8 @@ public sealed record RestoreReport(
|
||||
IReadOnlyList<RestoreTableReport> Tables,
|
||||
int AttachmentsRestored,
|
||||
int AttachmentsExpected,
|
||||
IReadOnlyList<string> Warnings)
|
||||
IReadOnlyList<string> Warnings,
|
||||
string? ManifestSha256 = null)
|
||||
{
|
||||
public bool AttachmentsMatch => AttachmentsRestored == AttachmentsExpected;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Application.Operations;
|
||||
|
||||
/// <summary>Closed dimensions only: no actor, SQL, source URL, document, path or correlation label.</summary>
|
||||
public sealed class PlatformMetrics : IDisposable
|
||||
{
|
||||
private readonly Meter meter = new("ItWorx.KnowledgeHub.Platform");
|
||||
private readonly Counter<long> operations;
|
||||
private readonly Histogram<double> duration;
|
||||
private readonly Histogram<long> records;
|
||||
private QueueState? queueState;
|
||||
private OperationalAges? operationalAges;
|
||||
|
||||
public PlatformMetrics()
|
||||
{
|
||||
operations = meter.CreateCounter<long>("platform_operations_total");
|
||||
duration = meter.CreateHistogram<double>("platform_operation_duration_seconds", "s");
|
||||
records = meter.CreateHistogram<long>("platform_operation_records");
|
||||
meter.CreateObservableGauge("platform_jobs", ObserveJobs);
|
||||
meter.CreateObservableGauge("platform_queue_age_seconds", ObserveQueueAge, "s");
|
||||
meter.CreateObservableGauge("platform_evidence_age_seconds", ObserveEvidenceAges, "s");
|
||||
}
|
||||
|
||||
public void SetQueueState(long queued, long running, long failed, double? oldestPendingAgeSeconds) =>
|
||||
Volatile.Write(ref queueState, new QueueState(Math.Max(0, queued), Math.Max(0, running), Math.Max(0, failed), ValidAge(oldestPendingAgeSeconds)));
|
||||
|
||||
public void SetOperationalAges(double? ingestionAge, double? inventoryAge, double? backupAge) =>
|
||||
Volatile.Write(ref operationalAges, new OperationalAges(ValidAge(ingestionAge), ValidAge(inventoryAge), ValidAge(backupAge)));
|
||||
|
||||
public void ClearSnapshots()
|
||||
{
|
||||
Volatile.Write(ref queueState, null);
|
||||
Volatile.Write(ref operationalAges, null);
|
||||
}
|
||||
|
||||
private static double? ValidAge(double? age) => age is { } value && double.IsFinite(value) ? Math.Max(0, value) : null;
|
||||
|
||||
private IEnumerable<System.Diagnostics.Metrics.Measurement<long>> ObserveJobs()
|
||||
{
|
||||
if (Volatile.Read(ref queueState) is not { } state) yield break;
|
||||
yield return new(state.Queued, new KeyValuePair<string, object?>("state", "queued"));
|
||||
yield return new(state.Running, new KeyValuePair<string, object?>("state", "running"));
|
||||
yield return new(state.Failed, new KeyValuePair<string, object?>("state", "failed"));
|
||||
}
|
||||
|
||||
private IEnumerable<System.Diagnostics.Metrics.Measurement<double>> ObserveQueueAge()
|
||||
{
|
||||
if (Volatile.Read(ref queueState)?.OldestPendingAge is { } age)
|
||||
yield return new(age, new KeyValuePair<string, object?>("queue", "jobs"));
|
||||
}
|
||||
|
||||
private IEnumerable<System.Diagnostics.Metrics.Measurement<double>> ObserveEvidenceAges()
|
||||
{
|
||||
if (Volatile.Read(ref operationalAges) is not { } ages) yield break;
|
||||
if (ages.Ingestion is { } ingestion) yield return new(ingestion, new KeyValuePair<string, object?>("evidence", "ingestion_queue"));
|
||||
if (ages.Inventory is { } inventory) yield return new(inventory, new KeyValuePair<string, object?>("evidence", "inventory"));
|
||||
if (ages.Backup is { } backup) yield return new(backup, new KeyValuePair<string, object?>("evidence", "backup"));
|
||||
}
|
||||
|
||||
private sealed record QueueState(long Queued, long Running, long Failed, double? OldestPendingAge);
|
||||
private sealed record OperationalAges(double? Ingestion, double? Inventory, double? Backup);
|
||||
|
||||
public void Record(string operation, string outcome, double seconds, long recordCount = 0)
|
||||
{
|
||||
var tags = new TagList
|
||||
{
|
||||
{ "operation", operation is "database" or "search" or "inventory" or "backup" or "job" ? operation : "other" },
|
||||
{ "outcome", outcome is "success" or "failure" or "cancelled" ? outcome : "other" }
|
||||
};
|
||||
operations.Add(1, tags);
|
||||
duration.Record(double.IsFinite(seconds) ? Math.Max(0, seconds) : 0, tags);
|
||||
records.Record(Math.Max(0, recordCount), tags);
|
||||
}
|
||||
|
||||
public Measurement Start(string operation) => new(this, operation);
|
||||
public void Dispose() => meter.Dispose();
|
||||
|
||||
public sealed class Measurement(PlatformMetrics owner, string operation) : IDisposable
|
||||
{
|
||||
private readonly long started = Stopwatch.GetTimestamp();
|
||||
private string outcome = "failure";
|
||||
private long records;
|
||||
public void Complete(long recordCount = 0) { outcome = "success"; records = recordCount; }
|
||||
public void Cancel() => outcome = "cancelled";
|
||||
public void Dispose() => owner.Record(operation, outcome, Stopwatch.GetElapsedTime(started).TotalSeconds, records);
|
||||
}
|
||||
}
|
||||
+3
-1
@@ -8,10 +8,11 @@ public sealed record CreateBackupCommand(Guid ActorId, string? CorrelationId);
|
||||
/// an audit event on both success and failure — a failed backup is itself an
|
||||
/// operationally significant event (docs/19-observability.md "repeated backup
|
||||
/// failure" alert).</summary>
|
||||
public sealed class CreateBackupUseCase(IBackupService backups, IAuditEventRecorder auditEvents, TimeProvider timeProvider)
|
||||
public sealed class CreateBackupUseCase(IBackupService backups, IAuditEventRecorder auditEvents, TimeProvider timeProvider, PlatformMetrics? metrics = null)
|
||||
{
|
||||
public async Task<BackupSummary> ExecuteAsync(CreateBackupCommand command, CancellationToken cancellationToken)
|
||||
{
|
||||
using var measurement = metrics?.Start("backup");
|
||||
var now = timeProvider.GetUtcNow();
|
||||
try
|
||||
{
|
||||
@@ -30,6 +31,7 @@ public sealed class CreateBackupUseCase(IBackupService backups, IAuditEventRecor
|
||||
now),
|
||||
cancellationToken);
|
||||
|
||||
measurement?.Complete(summary.AttachmentCount);
|
||||
return summary;
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
using ItWorx.KnowledgeHub.Domain.Platform;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Application.Platform;
|
||||
|
||||
public sealed record PlatformSettingsSnapshot(
|
||||
PlatformSettingsValues Values,
|
||||
int Version,
|
||||
bool IsDefault,
|
||||
DateTimeOffset? UpdatedAt);
|
||||
|
||||
public interface IPlatformSettingsReader
|
||||
{
|
||||
Task<PlatformSettingsSnapshot> GetEffectiveAsync(CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public interface IPlatformSettingsRepository : IPlatformSettingsReader
|
||||
{
|
||||
Task<PlatformSettings?> FindPersistedAsync(CancellationToken cancellationToken);
|
||||
|
||||
void Add(PlatformSettings settings);
|
||||
|
||||
/// <summary>Returns false for an optimistic-concurrency or singleton-insert race.</summary>
|
||||
Task<bool> SaveChangesAsync(CancellationToken cancellationToken);
|
||||
}
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
using System.Text.RegularExpressions;
|
||||
using ItWorx.KnowledgeHub.Application.Audit;
|
||||
using ItWorx.KnowledgeHub.Domain.Platform;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Application.Platform.UseCases;
|
||||
|
||||
public sealed class GetPlatformSettingsUseCase(IPlatformSettingsReader settings)
|
||||
{
|
||||
public Task<PlatformSettingsSnapshot> ExecuteAsync(CancellationToken cancellationToken) =>
|
||||
settings.GetEffectiveAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public sealed record UpdatePlatformSettingsCommand(
|
||||
int ExpectedVersion,
|
||||
Guid ActorId,
|
||||
string? CorrelationId,
|
||||
PlatformSettingsValues Values);
|
||||
|
||||
public enum UpdatePlatformSettingsStatus { Success, PreconditionFailed, ValidationFailed }
|
||||
|
||||
public sealed record UpdatePlatformSettingsResult(
|
||||
UpdatePlatformSettingsStatus Status,
|
||||
PlatformSettingsSnapshot? Snapshot,
|
||||
IReadOnlyList<string> Errors)
|
||||
{
|
||||
public static UpdatePlatformSettingsResult Failure(UpdatePlatformSettingsStatus status, params string[] errors) =>
|
||||
new(status, null, errors);
|
||||
}
|
||||
|
||||
/// <summary>Validates and persists versioned operator policy. Backup values are
|
||||
/// expectations only because their scheduler and retention executor are external.</summary>
|
||||
public sealed class UpdatePlatformSettingsUseCase(
|
||||
IPlatformSettingsRepository settings,
|
||||
IAuditEventRecorder auditEvents,
|
||||
TimeProvider timeProvider)
|
||||
{
|
||||
private const long MaximumIntakeBytes = 104_857_600;
|
||||
private const int MaximumIntakeSources = 20;
|
||||
|
||||
public async Task<UpdatePlatformSettingsResult> ExecuteAsync(
|
||||
UpdatePlatformSettingsCommand command,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var errors = Validate(command.Values);
|
||||
if (errors.Count > 0)
|
||||
{
|
||||
return new(UpdatePlatformSettingsStatus.ValidationFailed, null, errors);
|
||||
}
|
||||
|
||||
var current = await settings.FindPersistedAsync(cancellationToken);
|
||||
if (current is null)
|
||||
{
|
||||
if (command.ExpectedVersion != 0)
|
||||
{
|
||||
return UpdatePlatformSettingsResult.Failure(UpdatePlatformSettingsStatus.PreconditionFailed);
|
||||
}
|
||||
|
||||
current = PlatformSettings.Create(command.Values, timeProvider.GetUtcNow());
|
||||
settings.Add(current);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (current.Version != command.ExpectedVersion)
|
||||
{
|
||||
return UpdatePlatformSettingsResult.Failure(UpdatePlatformSettingsStatus.PreconditionFailed);
|
||||
}
|
||||
|
||||
current.Replace(command.Values, timeProvider.GetUtcNow());
|
||||
}
|
||||
|
||||
auditEvents.Add(new AuditEventRecord(
|
||||
AuditActorKind.User,
|
||||
command.ActorId,
|
||||
"web",
|
||||
"platform.settings.updated",
|
||||
"PlatformSettings",
|
||||
PlatformSettings.SingletonId.ToString(),
|
||||
"Operator platform settings updated.",
|
||||
command.CorrelationId,
|
||||
timeProvider.GetUtcNow()));
|
||||
|
||||
if (!await settings.SaveChangesAsync(cancellationToken))
|
||||
{
|
||||
return UpdatePlatformSettingsResult.Failure(UpdatePlatformSettingsStatus.PreconditionFailed);
|
||||
}
|
||||
|
||||
return new(
|
||||
UpdatePlatformSettingsStatus.Success,
|
||||
new PlatformSettingsSnapshot(current.ToValues(), current.Version, IsDefault: false, current.UpdatedAt),
|
||||
[]);
|
||||
}
|
||||
|
||||
private static List<string> Validate(PlatformSettingsValues? values)
|
||||
{
|
||||
var errors = new List<string>();
|
||||
if (values is null)
|
||||
{
|
||||
errors.Add("Platforminstellingen ontbreken.");
|
||||
return errors;
|
||||
}
|
||||
|
||||
var displayName = values.DisplayName?.Trim();
|
||||
if (string.IsNullOrWhiteSpace(displayName) || displayName.Length > 100 || displayName.Any(char.IsControl))
|
||||
{
|
||||
errors.Add("De sitenaam moet 1 tot 100 tekens bevatten.");
|
||||
}
|
||||
if (values.PrimaryLocale is not "nl-BE") errors.Add("Alleen nl-BE is momenteel beschikbaar.");
|
||||
if (!IsKnownTimeZone(values.TimeZone)) errors.Add("De tijdzone moet een bekende IANA-tijdzone zijn.");
|
||||
|
||||
if (values.IngestionProposals is null
|
||||
|| values.IngestionProposals.MaximumSourceCount is < 1 or > MaximumIntakeSources
|
||||
|| values.IngestionProposals.MaximumSourceBytes is < 1 or > MaximumIntakeBytes
|
||||
|| values.IngestionProposals.FailedRetryLimit is < 1 or > 3)
|
||||
{
|
||||
errors.Add("De intake-instellingen overschrijden de vaste veiligheidslimieten.");
|
||||
}
|
||||
|
||||
var backup = values.BackupPolicyExpectation;
|
||||
if (backup is null
|
||||
|| backup.ExpectedIntervalHours is < 1 or > 8760
|
||||
|| string.IsNullOrWhiteSpace(backup.ScheduleDescription)
|
||||
|| backup.ScheduleDescription.Length > 200
|
||||
|| backup.ScheduleDescription.Any(char.IsControl)
|
||||
|| backup.DailyRetentionCount is < 1 or > 365
|
||||
|| backup.WeeklyRetentionCount is < 1 or > 104
|
||||
|| backup.MonthlyRetentionCount is < 1 or > 120
|
||||
|| backup.ProtectedBackupCount is < 1 or > 100
|
||||
|| backup.RestoreVerificationMaxAgeDays is < 1 or > 365)
|
||||
{
|
||||
errors.Add("De externe back-upverwachting bevat waarden buiten de toegestane grenzen.");
|
||||
}
|
||||
|
||||
if (values.InventoryFreshnessPolicy is null || values.InventoryFreshnessPolicy.WarningAfterHours is < 1 or > 8760)
|
||||
{
|
||||
errors.Add("De inventariswaarschuwing moet tussen 1 en 8760 uur liggen.");
|
||||
}
|
||||
|
||||
return errors;
|
||||
}
|
||||
|
||||
private static bool IsKnownTimeZone(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value) || value.Length > 100 || !Regex.IsMatch(value, "^[A-Za-z0-9_+./-]+$", RegexOptions.CultureInvariant))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
_ = TimeZoneInfo.FindSystemTimeZoneById(value);
|
||||
return true;
|
||||
}
|
||||
catch (TimeZoneNotFoundException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
catch (InvalidTimeZoneException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
+147
@@ -0,0 +1,147 @@
|
||||
using ItWorx.KnowledgeHub.Domain.Platform;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Application.Platform.UseCases;
|
||||
|
||||
public static class UserPreferencesDefaults
|
||||
{
|
||||
public static readonly UserPreferencesValues Values = new(
|
||||
"system",
|
||||
"comfortable",
|
||||
PrimaryNavigationCollapsed: false,
|
||||
ContextPanelOpen: false,
|
||||
NavigationOrder: ["/", "/infrastructure", "/infrastructure/services", "/projects", "/runbooks", "/knowledge", "/activity", "/search"],
|
||||
DashboardWidgetOrder: ["attention", "server", "projects", "activity", "runbooks", "favorites", "quick-actions"],
|
||||
HiddenDashboardWidgets: [],
|
||||
DefaultLandingPage: "/",
|
||||
CatalogView: "list",
|
||||
ContextPanelWidth: 304,
|
||||
NavigationWidth: 224);
|
||||
}
|
||||
|
||||
public sealed class GetUserPreferencesUseCase(IUserPreferencesRepository preferences)
|
||||
{
|
||||
public async Task<UserPreferencesSnapshot> ExecuteAsync(Guid workspaceId, Guid userId, CancellationToken cancellationToken)
|
||||
{
|
||||
var current = await preferences.FindAsync(workspaceId, userId, cancellationToken);
|
||||
return current is null
|
||||
? new UserPreferencesSnapshot(UserPreferencesDefaults.Values, 0, IsDefault: true, UpdatedAt: null)
|
||||
: new UserPreferencesSnapshot(current.ToValues(), current.Version, IsDefault: false, current.UpdatedAt);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record UpdateUserPreferencesCommand(
|
||||
Guid WorkspaceId,
|
||||
Guid UserId,
|
||||
int ExpectedVersion,
|
||||
UserPreferencesValues Values);
|
||||
|
||||
public enum UpdateUserPreferencesStatus { Success, PreconditionFailed, ValidationFailed }
|
||||
|
||||
public sealed record UpdateUserPreferencesResult(
|
||||
UpdateUserPreferencesStatus Status,
|
||||
UserPreferencesSnapshot? Snapshot,
|
||||
IReadOnlyList<string> Errors)
|
||||
{
|
||||
public static UpdateUserPreferencesResult Failure(UpdateUserPreferencesStatus status, params string[] errors) =>
|
||||
new(status, null, errors);
|
||||
}
|
||||
|
||||
/// <summary>Validates complete user preference replacements and commits by ETag.</summary>
|
||||
public sealed class UpdateUserPreferencesUseCase(
|
||||
IUserPreferencesRepository preferences,
|
||||
TimeProvider timeProvider)
|
||||
{
|
||||
private static readonly HashSet<string> NavigationRoutes = new(StringComparer.Ordinal)
|
||||
{
|
||||
"/", "/infrastructure", "/infrastructure/services", "/infrastructure/storage", "/projects",
|
||||
"/runbooks", "/knowledge", "/activity", "/search", "/settings"
|
||||
};
|
||||
|
||||
private static readonly HashSet<string> LandingRoutes = new(StringComparer.Ordinal)
|
||||
{
|
||||
"/", "/infrastructure", "/infrastructure/services", "/projects", "/knowledge"
|
||||
};
|
||||
|
||||
private static readonly HashSet<string> DashboardWidgets = new(StringComparer.Ordinal)
|
||||
{
|
||||
"attention", "server", "projects", "activity", "runbooks", "favorites", "quick-actions"
|
||||
};
|
||||
|
||||
public async Task<UpdateUserPreferencesResult> ExecuteAsync(
|
||||
UpdateUserPreferencesCommand command,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var errors = Validate(command.Values);
|
||||
if (errors.Count > 0)
|
||||
{
|
||||
return new(UpdateUserPreferencesStatus.ValidationFailed, null, errors);
|
||||
}
|
||||
|
||||
var current = await preferences.FindAsync(command.WorkspaceId, command.UserId, cancellationToken);
|
||||
if (current is null)
|
||||
{
|
||||
if (command.ExpectedVersion != 0)
|
||||
{
|
||||
return UpdateUserPreferencesResult.Failure(UpdateUserPreferencesStatus.PreconditionFailed);
|
||||
}
|
||||
|
||||
current = UserPreferences.Create(command.WorkspaceId, command.UserId, command.Values, timeProvider.GetUtcNow());
|
||||
preferences.Add(current);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (current.Version != command.ExpectedVersion)
|
||||
{
|
||||
return UpdateUserPreferencesResult.Failure(UpdateUserPreferencesStatus.PreconditionFailed);
|
||||
}
|
||||
|
||||
current.Replace(command.Values, timeProvider.GetUtcNow());
|
||||
}
|
||||
|
||||
if (!await preferences.SaveChangesAsync(cancellationToken))
|
||||
{
|
||||
return UpdateUserPreferencesResult.Failure(UpdateUserPreferencesStatus.PreconditionFailed);
|
||||
}
|
||||
|
||||
return new(
|
||||
UpdateUserPreferencesStatus.Success,
|
||||
new UserPreferencesSnapshot(current.ToValues(), current.Version, IsDefault: false, current.UpdatedAt),
|
||||
[]);
|
||||
}
|
||||
|
||||
private static List<string> Validate(UserPreferencesValues? values)
|
||||
{
|
||||
var errors = new List<string>();
|
||||
if (values is null)
|
||||
{
|
||||
errors.Add("Voorkeuren ontbreken.");
|
||||
return errors;
|
||||
}
|
||||
|
||||
if (values.Theme is not ("system" or "light" or "dark")) errors.Add("Ongeldig thema.");
|
||||
if (values.Density is not ("comfortable" or "compact")) errors.Add("Ongeldige datadichtheid.");
|
||||
if (values.CatalogView is not ("list" or "grid")) errors.Add("Ongeldige catalogusweergave.");
|
||||
if (!LandingRoutes.Contains(values.DefaultLandingPage ?? string.Empty)) errors.Add("Ongeldige standaardlandingspagina.");
|
||||
ValidateList(values.NavigationOrder, NavigationRoutes, "navigatie", errors);
|
||||
ValidateList(values.DashboardWidgetOrder, DashboardWidgets, "dashboardwidgets", errors);
|
||||
ValidateList(values.HiddenDashboardWidgets, DashboardWidgets, "verborgen dashboardwidgets", errors);
|
||||
if (values.HiddenDashboardWidgets is not null
|
||||
&& values.DashboardWidgetOrder is not null
|
||||
&& values.HiddenDashboardWidgets.Except(values.DashboardWidgetOrder, StringComparer.Ordinal).Any())
|
||||
{
|
||||
errors.Add("Verborgen widgets moeten in de widgetvolgorde voorkomen.");
|
||||
}
|
||||
if (values.ContextPanelWidth is < 256 or > 440) errors.Add("De contextpaneelbreedte moet tussen 256 en 440 pixels liggen.");
|
||||
if (values.NavigationWidth is < 192 or > 320) errors.Add("De navigatiebreedte moet tussen 192 en 320 pixels liggen.");
|
||||
return errors;
|
||||
}
|
||||
|
||||
private static void ValidateList(IReadOnlyList<string>? values, HashSet<string> allowed, string label, List<string> errors)
|
||||
{
|
||||
if (values is null || values.Count > allowed.Count || values.Any(value => !allowed.Contains(value))
|
||||
|| values.Distinct(StringComparer.Ordinal).Count() != values.Count)
|
||||
{
|
||||
errors.Add($"De lijst met {label} bevat onbekende of dubbele waarden.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
using ItWorx.KnowledgeHub.Domain.Platform;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Application.Platform;
|
||||
|
||||
public sealed record UserPreferencesSnapshot(UserPreferencesValues Values, int Version, bool IsDefault, DateTimeOffset? UpdatedAt);
|
||||
|
||||
public interface IUserPreferencesRepository
|
||||
{
|
||||
Task<UserPreferences?> FindAsync(Guid workspaceId, Guid userId, CancellationToken cancellationToken);
|
||||
|
||||
void Add(UserPreferences preferences);
|
||||
|
||||
/// <summary>Returns false for an optimistic-concurrency or first-insert race.</summary>
|
||||
Task<bool> SaveChangesAsync(CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
using System.Globalization;
|
||||
using ItWorx.KnowledgeHub.Application.Audit;
|
||||
using ItWorx.KnowledgeHub.Application.Common;
|
||||
using ItWorx.KnowledgeHub.Domain.Sources;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Application.Sources;
|
||||
|
||||
public sealed class DataSourceUseCases(IDataSourceRepository sources, TimeProvider clock)
|
||||
{
|
||||
public Task<IReadOnlyList<DataSource>> ListAsync(Guid workspaceId, Guid? afterId, int limit, CancellationToken cancellationToken) =>
|
||||
sources.ListAsync(workspaceId, afterId, limit, cancellationToken);
|
||||
|
||||
public async Task<OperationResultValue<DataSource>> RegisterAsync(Guid workspaceId, string key, string name, string kind, int intervalHours, Guid actorId, string? correlationId, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!Enum.TryParse<DataSourceKind>(kind, out var parsedKind) || !Enum.IsDefined(parsedKind))
|
||||
return OperationResultValue.Failure<DataSource>(OperationStatus.ValidationFailed);
|
||||
// This key is owned by the inventory importer. A generic registration
|
||||
// could otherwise label its observations as a different source kind.
|
||||
if (string.Equals(key, "inventory:default", StringComparison.OrdinalIgnoreCase))
|
||||
return OperationResultValue.Failure<DataSource>(OperationStatus.ValidationFailed);
|
||||
DataSource source;
|
||||
try { source = DataSource.Create(workspaceId, key, name, parsedKind, intervalHours, clock.GetUtcNow()); }
|
||||
catch (ArgumentException) { return OperationResultValue.Failure<DataSource>(OperationStatus.ValidationFailed); }
|
||||
sources.Add(source);
|
||||
return await sources.SaveAsync(Audit(source, actorId, "source.registered", correlationId), cancellationToken)
|
||||
? OperationResultValue.Success(source) : OperationResultValue.Failure<DataSource>(OperationStatus.ConcurrencyConflict);
|
||||
}
|
||||
|
||||
public async Task<OperationResultValue<DataSource>> ConfigureAsync(Guid workspaceId, Guid id, string expectedVersion, string name, bool enabled, int intervalHours, Guid actorId, string? correlationId, CancellationToken cancellationToken)
|
||||
{
|
||||
var source = await sources.FindAsync(workspaceId, id, cancellationToken);
|
||||
if (source is null) return OperationResultValue.Failure<DataSource>(OperationStatus.NotFound);
|
||||
if (expectedVersion != source.Version.ToString(CultureInfo.InvariantCulture)) return OperationResultValue.Failure<DataSource>(OperationStatus.ConcurrencyConflict);
|
||||
try { source.Configure(name, enabled, intervalHours); }
|
||||
catch (ArgumentException) { return OperationResultValue.Failure<DataSource>(OperationStatus.ValidationFailed); }
|
||||
return await sources.SaveAsync(Audit(source, actorId, "source.configured", correlationId), cancellationToken)
|
||||
? OperationResultValue.Success(source) : OperationResultValue.Failure<DataSource>(OperationStatus.ConcurrencyConflict);
|
||||
}
|
||||
|
||||
public async Task<OperationResultValue<DataSource>> ReportAsync(Guid workspaceId, Guid id, string expectedVersion, bool success, long? records, string? errorCode, string? runCorrelationId, Guid actorId, string? correlationId, CancellationToken cancellationToken)
|
||||
{
|
||||
var source = await sources.FindAsync(workspaceId, id, cancellationToken);
|
||||
if (source is null) return OperationResultValue.Failure<DataSource>(OperationStatus.NotFound);
|
||||
if (expectedVersion != source.Version.ToString(CultureInfo.InvariantCulture)) return OperationResultValue.Failure<DataSource>(OperationStatus.ConcurrencyConflict);
|
||||
try { source.Observe(success, records, errorCode, runCorrelationId, clock.GetUtcNow()); }
|
||||
catch (ArgumentException) { return OperationResultValue.Failure<DataSource>(OperationStatus.ValidationFailed); }
|
||||
return await sources.SaveAsync(Audit(source, actorId, "source.observation-recorded", correlationId), cancellationToken)
|
||||
? OperationResultValue.Success(source) : OperationResultValue.Failure<DataSource>(OperationStatus.ConcurrencyConflict);
|
||||
}
|
||||
|
||||
private AuditEventRecord Audit(DataSource source, Guid actorId, string action, string? correlationId) =>
|
||||
new(AuditActorKind.User, actorId, "web", action, "data-source", source.Id.ToString(), "Bronregistratie gewijzigd.", correlationId, clock.GetUtcNow());
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
using ItWorx.KnowledgeHub.Application.Audit;
|
||||
using ItWorx.KnowledgeHub.Domain.Sources;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Application.Sources;
|
||||
|
||||
public interface IDataSourceRepository
|
||||
{
|
||||
Task<IReadOnlyList<DataSource>> ListAsync(Guid workspaceId, Guid? afterId, int limit, CancellationToken cancellationToken);
|
||||
Task<DataSource?> FindAsync(Guid workspaceId, Guid id, CancellationToken cancellationToken);
|
||||
Task<DataSource?> FindByKeyAsync(Guid workspaceId, string sourceKey, CancellationToken cancellationToken);
|
||||
void Add(DataSource source);
|
||||
Task<bool> SaveAsync(AuditEventRecord? auditEvent, CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
namespace ItWorx.KnowledgeHub.Application.UnifiedSearch;
|
||||
|
||||
public enum UnifiedSearchKind
|
||||
{
|
||||
Knowledge,
|
||||
Projects,
|
||||
Services,
|
||||
Runbooks,
|
||||
Decisions
|
||||
}
|
||||
|
||||
public sealed record UnifiedSearchHit(
|
||||
string Kind,
|
||||
Guid Id,
|
||||
string Title,
|
||||
string Summary,
|
||||
string Snippet,
|
||||
DateTimeOffset UpdatedAt,
|
||||
string Href,
|
||||
double Rank);
|
||||
|
||||
public sealed record UnifiedSearchCountsResult(
|
||||
int Knowledge,
|
||||
int Projects,
|
||||
int Services,
|
||||
int Runbooks,
|
||||
int Decisions);
|
||||
|
||||
public sealed record UnifiedSearchPage(
|
||||
IReadOnlyList<UnifiedSearchHit> Items,
|
||||
UnifiedSearchCountsResult Counts,
|
||||
string? NextCursor);
|
||||
|
||||
/// <summary>Workspace-scoped search port. Implementations must apply visibility and trash
|
||||
/// predicates before producing counts, snippets, or paginated results.</summary>
|
||||
public interface IUnifiedSearchRepository
|
||||
{
|
||||
Task<UnifiedSearchPage> SearchAsync(
|
||||
Guid workspaceId,
|
||||
string query,
|
||||
string? tag,
|
||||
UnifiedSearchKind? kind,
|
||||
string? cursor,
|
||||
int limit,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
namespace ItWorx.KnowledgeHub.Application.UnifiedSearch;
|
||||
|
||||
public sealed class UnifiedSearchUseCase(IUnifiedSearchRepository repository)
|
||||
{
|
||||
public Task<UnifiedSearchPage> ExecuteAsync(
|
||||
Guid workspaceId,
|
||||
string? query,
|
||||
string? tag,
|
||||
UnifiedSearchKind? kind,
|
||||
string? cursor,
|
||||
int limit,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedQuery = (query ?? string.Empty).Trim();
|
||||
if (normalizedQuery.Length > 500)
|
||||
{
|
||||
normalizedQuery = normalizedQuery[..500];
|
||||
}
|
||||
|
||||
var normalizedTag = string.IsNullOrWhiteSpace(tag) ? null : tag.Trim().ToLowerInvariant();
|
||||
|
||||
// An empty query is useful for tag-only browsing, but never means an
|
||||
// unfiltered workspace scan.
|
||||
if (normalizedQuery.Length == 0 && normalizedTag is null)
|
||||
{
|
||||
return Task.FromResult(new UnifiedSearchPage(
|
||||
[], new UnifiedSearchCountsResult(0, 0, 0, 0, 0), null));
|
||||
}
|
||||
|
||||
return repository.SearchAsync(workspaceId, normalizedQuery, normalizedTag, kind, cursor, limit, cancellationToken);
|
||||
}
|
||||
}
|
||||
@@ -4,4 +4,10 @@ public sealed record ActivityItemResponse(
|
||||
long Id, string ActorKind, Guid? ActorId, string Source, string Action,
|
||||
string? TargetType, string? TargetId, string Summary, DateTimeOffset OccurredAt);
|
||||
|
||||
public sealed record ActivityListResponse(IReadOnlyList<ActivityItemResponse> Items);
|
||||
public sealed record ActivityListResponse(IReadOnlyList<ActivityItemResponse> Items, long? NextCursor = null);
|
||||
|
||||
public sealed record AdminAuditItemResponse(
|
||||
long Id, string ActorKind, Guid? ActorId, string Source, string Action,
|
||||
string? TargetType, string? TargetId, string Summary, DateTimeOffset OccurredAt, string? CorrelationId);
|
||||
|
||||
public sealed record AdminAuditListResponse(IReadOnlyList<AdminAuditItemResponse> Items, long? NextCursor);
|
||||
@@ -0,0 +1,33 @@
|
||||
using System.Text.Json;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Contracts.Jobs;
|
||||
|
||||
public sealed record QueueCourseTaggingJobRequest(Guid RootPageId, string Tag);
|
||||
|
||||
public sealed record QueueBackupJobRequest;
|
||||
|
||||
public sealed record BackgroundJobErrorResponse(string Code, string Message, Guid? ItemId);
|
||||
|
||||
public sealed record BackgroundJobResponse(
|
||||
Guid Id,
|
||||
Guid WorkspaceId,
|
||||
Guid ActorId,
|
||||
string Type,
|
||||
string Status,
|
||||
DateTimeOffset CreatedAt,
|
||||
DateTimeOffset? StartedAt,
|
||||
DateTimeOffset? CompletedAt,
|
||||
DateTimeOffset? CancelRequestedAt,
|
||||
bool CancelRequested,
|
||||
int Progress,
|
||||
int Total,
|
||||
int Processed,
|
||||
int Succeeded,
|
||||
int Skipped,
|
||||
int Failed,
|
||||
IReadOnlyList<BackgroundJobErrorResponse> Errors,
|
||||
JsonElement? Result,
|
||||
string? CorrelationId,
|
||||
long Version);
|
||||
|
||||
public sealed record BackgroundJobListResponse(IReadOnlyList<BackgroundJobResponse> Items);
|
||||
@@ -11,6 +11,32 @@ public sealed record BackupSummaryResponse(
|
||||
long TotalRowCount,
|
||||
int AttachmentCount,
|
||||
string SchemaVersion,
|
||||
string ProductVersion);
|
||||
string ProductVersion,
|
||||
bool Protected = false,
|
||||
string ProtectionETag = "0",
|
||||
DateTimeOffset? IntegrityVerifiedAt = null,
|
||||
DateTimeOffset? RestoreVerifiedAt = null,
|
||||
bool? LastRestoreSucceeded = null);
|
||||
|
||||
public sealed record BackupListResponse(IReadOnlyList<BackupSummaryResponse> Items);
|
||||
|
||||
public sealed record BackupProtectionRequest(bool Protected);
|
||||
|
||||
public sealed record BackupStatusResponse(
|
||||
string State,
|
||||
string ScheduleAuthority,
|
||||
string ScheduleDescription,
|
||||
DateTimeOffset? NextScheduledRun,
|
||||
DateTimeOffset? NextExpectedBy,
|
||||
DateTimeOffset? LastSuccessfulBackupAt,
|
||||
DateTimeOffset? LastIntegrityVerifiedAt,
|
||||
DateTimeOffset? LastVerifiedRestoreAt,
|
||||
string RestoreConfidence,
|
||||
long StoredBackupBytes,
|
||||
int ProtectedCount,
|
||||
int ExpectedIntervalHours,
|
||||
int ApplicationRetentionCount,
|
||||
int DailyRetentionExpectation,
|
||||
int WeeklyRetentionExpectation,
|
||||
int MonthlyRetentionExpectation,
|
||||
int RestoreVerificationMaxAgeDays);
|
||||
@@ -0,0 +1,48 @@
|
||||
namespace ItWorx.KnowledgeHub.Contracts.Platform;
|
||||
|
||||
public sealed record IngestionProposalPolicyContract(
|
||||
bool Enabled,
|
||||
int MaximumSourceCount,
|
||||
long MaximumSourceBytes,
|
||||
int FailedRetryLimit);
|
||||
|
||||
public sealed record BackupPolicyExpectationRequest(
|
||||
int ExpectedIntervalHours,
|
||||
string ScheduleDescription,
|
||||
int DailyRetentionCount,
|
||||
int WeeklyRetentionCount,
|
||||
int MonthlyRetentionCount,
|
||||
int ProtectedBackupCount,
|
||||
int RestoreVerificationMaxAgeDays);
|
||||
|
||||
public sealed record BackupPolicyExpectationResponse(
|
||||
string Enforcement,
|
||||
int ExpectedIntervalHours,
|
||||
string ScheduleDescription,
|
||||
int DailyRetentionCount,
|
||||
int WeeklyRetentionCount,
|
||||
int MonthlyRetentionCount,
|
||||
int ProtectedBackupCount,
|
||||
int RestoreVerificationMaxAgeDays);
|
||||
|
||||
public sealed record InventoryFreshnessPolicyContract(int WarningAfterHours);
|
||||
|
||||
public sealed record PlatformSettingsRequest(
|
||||
string DisplayName,
|
||||
string PrimaryLocale,
|
||||
string TimeZone,
|
||||
IngestionProposalPolicyContract IngestionProposals,
|
||||
BackupPolicyExpectationRequest BackupPolicyExpectation,
|
||||
InventoryFreshnessPolicyContract InventoryFreshnessPolicy);
|
||||
|
||||
public sealed record PlatformSettingsResponse(
|
||||
int SchemaVersion,
|
||||
string ETag,
|
||||
bool IsDefault,
|
||||
string DisplayName,
|
||||
string PrimaryLocale,
|
||||
string TimeZone,
|
||||
IngestionProposalPolicyContract IngestionProposals,
|
||||
BackupPolicyExpectationResponse BackupPolicyExpectation,
|
||||
InventoryFreshnessPolicyContract InventoryFreshnessPolicy,
|
||||
DateTimeOffset? UpdatedAt);
|
||||
@@ -0,0 +1,31 @@
|
||||
namespace ItWorx.KnowledgeHub.Contracts.Platform;
|
||||
|
||||
public sealed record UserPreferencesRequest(
|
||||
string Theme,
|
||||
string Density,
|
||||
bool PrimaryNavigationCollapsed,
|
||||
bool ContextPanelOpen,
|
||||
IReadOnlyList<string> NavigationOrder,
|
||||
IReadOnlyList<string> DashboardWidgetOrder,
|
||||
IReadOnlyList<string> HiddenDashboardWidgets,
|
||||
string DefaultLandingPage,
|
||||
string CatalogView,
|
||||
int ContextPanelWidth,
|
||||
int NavigationWidth);
|
||||
|
||||
public sealed record UserPreferencesResponse(
|
||||
int SchemaVersion,
|
||||
string ETag,
|
||||
bool IsDefault,
|
||||
string Theme,
|
||||
string Density,
|
||||
bool PrimaryNavigationCollapsed,
|
||||
bool ContextPanelOpen,
|
||||
IReadOnlyList<string> NavigationOrder,
|
||||
IReadOnlyList<string> DashboardWidgetOrder,
|
||||
IReadOnlyList<string> HiddenDashboardWidgets,
|
||||
string DefaultLandingPage,
|
||||
string CatalogView,
|
||||
int ContextPanelWidth,
|
||||
int NavigationWidth,
|
||||
DateTimeOffset? UpdatedAt);
|
||||
@@ -0,0 +1,10 @@
|
||||
namespace ItWorx.KnowledgeHub.Contracts.Sources;
|
||||
|
||||
public sealed record DataSourceResponse(Guid Id, string SourceKey, string DisplayName, string Kind, bool Enabled,
|
||||
string State, string ConfigurationState, string SyncAuthority, int ExpectedIntervalHours, DateTimeOffset? LastAttemptAt,
|
||||
DateTimeOffset? LastSuccessAt, DateTimeOffset? LastFailureAt, DateTimeOffset? DataObservedAt, long? RecordCount, string? LastErrorCode,
|
||||
string? LastCorrelationId, string ETag, IReadOnlyList<string> Actions);
|
||||
public sealed record DataSourceListResponse(IReadOnlyList<DataSourceResponse> Items, string? NextCursor);
|
||||
public sealed record RegisterDataSourceRequest(string SourceKey, string DisplayName, string Kind, int ExpectedIntervalHours);
|
||||
public sealed record ConfigureDataSourceRequest(string DisplayName, bool Enabled, int ExpectedIntervalHours);
|
||||
public sealed record ReportDataSourceRequest(bool Success, long? RecordCount, string? ErrorCode, string? CorrelationId);
|
||||
@@ -0,0 +1,27 @@
|
||||
namespace ItWorx.KnowledgeHub.Contracts.UnifiedSearch;
|
||||
|
||||
/// <summary>One permission-filtered, cursor-paginated page across the workspace search corpus.</summary>
|
||||
public sealed record UnifiedSearchResponse(
|
||||
IReadOnlyList<UnifiedSearchResultItem> Items,
|
||||
UnifiedSearchCounts Counts,
|
||||
string? NextCursor);
|
||||
|
||||
/// <summary>Counts over the complete matching visible population, before pagination.</summary>
|
||||
public sealed record UnifiedSearchCounts(
|
||||
int Knowledge,
|
||||
int Projects,
|
||||
int Services,
|
||||
int Runbooks,
|
||||
int Decisions)
|
||||
{
|
||||
public int Total => Knowledge + Projects + Services + Runbooks + Decisions;
|
||||
}
|
||||
|
||||
public sealed record UnifiedSearchResultItem(
|
||||
string Kind,
|
||||
Guid Id,
|
||||
string Title,
|
||||
string Summary,
|
||||
string Snippet,
|
||||
DateTimeOffset UpdatedAt,
|
||||
string Href);
|
||||
@@ -150,9 +150,10 @@ public sealed class InfrastructureService
|
||||
if (Presence != InventoryPresence.Archived)
|
||||
{
|
||||
Presence = InventoryPresence.Missing;
|
||||
Health = HealthState.Unknown;
|
||||
RuntimeStatus = "unknown";
|
||||
}
|
||||
|
||||
LastSeenAt = observedAt;
|
||||
// Keep the time it was last actually present; absence is captured by the snapshot.
|
||||
}
|
||||
|
||||
public void Archive() => Presence = InventoryPresence.Archived;
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
using System.Text.Json;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Domain.Jobs;
|
||||
|
||||
/// <summary>A durable, workspace-scoped unit of operator work. Payload and result JSON
|
||||
/// contain only bounded, non-secret product data; persisted failures are sanitized
|
||||
/// summaries rather than exception messages.</summary>
|
||||
public sealed class BackgroundJob
|
||||
{
|
||||
public const int MaxStoredErrors = 20;
|
||||
public const int MaxErrorMessageLength = 280;
|
||||
|
||||
private BackgroundJob() { }
|
||||
|
||||
public Guid Id { get; private set; }
|
||||
public Guid WorkspaceId { get; private set; }
|
||||
public Guid ActorId { get; private set; }
|
||||
public BackgroundJobType Type { get; private set; }
|
||||
public BackgroundJobStatus Status { get; private set; }
|
||||
public string PayloadJson { get; private set; } = "{}";
|
||||
public string? ResultJson { get; private set; }
|
||||
public string ErrorsJson { get; private set; } = "[]";
|
||||
public string IdempotencyKeyHash { get; private set; } = string.Empty;
|
||||
public string PayloadHash { get; private set; } = string.Empty;
|
||||
public string? CorrelationId { get; private set; }
|
||||
public DateTimeOffset CreatedAt { get; private set; }
|
||||
public DateTimeOffset? StartedAt { get; private set; }
|
||||
public DateTimeOffset? CompletedAt { get; private set; }
|
||||
public DateTimeOffset NextAttemptAt { get; private set; }
|
||||
public DateTimeOffset? CancelRequestedAt { get; private set; }
|
||||
public DateTimeOffset? LeaseUntil { get; private set; }
|
||||
public Guid? LeaseToken { get; private set; }
|
||||
public int AttemptCount { get; private set; }
|
||||
public int Total { get; private set; }
|
||||
public int Processed { get; private set; }
|
||||
public int Succeeded { get; private set; }
|
||||
public int Skipped { get; private set; }
|
||||
public int Failed { get; private set; }
|
||||
public long Version { get; private set; }
|
||||
|
||||
public static BackgroundJob Create(
|
||||
Guid id,
|
||||
Guid workspaceId,
|
||||
Guid actorId,
|
||||
BackgroundJobType type,
|
||||
string payloadJson,
|
||||
string idempotencyKeyHash,
|
||||
string payloadHash,
|
||||
string? correlationId,
|
||||
DateTimeOffset now,
|
||||
int total)
|
||||
{
|
||||
if (id == Guid.Empty || workspaceId == Guid.Empty || actorId == Guid.Empty)
|
||||
throw new ArgumentException("Job identity fields must be non-empty.");
|
||||
ArgumentOutOfRangeException.ThrowIfNegative(total);
|
||||
if (payloadJson.Length > 2048 || idempotencyKeyHash.Length != 64 || payloadHash.Length != 64)
|
||||
throw new ArgumentException("Job payload and idempotency values must be bounded hashes.");
|
||||
|
||||
return new BackgroundJob
|
||||
{
|
||||
Id = id,
|
||||
WorkspaceId = workspaceId,
|
||||
ActorId = actorId,
|
||||
Type = type,
|
||||
Status = BackgroundJobStatus.Queued,
|
||||
PayloadJson = payloadJson,
|
||||
IdempotencyKeyHash = idempotencyKeyHash,
|
||||
PayloadHash = payloadHash,
|
||||
CorrelationId = Limit(correlationId, 128),
|
||||
CreatedAt = now,
|
||||
NextAttemptAt = now,
|
||||
Total = total,
|
||||
Version = 1
|
||||
};
|
||||
}
|
||||
|
||||
public bool IsTerminal => Status is BackgroundJobStatus.Succeeded
|
||||
or BackgroundJobStatus.PartiallySucceeded or BackgroundJobStatus.Failed or BackgroundJobStatus.Cancelled;
|
||||
|
||||
public int Progress => Total == 0 ? (IsTerminal ? 100 : 0) : Math.Min(100, Processed * 100 / Total);
|
||||
|
||||
public void Start(Guid leaseToken, DateTimeOffset now, DateTimeOffset leaseUntil)
|
||||
{
|
||||
if (IsTerminal)
|
||||
return;
|
||||
Status = BackgroundJobStatus.Running;
|
||||
StartedAt ??= now;
|
||||
LeaseToken = leaseToken;
|
||||
LeaseUntil = leaseUntil;
|
||||
AttemptCount++;
|
||||
Version++;
|
||||
}
|
||||
|
||||
public void RenewLease(Guid leaseToken, DateTimeOffset leaseUntil)
|
||||
{
|
||||
if (Status == BackgroundJobStatus.Running && LeaseToken == leaseToken)
|
||||
LeaseUntil = leaseUntil;
|
||||
}
|
||||
|
||||
public void RequestCancellation(DateTimeOffset now)
|
||||
{
|
||||
if (IsTerminal)
|
||||
return;
|
||||
CancelRequestedAt ??= now;
|
||||
if (Status == BackgroundJobStatus.Queued)
|
||||
{
|
||||
Status = BackgroundJobStatus.Cancelled;
|
||||
CompletedAt = now;
|
||||
}
|
||||
Version++;
|
||||
}
|
||||
|
||||
public void RecordOutcome(BackgroundJobItemState state, string? code = null, string? message = null, Guid? itemId = null)
|
||||
{
|
||||
if (state is not (BackgroundJobItemState.Succeeded or BackgroundJobItemState.Skipped or BackgroundJobItemState.Failed))
|
||||
throw new ArgumentOutOfRangeException(nameof(state));
|
||||
if (Processed >= Total)
|
||||
throw new InvalidOperationException("A job cannot process more work items than its persisted total.");
|
||||
|
||||
Processed++;
|
||||
switch (state)
|
||||
{
|
||||
case BackgroundJobItemState.Succeeded: Succeeded++; break;
|
||||
case BackgroundJobItemState.Skipped: Skipped++; break;
|
||||
case BackgroundJobItemState.Failed:
|
||||
Failed++;
|
||||
if (code is not null && message is not null)
|
||||
AddError(code, message, itemId);
|
||||
break;
|
||||
}
|
||||
Version++;
|
||||
}
|
||||
|
||||
public void Touch() => Version++;
|
||||
|
||||
public void Complete(DateTimeOffset now, string? resultJson = null)
|
||||
{
|
||||
if (IsTerminal)
|
||||
return;
|
||||
ResultJson = resultJson is { Length: <= 4096 } ? resultJson : null;
|
||||
Status = Failed == 0 ? BackgroundJobStatus.Succeeded
|
||||
: Succeeded == 0 && Skipped == 0 ? BackgroundJobStatus.Failed
|
||||
: BackgroundJobStatus.PartiallySucceeded;
|
||||
CompletedAt = now;
|
||||
LeaseToken = null;
|
||||
LeaseUntil = null;
|
||||
Version++;
|
||||
}
|
||||
|
||||
public void Cancel(DateTimeOffset now)
|
||||
{
|
||||
if (IsTerminal)
|
||||
return;
|
||||
Status = BackgroundJobStatus.Cancelled;
|
||||
CompletedAt = now;
|
||||
LeaseToken = null;
|
||||
LeaseUntil = null;
|
||||
Version++;
|
||||
}
|
||||
|
||||
public void ScheduleRetry(DateTimeOffset now, DateTimeOffset retryAt, string code, string message)
|
||||
{
|
||||
if (IsTerminal)
|
||||
return;
|
||||
AddError(code, message, null);
|
||||
LeaseToken = null;
|
||||
LeaseUntil = null;
|
||||
if (AttemptCount <= 3 && CancelRequestedAt is null)
|
||||
{
|
||||
Status = BackgroundJobStatus.Queued;
|
||||
NextAttemptAt = retryAt;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (Processed < Total)
|
||||
Processed++;
|
||||
Failed++;
|
||||
Status = Succeeded == 0 && Skipped == 0 ? BackgroundJobStatus.Failed : BackgroundJobStatus.PartiallySucceeded;
|
||||
CompletedAt = now;
|
||||
}
|
||||
Version++;
|
||||
}
|
||||
|
||||
public void Fail(DateTimeOffset now, string code, string safeMessage)
|
||||
{
|
||||
if (IsTerminal)
|
||||
return;
|
||||
AddError(code, safeMessage, null);
|
||||
Failed++;
|
||||
Status = Succeeded == 0 && Skipped == 0 ? BackgroundJobStatus.Failed : BackgroundJobStatus.PartiallySucceeded;
|
||||
CompletedAt = now;
|
||||
LeaseToken = null;
|
||||
LeaseUntil = null;
|
||||
Version++;
|
||||
}
|
||||
|
||||
public void SetResult(string resultJson)
|
||||
{
|
||||
ResultJson = resultJson.Length <= 4096 ? resultJson : null;
|
||||
Version++;
|
||||
}
|
||||
|
||||
public void SetTotal(int total)
|
||||
{
|
||||
if (Status != BackgroundJobStatus.Queued || Processed != 0 || total < 0)
|
||||
throw new InvalidOperationException("The total can only be assigned before a queued job starts.");
|
||||
Total = total;
|
||||
Version++;
|
||||
}
|
||||
|
||||
private void AddError(string code, string message, Guid? itemId)
|
||||
{
|
||||
try
|
||||
{
|
||||
var errors = JsonSerializer.Deserialize<List<BackgroundJobError>>(ErrorsJson) ?? [];
|
||||
if (errors.Count >= MaxStoredErrors)
|
||||
return;
|
||||
errors.Add(new BackgroundJobError(
|
||||
Limit(code, 64) ?? "job_error",
|
||||
Limit(message, MaxErrorMessageLength) ?? "De taak is mislukt.",
|
||||
itemId));
|
||||
ErrorsJson = JsonSerializer.Serialize(errors);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
ErrorsJson = "[]";
|
||||
}
|
||||
}
|
||||
|
||||
private static string? Limit(string? value, int length) =>
|
||||
string.IsNullOrEmpty(value) ? value : value.Length <= length ? value : value[..length];
|
||||
}
|
||||
|
||||
public sealed record BackgroundJobError(string Code, string Message, Guid? ItemId);
|
||||
@@ -0,0 +1,34 @@
|
||||
namespace ItWorx.KnowledgeHub.Domain.Jobs;
|
||||
|
||||
/// <summary>One row in the database-side snapshot of a course-tagging job. The stable
|
||||
/// ordinal lets workers resume after a process restart without loading a subtree into
|
||||
/// application memory.</summary>
|
||||
public sealed class BackgroundJobItem
|
||||
{
|
||||
private BackgroundJobItem() { }
|
||||
|
||||
public long Ordinal { get; private set; }
|
||||
public Guid JobId { get; private set; }
|
||||
public Guid PageId { get; private set; }
|
||||
public bool InitiallyTagged { get; private set; }
|
||||
public BackgroundJobItemState State { get; private set; }
|
||||
public DateTimeOffset? StartedAt { get; private set; }
|
||||
public DateTimeOffset? CompletedAt { get; private set; }
|
||||
|
||||
public void MarkProcessing(DateTimeOffset now)
|
||||
{
|
||||
if (State is BackgroundJobItemState.Pending or BackgroundJobItemState.Processing)
|
||||
{
|
||||
State = BackgroundJobItemState.Processing;
|
||||
StartedAt ??= now;
|
||||
}
|
||||
}
|
||||
|
||||
public void Complete(BackgroundJobItemState state, DateTimeOffset now)
|
||||
{
|
||||
if (state is not (BackgroundJobItemState.Succeeded or BackgroundJobItemState.Skipped or BackgroundJobItemState.Failed))
|
||||
throw new ArgumentOutOfRangeException(nameof(state));
|
||||
State = state;
|
||||
CompletedAt = now;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
namespace ItWorx.KnowledgeHub.Domain.Jobs;
|
||||
|
||||
public enum BackgroundJobItemState
|
||||
{
|
||||
Pending,
|
||||
Processing,
|
||||
Succeeded,
|
||||
Skipped,
|
||||
Failed
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
namespace ItWorx.KnowledgeHub.Domain.Jobs;
|
||||
|
||||
public enum BackgroundJobStatus
|
||||
{
|
||||
Queued,
|
||||
Running,
|
||||
Succeeded,
|
||||
PartiallySucceeded,
|
||||
Failed,
|
||||
Cancelled
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
namespace ItWorx.KnowledgeHub.Domain.Jobs;
|
||||
|
||||
public enum BackgroundJobType
|
||||
{
|
||||
CourseTagging,
|
||||
Backup
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
namespace ItWorx.KnowledgeHub.Domain.Platform;
|
||||
|
||||
public sealed record IngestionProposalPolicyValues(
|
||||
bool Enabled,
|
||||
int MaximumSourceCount,
|
||||
long MaximumSourceBytes,
|
||||
int FailedRetryLimit);
|
||||
|
||||
/// <summary>
|
||||
/// Expectations about operator-managed backup jobs. The application records and reports
|
||||
/// these values; it does not schedule or prune external backup jobs.
|
||||
/// </summary>
|
||||
public sealed record BackupPolicyExpectationValues(
|
||||
int ExpectedIntervalHours,
|
||||
string ScheduleDescription,
|
||||
int DailyRetentionCount,
|
||||
int WeeklyRetentionCount,
|
||||
int MonthlyRetentionCount,
|
||||
int ProtectedBackupCount,
|
||||
int RestoreVerificationMaxAgeDays);
|
||||
|
||||
public sealed record InventoryFreshnessPolicyValues(int WarningAfterHours);
|
||||
|
||||
public sealed record PlatformSettingsValues(
|
||||
string DisplayName,
|
||||
string PrimaryLocale,
|
||||
string TimeZone,
|
||||
IngestionProposalPolicyValues IngestionProposals,
|
||||
BackupPolicyExpectationValues BackupPolicyExpectation,
|
||||
InventoryFreshnessPolicyValues InventoryFreshnessPolicy);
|
||||
|
||||
/// <summary>Versioned, installation-wide operator policy.</summary>
|
||||
public sealed class PlatformSettings
|
||||
{
|
||||
public static readonly Guid SingletonId = new("00000000-0000-0000-0000-000000000002");
|
||||
|
||||
private PlatformSettings() { }
|
||||
|
||||
private PlatformSettings(PlatformSettingsValues values, DateTimeOffset now)
|
||||
{
|
||||
Id = SingletonId;
|
||||
Apply(values);
|
||||
Version = 1;
|
||||
UpdatedAt = now;
|
||||
}
|
||||
|
||||
public Guid Id { get; private set; }
|
||||
|
||||
public string DisplayName { get; private set; } = "ItWorx Knowledge Hub";
|
||||
|
||||
public string PrimaryLocale { get; private set; } = "nl-BE";
|
||||
|
||||
public string TimeZone { get; private set; } = "Europe/Brussels";
|
||||
|
||||
public bool IngestionEnabled { get; private set; }
|
||||
|
||||
public int MaximumSourceCount { get; private set; } = 20;
|
||||
|
||||
public long MaximumSourceBytes { get; private set; } = 104_857_600;
|
||||
|
||||
public int FailedRetryLimit { get; private set; } = 3;
|
||||
|
||||
public int BackupExpectedIntervalHours { get; private set; } = 24;
|
||||
|
||||
public string BackupScheduleDescription { get; private set; } = "Extern beheerd";
|
||||
|
||||
public int BackupDailyRetentionCount { get; private set; } = 7;
|
||||
|
||||
public int BackupWeeklyRetentionCount { get; private set; } = 4;
|
||||
|
||||
public int BackupMonthlyRetentionCount { get; private set; } = 6;
|
||||
|
||||
public int ProtectedBackupCount { get; private set; } = 1;
|
||||
|
||||
public int RestoreVerificationMaxAgeDays { get; private set; } = 90;
|
||||
|
||||
public int InventoryWarningAfterHours { get; private set; } = 24;
|
||||
|
||||
/// <summary>Monotonic strong ETag for this row.</summary>
|
||||
public int Version { get; private set; }
|
||||
|
||||
public DateTimeOffset UpdatedAt { get; private set; }
|
||||
|
||||
public static PlatformSettings Create(PlatformSettingsValues values, DateTimeOffset now)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(values);
|
||||
return new PlatformSettings(values, now);
|
||||
}
|
||||
|
||||
public PlatformSettingsValues ToValues() => new(
|
||||
DisplayName,
|
||||
PrimaryLocale,
|
||||
TimeZone,
|
||||
new IngestionProposalPolicyValues(IngestionEnabled, MaximumSourceCount, MaximumSourceBytes, FailedRetryLimit),
|
||||
new BackupPolicyExpectationValues(
|
||||
BackupExpectedIntervalHours, BackupScheduleDescription, BackupDailyRetentionCount, BackupWeeklyRetentionCount,
|
||||
BackupMonthlyRetentionCount, ProtectedBackupCount, RestoreVerificationMaxAgeDays),
|
||||
new InventoryFreshnessPolicyValues(InventoryWarningAfterHours));
|
||||
|
||||
public void Replace(PlatformSettingsValues values, DateTimeOffset now)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(values);
|
||||
Apply(values);
|
||||
Version = checked(Version + 1);
|
||||
UpdatedAt = now;
|
||||
}
|
||||
|
||||
private void Apply(PlatformSettingsValues values)
|
||||
{
|
||||
DisplayName = values.DisplayName;
|
||||
PrimaryLocale = values.PrimaryLocale;
|
||||
TimeZone = values.TimeZone;
|
||||
IngestionEnabled = values.IngestionProposals.Enabled;
|
||||
MaximumSourceCount = values.IngestionProposals.MaximumSourceCount;
|
||||
MaximumSourceBytes = values.IngestionProposals.MaximumSourceBytes;
|
||||
FailedRetryLimit = values.IngestionProposals.FailedRetryLimit;
|
||||
BackupExpectedIntervalHours = values.BackupPolicyExpectation.ExpectedIntervalHours;
|
||||
BackupScheduleDescription = values.BackupPolicyExpectation.ScheduleDescription;
|
||||
BackupDailyRetentionCount = values.BackupPolicyExpectation.DailyRetentionCount;
|
||||
BackupWeeklyRetentionCount = values.BackupPolicyExpectation.WeeklyRetentionCount;
|
||||
BackupMonthlyRetentionCount = values.BackupPolicyExpectation.MonthlyRetentionCount;
|
||||
ProtectedBackupCount = values.BackupPolicyExpectation.ProtectedBackupCount;
|
||||
RestoreVerificationMaxAgeDays = values.BackupPolicyExpectation.RestoreVerificationMaxAgeDays;
|
||||
InventoryWarningAfterHours = values.InventoryFreshnessPolicy.WarningAfterHours;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
namespace ItWorx.KnowledgeHub.Domain.Platform;
|
||||
|
||||
public sealed record UserPreferencesValues(
|
||||
string Theme,
|
||||
string Density,
|
||||
bool PrimaryNavigationCollapsed,
|
||||
bool ContextPanelOpen,
|
||||
IReadOnlyList<string> NavigationOrder,
|
||||
IReadOnlyList<string> DashboardWidgetOrder,
|
||||
IReadOnlyList<string> HiddenDashboardWidgets,
|
||||
string DefaultLandingPage,
|
||||
string CatalogView,
|
||||
int ContextPanelWidth,
|
||||
int NavigationWidth);
|
||||
|
||||
/// <summary>Workspace-scoped presentation preferences for one interactive user.</summary>
|
||||
public sealed class UserPreferences
|
||||
{
|
||||
private UserPreferences() { }
|
||||
|
||||
private UserPreferences(Guid workspaceId, Guid userId, UserPreferencesValues values, DateTimeOffset now)
|
||||
{
|
||||
WorkspaceId = workspaceId;
|
||||
UserId = userId;
|
||||
Apply(values);
|
||||
Version = 1;
|
||||
UpdatedAt = now;
|
||||
}
|
||||
|
||||
public Guid WorkspaceId { get; private set; }
|
||||
|
||||
public Guid UserId { get; private set; }
|
||||
|
||||
public string Theme { get; private set; } = "system";
|
||||
|
||||
public string Density { get; private set; } = "comfortable";
|
||||
|
||||
public bool PrimaryNavigationCollapsed { get; private set; }
|
||||
|
||||
public bool ContextPanelOpen { get; private set; }
|
||||
|
||||
public string[] NavigationOrder { get; private set; } = [];
|
||||
|
||||
public string[] DashboardWidgetOrder { get; private set; } = [];
|
||||
|
||||
public string[] HiddenDashboardWidgets { get; private set; } = [];
|
||||
|
||||
public string DefaultLandingPage { get; private set; } = "/";
|
||||
|
||||
public string CatalogView { get; private set; } = "list";
|
||||
|
||||
public int ContextPanelWidth { get; private set; } = 304;
|
||||
|
||||
public int NavigationWidth { get; private set; } = 224;
|
||||
|
||||
/// <summary>Monotonic strong ETag for this preference row.</summary>
|
||||
public int Version { get; private set; }
|
||||
|
||||
public DateTimeOffset UpdatedAt { get; private set; }
|
||||
|
||||
public static UserPreferences Create(Guid workspaceId, Guid userId, UserPreferencesValues values, DateTimeOffset now)
|
||||
{
|
||||
if (workspaceId == Guid.Empty || userId == Guid.Empty)
|
||||
{
|
||||
throw new ArgumentException("Workspace and user IDs are required.");
|
||||
}
|
||||
|
||||
ArgumentNullException.ThrowIfNull(values);
|
||||
return new UserPreferences(workspaceId, userId, values, now);
|
||||
}
|
||||
|
||||
public UserPreferencesValues ToValues() => new(
|
||||
Theme,
|
||||
Density,
|
||||
PrimaryNavigationCollapsed,
|
||||
ContextPanelOpen,
|
||||
NavigationOrder.ToArray(),
|
||||
DashboardWidgetOrder.ToArray(),
|
||||
HiddenDashboardWidgets.ToArray(),
|
||||
DefaultLandingPage,
|
||||
CatalogView,
|
||||
ContextPanelWidth,
|
||||
NavigationWidth);
|
||||
|
||||
public void Replace(UserPreferencesValues values, DateTimeOffset now)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(values);
|
||||
Apply(values);
|
||||
Version = checked(Version + 1);
|
||||
UpdatedAt = now;
|
||||
}
|
||||
|
||||
private void Apply(UserPreferencesValues values)
|
||||
{
|
||||
Theme = values.Theme;
|
||||
Density = values.Density;
|
||||
PrimaryNavigationCollapsed = values.PrimaryNavigationCollapsed;
|
||||
ContextPanelOpen = values.ContextPanelOpen;
|
||||
NavigationOrder = values.NavigationOrder.ToArray();
|
||||
DashboardWidgetOrder = values.DashboardWidgetOrder.ToArray();
|
||||
HiddenDashboardWidgets = values.HiddenDashboardWidgets.ToArray();
|
||||
DefaultLandingPage = values.DefaultLandingPage;
|
||||
CatalogView = values.CatalogView;
|
||||
ContextPanelWidth = values.ContextPanelWidth;
|
||||
NavigationWidth = values.NavigationWidth;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Domain.Sources;
|
||||
|
||||
public enum DataSourceKind { Inventory, Gitea, KnowledgeClient, ManualIntake, LeafWiki, CuratedPackage }
|
||||
|
||||
/// <summary>Operator registration and observed evidence; credentials never enter this aggregate.</summary>
|
||||
public sealed class DataSource
|
||||
{
|
||||
private DataSource() { }
|
||||
public Guid Id { get; private set; }
|
||||
public Guid WorkspaceId { get; private set; }
|
||||
public string SourceKey { get; private set; } = "";
|
||||
public string DisplayName { get; private set; } = "";
|
||||
public DataSourceKind Kind { get; private set; }
|
||||
public bool Enabled { get; private set; } = true;
|
||||
public int ExpectedIntervalHours { get; private set; }
|
||||
public int Version { get; private set; } = 1;
|
||||
public DateTimeOffset CreatedAt { get; private set; }
|
||||
public DateTimeOffset? LastAttemptAt { get; private set; }
|
||||
public DateTimeOffset? LastSuccessAt { get; private set; }
|
||||
public DateTimeOffset? LastFailureAt { get; private set; }
|
||||
public DateTimeOffset? DataObservedAt { get; private set; }
|
||||
public long? RecordCount { get; private set; }
|
||||
public string? LastErrorCode { get; private set; }
|
||||
public string? LastCorrelationId { get; private set; }
|
||||
|
||||
public static DataSource Create(Guid workspaceId, string sourceKey, string displayName, DataSourceKind kind, int expectedIntervalHours, DateTimeOffset now)
|
||||
{
|
||||
if (workspaceId == Guid.Empty || !Enum.IsDefined(kind) || !Regex.IsMatch(sourceKey, "^[a-zA-Z0-9][a-zA-Z0-9_.:-]{0,199}$", RegexOptions.CultureInvariant))
|
||||
{
|
||||
throw new ArgumentException("Invalid source identity.");
|
||||
}
|
||||
var source = new DataSource { Id = Guid.NewGuid(), WorkspaceId = workspaceId, SourceKey = sourceKey, Kind = kind, CreatedAt = now, Version = 0 };
|
||||
source.Configure(displayName, true, expectedIntervalHours);
|
||||
return source;
|
||||
}
|
||||
|
||||
public void Configure(string displayName, bool enabled, int expectedIntervalHours)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(displayName) || displayName.Length > 120 || displayName.Any(char.IsControl) || expectedIntervalHours is < 1 or > 8760)
|
||||
{
|
||||
throw new ArgumentException("Invalid source policy.");
|
||||
}
|
||||
DisplayName = displayName.Trim();
|
||||
Enabled = enabled;
|
||||
ExpectedIntervalHours = expectedIntervalHours;
|
||||
Version++;
|
||||
}
|
||||
|
||||
public void Observe(bool success, long? recordCount, string? errorCode, string? correlationId, DateTimeOffset observedAt, DateTimeOffset? dataObservedAt = null)
|
||||
{
|
||||
if (recordCount < 0 || (!success && errorCode is not ("authentication" or "connectivity" or "validation" or "unexpected"))
|
||||
|| (LastAttemptAt is { } previous && observedAt < previous)
|
||||
|| dataObservedAt > observedAt || correlationId?.Length > 100 || correlationId?.Any(char.IsControl) == true)
|
||||
{
|
||||
throw new ArgumentException("Invalid source observation.");
|
||||
}
|
||||
LastAttemptAt = observedAt;
|
||||
LastCorrelationId = correlationId;
|
||||
if (success)
|
||||
{
|
||||
LastSuccessAt = observedAt;
|
||||
var captured = dataObservedAt ?? observedAt;
|
||||
if (DataObservedAt is null || captured >= DataObservedAt)
|
||||
{
|
||||
DataObservedAt = captured;
|
||||
RecordCount = recordCount;
|
||||
}
|
||||
LastErrorCode = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
LastFailureAt = observedAt;
|
||||
LastErrorCode = errorCode;
|
||||
}
|
||||
Version++;
|
||||
}
|
||||
|
||||
public string GetState(DateTimeOffset now)
|
||||
{
|
||||
if (!Enabled) return "Disabled";
|
||||
if (LastErrorCode is not null) return "Failed";
|
||||
if (DataObservedAt is null || DataObservedAt > now) return "Unknown";
|
||||
return now - DataObservedAt.Value > TimeSpan.FromHours(ExpectedIntervalHours) ? "Stale" : "Current";
|
||||
}
|
||||
}
|
||||
@@ -4,8 +4,33 @@ using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Infrastructure.Audit;
|
||||
|
||||
public sealed class AuditEventReader(KnowledgeHubDbContext db) : IAuditEventReader
|
||||
public sealed class AuditEventReader(KnowledgeHubDbContext db) : IAuditEventReader, IActivityEventReader
|
||||
{
|
||||
private static readonly string[] OperationalTargets =
|
||||
["page", "project", "decision", "runbook", "milestone", "project-update", "inventory-snapshot", "infrastructure-service"];
|
||||
|
||||
public async Task<OperationalActivityPage> ListPageAsync(int limit, long? before, CancellationToken cancellationToken)
|
||||
{
|
||||
var size = Math.Clamp(limit, 1, 200);
|
||||
var query = db.AuditEvents.AsNoTracking();
|
||||
if (before is not null) query = query.Where(item => item.Id < before);
|
||||
var items = await query.OrderByDescending(item => item.Id).Take(size + 1)
|
||||
.Select(item => new AuditEventView(item.Id, item.ActorKind, item.ActorId, item.Source, item.Action,
|
||||
item.TargetType, item.TargetId, item.Summary, item.OccurredAt, item.CorrelationId)).ToListAsync(cancellationToken);
|
||||
return new OperationalActivityPage(items.Take(size).ToArray(), items.Count > size ? items[size - 1].Id : null);
|
||||
}
|
||||
|
||||
public async Task<OperationalActivityPage> ListAsync(int limit, long? before, CancellationToken cancellationToken)
|
||||
{
|
||||
var size = Math.Clamp(limit, 1, 200);
|
||||
var query = db.AuditEvents.AsNoTracking().Where(item => OperationalTargets.Contains(item.TargetType));
|
||||
if (before is not null) query = query.Where(item => item.Id < before);
|
||||
var items = await query.OrderByDescending(item => item.Id).Take(size + 1)
|
||||
.Select(item => new AuditEventView(item.Id, item.ActorKind, item.ActorId, item.Source, item.Action,
|
||||
item.TargetType, item.TargetId, item.Summary, item.OccurredAt)).ToListAsync(cancellationToken);
|
||||
return new OperationalActivityPage(items.Take(size).ToArray(), items.Count > size ? items[size - 1].Id : null);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<AuditEventView>> ListRecentAsync(int limit, CancellationToken cancellationToken) =>
|
||||
await db.AuditEvents
|
||||
.AsNoTracking()
|
||||
|
||||
+18
-1
@@ -3,6 +3,7 @@ using System.Text;
|
||||
using ItWorx.KnowledgeHub.Application.Common;
|
||||
using ItWorx.KnowledgeHub.Application.Infrastructure;
|
||||
using ItWorx.KnowledgeHub.Domain.Infrastructure;
|
||||
using ItWorx.KnowledgeHub.Domain.Sources;
|
||||
using ItWorx.KnowledgeHub.Infrastructure.Persistence;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Npgsql;
|
||||
@@ -399,6 +400,19 @@ public sealed class InfrastructureRepository(KnowledgeHubDbContext db) : IInfras
|
||||
Guid workspaceId, InventorySnapshotInput input, DateTimeOffset importedAt, CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedHash = input.SourceHash.Trim().ToLowerInvariant();
|
||||
await using var transaction = await db.Database.BeginTransactionAsync(cancellationToken);
|
||||
// Serialize imports for one workspace before checking idempotency or age.
|
||||
// Concurrent snapshots must not resurrect older container facts.
|
||||
var lockKey = "knowledge-hub:inventory:" + workspaceId.ToString("N");
|
||||
await db.Database.ExecuteSqlInterpolatedAsync($"SELECT pg_advisory_xact_lock(hashtext({lockKey}))", cancellationToken);
|
||||
// Lock the current registry row in the same transaction as the inventory
|
||||
// mutation. An operator disable that commits first is observed here; one
|
||||
// arriving later waits until this snapshot commits.
|
||||
var sourceRows = await db.Set<DataSource>()
|
||||
.FromSqlInterpolated($"SELECT * FROM data_sources WHERE workspace_id = {workspaceId} AND source_key = 'inventory:default' FOR SHARE")
|
||||
.AsNoTracking().ToListAsync(cancellationToken);
|
||||
if (sourceRows.Any(source => !source.Enabled || source.Kind != DataSourceKind.Inventory))
|
||||
throw new InactiveInventorySourceException();
|
||||
var existingSnapshot = await db.InventorySnapshots.AsNoTracking()
|
||||
.FirstOrDefaultAsync(
|
||||
snapshot => snapshot.WorkspaceId == workspaceId && snapshot.SourceHash == normalizedHash,
|
||||
@@ -412,7 +426,10 @@ public sealed class InfrastructureRepository(KnowledgeHubDbContext db) : IInfras
|
||||
await db.ManualOverrides.CountAsync(manualOverride => manualOverride.WorkspaceId == workspaceId, cancellationToken));
|
||||
}
|
||||
|
||||
await using var transaction = await db.Database.BeginTransactionAsync(cancellationToken);
|
||||
var latestCapture = await db.InventorySnapshots.AsNoTracking()
|
||||
.Where(item => item.WorkspaceId == workspaceId)
|
||||
.MaxAsync(item => (DateTimeOffset?)item.CapturedAt, cancellationToken);
|
||||
if (latestCapture > input.CapturedAt) throw new StaleInventorySnapshotException();
|
||||
var snapshot = InventorySnapshot.Create(
|
||||
workspaceId, input.SourceKey, normalizedHash, input.CapturedAt, importedAt, input.Hosts.Count, input.Services.Count);
|
||||
db.InventorySnapshots.Add(snapshot);
|
||||
|
||||
+29
-3
@@ -6,12 +6,14 @@ using ItWorx.KnowledgeHub.Application.Decisions;
|
||||
using ItWorx.KnowledgeHub.Application.Identity;
|
||||
using ItWorx.KnowledgeHub.Application.Infrastructure;
|
||||
using ItWorx.KnowledgeHub.Application.Ingestion;
|
||||
using ItWorx.KnowledgeHub.Application.Jobs;
|
||||
using ItWorx.KnowledgeHub.Application.Knowledge;
|
||||
using ItWorx.KnowledgeHub.Application.Knowledge.UseCases;
|
||||
using ItWorx.KnowledgeHub.Application.Migration;
|
||||
using ItWorx.KnowledgeHub.Application.Operations;
|
||||
using ItWorx.KnowledgeHub.Application.Projects;
|
||||
using ItWorx.KnowledgeHub.Application.Runbooks;
|
||||
using ItWorx.KnowledgeHub.Application.UnifiedSearch;
|
||||
using ItWorx.KnowledgeHub.Infrastructure.Attachments;
|
||||
using ItWorx.KnowledgeHub.Infrastructure.Audit;
|
||||
using ItWorx.KnowledgeHub.Infrastructure.Automation;
|
||||
@@ -19,12 +21,14 @@ using ItWorx.KnowledgeHub.Infrastructure.Decisions;
|
||||
using ItWorx.KnowledgeHub.Infrastructure.Identity;
|
||||
using ItWorx.KnowledgeHub.Infrastructure.InfrastructureKnowledge;
|
||||
using ItWorx.KnowledgeHub.Infrastructure.Ingestion;
|
||||
using ItWorx.KnowledgeHub.Infrastructure.Jobs;
|
||||
using ItWorx.KnowledgeHub.Infrastructure.Knowledge;
|
||||
using ItWorx.KnowledgeHub.Infrastructure.LeafWiki;
|
||||
using ItWorx.KnowledgeHub.Infrastructure.Operations;
|
||||
using ItWorx.KnowledgeHub.Infrastructure.Persistence;
|
||||
using ItWorx.KnowledgeHub.Infrastructure.Projects;
|
||||
using ItWorx.KnowledgeHub.Infrastructure.Runbooks;
|
||||
using ItWorx.KnowledgeHub.Infrastructure.UnifiedSearch;
|
||||
using Microsoft.AspNetCore.Identity;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
@@ -53,10 +57,15 @@ public static class InfrastructureServiceCollectionExtensions
|
||||
services.AddSingleton(provider =>
|
||||
new DatabaseConnectionOptions(DatabaseConnectionStringFactory.Build(provider.GetRequiredService<IConfiguration>())));
|
||||
|
||||
services.TryAddSingleton<PlatformMetrics>();
|
||||
services.AddScoped<IOperationalTelemetryReader, OperationalTelemetryReader>();
|
||||
services.AddScoped<IBackgroundJobRepository, BackgroundJobRepository>();
|
||||
services.AddSingleton<DatabaseMetricsInterceptor>();
|
||||
services.AddDbContext<KnowledgeHubDbContext>((provider, options) => options
|
||||
.UseNpgsql(provider.GetRequiredService<DatabaseConnectionOptions>().ConnectionString, npgsql => npgsql.MigrationsAssembly(
|
||||
typeof(KnowledgeHubDbContext).Assembly.GetName().Name))
|
||||
.UseSnakeCaseNamingConvention());
|
||||
.UseSnakeCaseNamingConvention()
|
||||
.AddInterceptors(provider.GetRequiredService<DatabaseMetricsInterceptor>()));
|
||||
|
||||
services.AddHttpContextAccessor();
|
||||
|
||||
@@ -103,6 +112,7 @@ public static class InfrastructureServiceCollectionExtensions
|
||||
services.AddScoped<IRecoveryTokenService, RecoveryTokenService>();
|
||||
services.AddScoped<IAuditEventRecorder, AuditEventRecorder>();
|
||||
services.AddScoped<IAuditEventReader, AuditEventReader>();
|
||||
services.AddScoped<IActivityEventReader, AuditEventReader>();
|
||||
|
||||
services.AddScoped<IPageRepository, PageRepository>();
|
||||
services.AddScoped<IIngestionProposalRepository, IngestionProposalRepository>();
|
||||
@@ -137,6 +147,7 @@ public static class InfrastructureServiceCollectionExtensions
|
||||
});
|
||||
services.AddScoped<IDraftRepository, DraftRepository>();
|
||||
services.AddScoped<ISearchRepository, SearchRepository>();
|
||||
services.AddScoped<IUnifiedSearchRepository, UnifiedSearchRepository>();
|
||||
services.AddSingleton<IMarkdownRenderer, MarkdownRenderer>();
|
||||
services.AddSingleton<ITemplateProvider, TemplateProvider>();
|
||||
services.AddScoped<LinkResolutionService>();
|
||||
@@ -152,7 +163,10 @@ public static class InfrastructureServiceCollectionExtensions
|
||||
services.AddScoped<IPageRollbackRepository, PageRollbackRepository>();
|
||||
services.AddScoped<ISourceBatchPurgeRepository, SourceBatchPurgeRepository>();
|
||||
services.AddScoped<ILeafWikiSourceReader, FileSystemLeafWikiSourceReader>();
|
||||
services.AddSingleton<ILeafWikiAdapterCatalog>(
|
||||
// The catalog resolves scoped adapters (including the SQLite reader).
|
||||
// Keep it scoped so request-time preflight never resolves them from the
|
||||
// root provider or holds a reader beyond its request scope.
|
||||
services.AddScoped<ILeafWikiAdapterCatalog>(
|
||||
provider => new LeafWikiAdapterCatalog(
|
||||
[
|
||||
// A database-backed adapter is preferred when both could apply: it can
|
||||
@@ -188,7 +202,15 @@ public static class InfrastructureServiceCollectionExtensions
|
||||
services.AddScoped<IIdempotencyStore, IdempotencyStore>();
|
||||
services.AddSingleton<IServiceAccountTokenIssuer, ServiceAccountTokenIssuer>();
|
||||
|
||||
services.AddSingleton(_ => new BackupPaths(ResolveBackupsDirectory(configuration), ResolveAttachmentsDirectory(configuration)));
|
||||
services.AddSingleton(provider =>
|
||||
{
|
||||
var resolvedConfiguration = provider.GetRequiredService<IConfiguration>();
|
||||
return new BackupPaths(
|
||||
ResolveBackupsDirectory(resolvedConfiguration),
|
||||
ResolveAttachmentsDirectory(resolvedConfiguration),
|
||||
ResolveKeysDirectory(resolvedConfiguration));
|
||||
});
|
||||
services.AddSingleton<IBackupEvidenceStore, FileBackupEvidenceStore>();
|
||||
services.AddSingleton<IBackupService>(provider => new PostgresBackupService(
|
||||
provider.GetRequiredService<DatabaseConnectionOptions>(),
|
||||
provider.GetRequiredService<BackupPaths>(),
|
||||
@@ -213,6 +235,10 @@ public static class InfrastructureServiceCollectionExtensions
|
||||
configuration["KnowledgeHub:Paths:Backups"]
|
||||
?? Path.Combine(AppContext.BaseDirectory, "backups-data");
|
||||
|
||||
private static string ResolveKeysDirectory(IConfiguration configuration) =>
|
||||
configuration["KnowledgeHub:Paths:Keys"]
|
||||
?? Path.Combine(AppContext.BaseDirectory, "keys-data");
|
||||
|
||||
private static string ResolveReleaseManifestPath(IConfiguration configuration) =>
|
||||
configuration["KnowledgeHub:Paths:ReleaseManifest"]
|
||||
?? Path.Combine(AppContext.BaseDirectory, "release-manifest.json");
|
||||
|
||||
@@ -0,0 +1,417 @@
|
||||
using System.Data;
|
||||
using ItWorx.KnowledgeHub.Application.Audit;
|
||||
using ItWorx.KnowledgeHub.Application.Jobs;
|
||||
using ItWorx.KnowledgeHub.Domain.Jobs;
|
||||
using ItWorx.KnowledgeHub.Infrastructure.Persistence;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Npgsql;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Infrastructure.Jobs;
|
||||
|
||||
public sealed class BackgroundJobRepository(
|
||||
KnowledgeHubDbContext db,
|
||||
IAuditEventRecorder auditEvents) : IBackgroundJobRepository
|
||||
{
|
||||
public async Task<EnqueueJobResult> EnqueueAsync(
|
||||
BackgroundJob job,
|
||||
Guid? rootPageId,
|
||||
string? tag,
|
||||
int maxCoursePages,
|
||||
BackgroundJobQueueLimits limits,
|
||||
AuditEventRecord auditEvent,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await using var transaction = await db.Database.BeginTransactionAsync(IsolationLevel.ReadCommitted, cancellationToken);
|
||||
// Serialize quota checks with insertion across processes; a burst of
|
||||
// distinct idempotency keys must not race past the durable queue budget.
|
||||
var lockKey = "knowledge-hub:job-enqueue:" + job.WorkspaceId.ToString("N");
|
||||
await db.Database.ExecuteSqlInterpolatedAsync($"SELECT pg_advisory_xact_lock(hashtext({lockKey}))", cancellationToken);
|
||||
var prior = await FindByIdempotencyCoreAsync(job.WorkspaceId, job.ActorId, job.IdempotencyKeyHash, cancellationToken);
|
||||
if (prior is not null)
|
||||
{
|
||||
await transaction.RollbackAsync(cancellationToken);
|
||||
return new EnqueueJobResult(
|
||||
prior.PayloadHash == job.PayloadHash ? EnqueueJobStatus.Existing : EnqueueJobStatus.IdempotencyConflict,
|
||||
prior.PayloadHash == job.PayloadHash ? prior : null);
|
||||
}
|
||||
|
||||
var pending = db.BackgroundJobs.Where(item => item.WorkspaceId == job.WorkspaceId
|
||||
&& (item.Status == BackgroundJobStatus.Queued || item.Status == BackgroundJobStatus.Running));
|
||||
var since = job.CreatedAt.AddDays(-1);
|
||||
if (await pending.CountAsync(cancellationToken) >= limits.PendingPerWorkspace
|
||||
|| await pending.CountAsync(item => item.ActorId == job.ActorId, cancellationToken) >= limits.PendingPerActor
|
||||
|| await db.BackgroundJobs.CountAsync(item => item.WorkspaceId == job.WorkspaceId && item.CreatedAt >= since, cancellationToken) >= limits.CreatedPerDay)
|
||||
{
|
||||
await transaction.RollbackAsync(cancellationToken);
|
||||
return new EnqueueJobResult(EnqueueJobStatus.QueueFull, null);
|
||||
}
|
||||
|
||||
db.BackgroundJobs.Add(job);
|
||||
try
|
||||
{
|
||||
await db.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
catch (DbUpdateException exception) when (IsIdempotencyViolation(exception))
|
||||
{
|
||||
await transaction.RollbackAsync(cancellationToken);
|
||||
await transaction.DisposeAsync();
|
||||
db.ChangeTracker.Clear();
|
||||
prior = await FindByIdempotencyCoreAsync(job.WorkspaceId, job.ActorId, job.IdempotencyKeyHash, cancellationToken);
|
||||
return prior is not null && prior.PayloadHash == job.PayloadHash
|
||||
? new EnqueueJobResult(EnqueueJobStatus.Existing, prior)
|
||||
: new EnqueueJobResult(EnqueueJobStatus.IdempotencyConflict, null);
|
||||
}
|
||||
|
||||
if (job.Type == BackgroundJobType.CourseTagging)
|
||||
{
|
||||
if (rootPageId is null || string.IsNullOrWhiteSpace(tag))
|
||||
throw new InvalidOperationException("Course-tagging payload is incomplete.");
|
||||
|
||||
await db.Database.ExecuteSqlInterpolatedAsync($"""
|
||||
WITH RECURSIVE subtree(page_id) AS (
|
||||
SELECT id FROM pages
|
||||
WHERE id = {rootPageId.Value} AND workspace_id = {job.WorkspaceId} AND trashed_at IS NULL
|
||||
UNION
|
||||
SELECT child.id FROM pages child
|
||||
INNER JOIN subtree parent ON child.parent_page_id = parent.page_id
|
||||
WHERE child.workspace_id = {job.WorkspaceId} AND child.trashed_at IS NULL
|
||||
)
|
||||
INSERT INTO background_job_items (job_id, page_id, initially_tagged, state)
|
||||
SELECT {job.Id}, page.id,
|
||||
EXISTS (SELECT 1 FROM unnest(page.tags) AS existing_tags(existing_tag) WHERE lower(existing_tags.existing_tag) = {tag}),
|
||||
'Pending'
|
||||
FROM subtree
|
||||
INNER JOIN pages page ON page.id = subtree.page_id
|
||||
LIMIT {maxCoursePages + 1}
|
||||
""", cancellationToken);
|
||||
|
||||
var itemCount = await db.BackgroundJobItems.CountAsync(item => item.JobId == job.Id, cancellationToken);
|
||||
if (itemCount == 0)
|
||||
{
|
||||
await transaction.RollbackAsync(cancellationToken);
|
||||
db.ChangeTracker.Clear();
|
||||
return new EnqueueJobResult(EnqueueJobStatus.RootNotFound, null);
|
||||
}
|
||||
if (itemCount > maxCoursePages)
|
||||
{
|
||||
await transaction.RollbackAsync(cancellationToken);
|
||||
db.ChangeTracker.Clear();
|
||||
return new EnqueueJobResult(EnqueueJobStatus.TooManyPages, null);
|
||||
}
|
||||
|
||||
job.SetTotal(itemCount);
|
||||
await db.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
|
||||
auditEvents.Add(auditEvent);
|
||||
await db.SaveChangesAsync(cancellationToken);
|
||||
await transaction.CommitAsync(cancellationToken);
|
||||
return new EnqueueJobResult(EnqueueJobStatus.Created, job);
|
||||
}
|
||||
|
||||
public Task<BackgroundJob?> FindAsync(Guid workspaceId, Guid jobId, CancellationToken cancellationToken) =>
|
||||
db.BackgroundJobs.AsNoTracking()
|
||||
.FirstOrDefaultAsync(job => job.WorkspaceId == workspaceId && job.Id == jobId, cancellationToken);
|
||||
|
||||
public async Task<IReadOnlyList<BackgroundJob>> ListAsync(Guid workspaceId, int limit, CancellationToken cancellationToken) =>
|
||||
await db.BackgroundJobs.AsNoTracking()
|
||||
.Where(job => job.WorkspaceId == workspaceId)
|
||||
.OrderByDescending(job => job.CreatedAt)
|
||||
.ThenByDescending(job => job.Id)
|
||||
.Take(Math.Clamp(limit, 1, 100))
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
public Task<BackgroundJob?> FindByIdempotencyAsync(
|
||||
Guid workspaceId, Guid actorId, string keyHash, CancellationToken cancellationToken) =>
|
||||
FindByIdempotencyCoreAsync(workspaceId, actorId, keyHash, cancellationToken);
|
||||
|
||||
public async Task<bool> RequestCancellationAsync(
|
||||
Guid workspaceId,
|
||||
Guid jobId,
|
||||
DateTimeOffset now,
|
||||
AuditEventRecord auditEvent,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await using var transaction = await db.Database.BeginTransactionAsync(cancellationToken);
|
||||
// Preserve cancellation intent even when a worker claims or renews the row
|
||||
// concurrently. A repeated request is a no-op, so it cannot emit duplicate
|
||||
// audit records or advance the snapshot version again.
|
||||
var updated = await db.Database.ExecuteSqlInterpolatedAsync($"""
|
||||
UPDATE background_jobs
|
||||
SET cancel_requested_at = COALESCE(cancel_requested_at, {now}),
|
||||
completed_at = CASE WHEN status = 'Queued' THEN {now} ELSE completed_at END,
|
||||
status = CASE WHEN status = 'Queued' THEN 'Cancelled' ELSE status END,
|
||||
version = version + 1
|
||||
WHERE workspace_id = {workspaceId} AND id = {jobId}
|
||||
AND status IN ('Queued', 'Running') AND cancel_requested_at IS NULL
|
||||
""", cancellationToken);
|
||||
if (updated > 0)
|
||||
{
|
||||
await db.BackgroundJobItems.Where(item => item.JobId == jobId
|
||||
&& db.BackgroundJobs.Any(job => job.Id == jobId && job.WorkspaceId == workspaceId && job.Status == BackgroundJobStatus.Cancelled))
|
||||
.ExecuteDeleteAsync(cancellationToken);
|
||||
auditEvents.Add(auditEvent);
|
||||
await db.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
await transaction.CommitAsync(cancellationToken);
|
||||
return updated > 0;
|
||||
}
|
||||
|
||||
public async Task<Guid?> TryClaimNextAsync(
|
||||
Guid leaseToken,
|
||||
DateTimeOffset now,
|
||||
DateTimeOffset leaseUntil,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await using var transaction = await db.Database.BeginTransactionAsync(IsolationLevel.ReadCommitted, cancellationToken);
|
||||
|
||||
var expired = await db.BackgroundJobs
|
||||
.Where(job => job.Status == BackgroundJobStatus.Running
|
||||
&& job.LeaseUntil <= now
|
||||
&& job.AttemptCount >= 4)
|
||||
.OrderBy(job => job.LeaseUntil)
|
||||
.Take(4)
|
||||
.ToListAsync(cancellationToken);
|
||||
foreach (var job in expired)
|
||||
job.ScheduleRetry(now, now, "worker_recovery_exhausted", "De taak kon na procesonderbrekingen niet worden hervat.");
|
||||
|
||||
var cancelledQueued = await db.BackgroundJobs
|
||||
.Where(job => job.Status == BackgroundJobStatus.Queued && job.CancelRequestedAt != null)
|
||||
.OrderBy(job => job.CreatedAt)
|
||||
.Take(20)
|
||||
.ToListAsync(cancellationToken);
|
||||
foreach (var job in cancelledQueued)
|
||||
job.Cancel(now);
|
||||
|
||||
if (expired.Count > 0 || cancelledQueued.Count > 0)
|
||||
{
|
||||
await db.SaveChangesAsync(cancellationToken);
|
||||
var terminalIds = expired.Concat(cancelledQueued).Where(job => job.IsTerminal).Select(job => job.Id).ToArray();
|
||||
await db.BackgroundJobItems.Where(item => terminalIds.Contains(item.JobId)).ExecuteDeleteAsync(cancellationToken);
|
||||
}
|
||||
|
||||
var candidate = await db.BackgroundJobs
|
||||
.FromSqlInterpolated($"""
|
||||
SELECT * FROM background_jobs
|
||||
WHERE (status = 'Queued' AND next_attempt_at <= {now})
|
||||
OR (status = 'Running' AND lease_until <= {now} AND attempt_count < 4)
|
||||
ORDER BY created_at, id
|
||||
LIMIT 1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
""")
|
||||
.FirstOrDefaultAsync(cancellationToken);
|
||||
|
||||
if (candidate is null)
|
||||
{
|
||||
await transaction.CommitAsync(cancellationToken);
|
||||
return null;
|
||||
}
|
||||
|
||||
candidate.Start(leaseToken, now, leaseUntil);
|
||||
await db.SaveChangesAsync(cancellationToken);
|
||||
await transaction.CommitAsync(cancellationToken);
|
||||
return candidate.Id;
|
||||
}
|
||||
|
||||
public async Task<bool> RenewLeaseAsync(Guid jobId, Guid leaseToken, DateTimeOffset leaseUntil, CancellationToken cancellationToken)
|
||||
{
|
||||
return await db.BackgroundJobs.Where(item => item.Id == jobId && item.Status == BackgroundJobStatus.Running && item.LeaseToken == leaseToken)
|
||||
.ExecuteUpdateAsync(update => update.SetProperty(item => item.LeaseUntil, leaseUntil), cancellationToken) == 1;
|
||||
}
|
||||
|
||||
public Task<BackgroundJob?> FindLeasedAsync(Guid jobId, Guid leaseToken, CancellationToken cancellationToken)
|
||||
{
|
||||
// Each mutation must use fresh cancellation and lease state rather than an
|
||||
// identity-resolved instance left in this long-lived worker scope.
|
||||
foreach (var entry in db.ChangeTracker.Entries<BackgroundJob>().Where(entry => entry.Entity.Id == jobId).ToArray())
|
||||
entry.State = EntityState.Detached;
|
||||
return db.BackgroundJobs.FirstOrDefaultAsync(
|
||||
job => job.Id == jobId && job.Status == BackgroundJobStatus.Running && job.LeaseToken == leaseToken,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<BackgroundJobPageWorkItem?> FindNextPageItemAsync(Guid jobId, Guid leaseToken, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!await HasLeaseAsync(jobId, leaseToken, cancellationToken))
|
||||
return null;
|
||||
return await db.BackgroundJobItems.AsNoTracking()
|
||||
.Where(item => item.JobId == jobId
|
||||
&& (item.State == BackgroundJobItemState.Pending || item.State == BackgroundJobItemState.Processing))
|
||||
.OrderBy(item => item.Ordinal)
|
||||
.Select(item => new BackgroundJobPageWorkItem(item.Ordinal, item.PageId, item.InitiallyTagged, item.State))
|
||||
.FirstOrDefaultAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public async Task MarkPageItemProcessingAsync(
|
||||
Guid jobId, Guid leaseToken, long ordinal, DateTimeOffset now, CancellationToken cancellationToken)
|
||||
{
|
||||
var job = await FindLeasedAsync(jobId, leaseToken, cancellationToken)
|
||||
?? throw new InvalidOperationException("The job lease is no longer valid.");
|
||||
var item = await db.BackgroundJobItems.FirstAsync(
|
||||
row => row.JobId == jobId && row.Ordinal == ordinal, cancellationToken);
|
||||
item.MarkProcessing(now);
|
||||
job.Touch();
|
||||
await SaveWorkerChangesAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public async Task RecordPageItemOutcomeAsync(
|
||||
Guid jobId,
|
||||
Guid leaseToken,
|
||||
long ordinal,
|
||||
BackgroundJobItemState outcome,
|
||||
string? errorCode,
|
||||
string? errorMessage,
|
||||
Guid? pageId,
|
||||
DateTimeOffset now,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var job = await FindLeasedAsync(jobId, leaseToken, cancellationToken)
|
||||
?? throw new InvalidOperationException("The job lease is no longer valid.");
|
||||
var item = await db.BackgroundJobItems.FirstAsync(
|
||||
row => row.JobId == jobId && row.Ordinal == ordinal, cancellationToken);
|
||||
if (item.State is BackgroundJobItemState.Succeeded or BackgroundJobItemState.Skipped or BackgroundJobItemState.Failed)
|
||||
return;
|
||||
item.Complete(outcome, now);
|
||||
job.RecordOutcome(outcome, errorCode, errorMessage, pageId);
|
||||
await SaveWorkerChangesAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public async Task RecordJobOutcomeAsync(
|
||||
Guid jobId,
|
||||
Guid leaseToken,
|
||||
BackgroundJobItemState outcome,
|
||||
string? errorCode,
|
||||
string? errorMessage,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var job = await FindLeasedAsync(jobId, leaseToken, cancellationToken)
|
||||
?? throw new InvalidOperationException("The job lease is no longer valid.");
|
||||
job.RecordOutcome(outcome, errorCode, errorMessage);
|
||||
await SaveWorkerChangesAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public async Task CompleteLeasedJobAsync(
|
||||
Guid jobId, Guid leaseToken, string? resultJson, DateTimeOffset now, CancellationToken cancellationToken)
|
||||
{
|
||||
var job = await FindLeasedAsync(jobId, leaseToken, cancellationToken)
|
||||
?? throw new InvalidOperationException("The job lease is no longer valid.");
|
||||
job.Complete(now, resultJson);
|
||||
await SaveStateAsync(job, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task CancelLeasedJobAsync(Guid jobId, Guid leaseToken, DateTimeOffset now, CancellationToken cancellationToken)
|
||||
{
|
||||
var job = await FindLeasedAsync(jobId, leaseToken, cancellationToken)
|
||||
?? throw new InvalidOperationException("The job lease is no longer valid.");
|
||||
job.Cancel(now);
|
||||
await SaveStateAsync(job, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task RetryLeasedJobAsync(
|
||||
Guid jobId, Guid leaseToken, DateTimeOffset now, DateTimeOffset retryAt, string code, string safeMessage,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var job = await FindLeasedAsync(jobId, leaseToken, cancellationToken)
|
||||
?? throw new InvalidOperationException("The job lease is no longer valid.");
|
||||
job.ScheduleRetry(now, retryAt, code, safeMessage);
|
||||
await SaveStateAsync(job, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task FailLeasedJobAsync(
|
||||
Guid jobId, Guid leaseToken, DateTimeOffset now, string code, string safeMessage,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var job = await FindLeasedAsync(jobId, leaseToken, cancellationToken)
|
||||
?? throw new InvalidOperationException("The job lease is no longer valid.");
|
||||
job.Fail(now, code, safeMessage);
|
||||
await SaveStateAsync(job, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task SaveStateAsync(BackgroundJob job, CancellationToken cancellationToken)
|
||||
{
|
||||
await using var transaction = await db.Database.BeginTransactionAsync(cancellationToken);
|
||||
await SaveWorkerChangesAsync(cancellationToken);
|
||||
// Terminal jobs retain their counters, result, bounded errors and audit
|
||||
// history. Their potentially large recovery snapshot is no longer needed.
|
||||
if (job.IsTerminal)
|
||||
await db.BackgroundJobItems.Where(item => item.JobId == job.Id).ExecuteDeleteAsync(cancellationToken);
|
||||
await transaction.CommitAsync(cancellationToken);
|
||||
}
|
||||
|
||||
private async Task SaveWorkerChangesAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await db.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
catch (DbUpdateConcurrencyException exception)
|
||||
{
|
||||
var jobEntry = exception.Entries.FirstOrDefault(entry => entry.Entity is BackgroundJob);
|
||||
if (jobEntry?.Entity is not BackgroundJob job)
|
||||
throw;
|
||||
|
||||
// Cancellation intentionally updates a running row with SQL so it cannot
|
||||
// be lost behind a tracked worker snapshot. If that update wins the race,
|
||||
// retry this worker save against the newer version while preserving the
|
||||
// cancellation columns (which this tracked instance never modified).
|
||||
var persisted = await db.BackgroundJobs.AsNoTracking()
|
||||
.Where(item => item.Id == job.Id)
|
||||
.Select(item => new { item.Version, item.Status, item.LeaseToken })
|
||||
.SingleOrDefaultAsync(cancellationToken);
|
||||
var version = jobEntry.Property(nameof(BackgroundJob.Version));
|
||||
if (persisted is null
|
||||
|| persisted.Status != BackgroundJobStatus.Running
|
||||
|| persisted.LeaseToken != job.LeaseToken
|
||||
|| persisted.Version <= (long)version.OriginalValue!)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
|
||||
version.OriginalValue = persisted.Version;
|
||||
version.CurrentValue = checked(persisted.Version + 1);
|
||||
await db.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<bool> IsCancellationRequestedAsync(Guid jobId, Guid leaseToken, CancellationToken cancellationToken) =>
|
||||
await db.BackgroundJobs.AsNoTracking().AnyAsync(
|
||||
job => job.Id == jobId && job.Status == BackgroundJobStatus.Running
|
||||
&& job.LeaseToken == leaseToken && job.CancelRequestedAt != null,
|
||||
cancellationToken);
|
||||
|
||||
public async Task<BackgroundJobQueueStatistics> GetQueueStatisticsAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var queued = await db.BackgroundJobs.LongCountAsync(job => job.Status == BackgroundJobStatus.Queued, cancellationToken);
|
||||
var running = await db.BackgroundJobs.LongCountAsync(job => job.Status == BackgroundJobStatus.Running, cancellationToken);
|
||||
var failed = await db.BackgroundJobs.LongCountAsync(job => job.Status == BackgroundJobStatus.Failed, cancellationToken);
|
||||
var oldest = queued == 0
|
||||
? null
|
||||
: await db.BackgroundJobs.Where(job => job.Status == BackgroundJobStatus.Queued)
|
||||
.MinAsync(job => (DateTimeOffset?)job.CreatedAt, cancellationToken);
|
||||
return new BackgroundJobQueueStatistics(queued, running, failed, oldest);
|
||||
}
|
||||
|
||||
private Task<bool> HasLeaseAsync(Guid jobId, Guid leaseToken, CancellationToken cancellationToken) =>
|
||||
db.BackgroundJobs.AsNoTracking().AnyAsync(
|
||||
job => job.Id == jobId && job.Status == BackgroundJobStatus.Running && job.LeaseToken == leaseToken,
|
||||
cancellationToken);
|
||||
|
||||
private Task<BackgroundJob?> FindByIdempotencyCoreAsync(
|
||||
Guid workspaceId, Guid actorId, string keyHash, CancellationToken cancellationToken) =>
|
||||
db.BackgroundJobs.AsNoTracking().FirstOrDefaultAsync(
|
||||
job => job.WorkspaceId == workspaceId && job.ActorId == actorId
|
||||
&& job.IdempotencyKeyHash == keyHash,
|
||||
cancellationToken);
|
||||
|
||||
private static bool IsIdempotencyViolation(Exception exception)
|
||||
{
|
||||
for (var current = exception; current is not null; current = current.InnerException!)
|
||||
{
|
||||
if (current is PostgresException { SqlState: PostgresErrorCodes.UniqueViolation, ConstraintName: "ux_background_jobs_idempotency" })
|
||||
return true;
|
||||
if (current.InnerException is null)
|
||||
break;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -3,4 +3,10 @@ namespace ItWorx.KnowledgeHub.Infrastructure.Operations;
|
||||
/// <summary>Resolved filesystem roots the backup/restore services read and write —
|
||||
/// kept as one small DI-registered value rather than each service re-reading
|
||||
/// configuration independently.</summary>
|
||||
public sealed record BackupPaths(string BackupsRoot, string AttachmentsRoot);
|
||||
public sealed record BackupPaths(string BackupsRoot, string AttachmentsRoot, string KeysRoot)
|
||||
{
|
||||
/// <summary>Authentication key for backup integrity receipts. This file must
|
||||
/// remain outside <see cref="BackupsRoot"/> so a backup-directory write access
|
||||
/// cannot be used to forge its own authenticity evidence.</summary>
|
||||
public string IntegrityKeyPath => Path.Combine(KeysRoot, "backup-integrity.key");
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
namespace ItWorx.KnowledgeHub.Infrastructure.Operations;
|
||||
|
||||
internal static class BackupStorageSafety
|
||||
{
|
||||
private static readonly UnixFileMode OwnerOnlyDirectory =
|
||||
UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute;
|
||||
private static readonly UnixFileMode OwnerOnlyFile = UnixFileMode.UserRead | UnixFileMode.UserWrite;
|
||||
|
||||
internal static bool SafeId(string? id) => id is { Length: > 0 and <= 100 }
|
||||
&& id.All(c => char.IsAsciiLetterOrDigit(c) || c is '-' or '_');
|
||||
|
||||
internal static void EnsurePrivateDirectory(string path)
|
||||
{
|
||||
var fullPath = Path.GetFullPath(path);
|
||||
var pathRoot = Path.GetPathRoot(fullPath)!;
|
||||
if (fullPath.TrimEnd(Path.DirectorySeparatorChar) == pathRoot.TrimEnd(Path.DirectorySeparatorChar)
|
||||
|| !IsSafePath(path, path))
|
||||
throw new InvalidOperationException("Onveilige back-upopslag.");
|
||||
|
||||
if (OperatingSystem.IsWindows())
|
||||
{
|
||||
Directory.CreateDirectory(path);
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(path, OwnerOnlyDirectory);
|
||||
File.SetUnixFileMode(path, OwnerOnlyDirectory);
|
||||
if (File.GetUnixFileMode(path) != OwnerOnlyDirectory)
|
||||
throw new IOException("Unix directory permissions did not match the requested private mode.");
|
||||
}
|
||||
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException or NotSupportedException)
|
||||
{
|
||||
throw new IOException("De back-upmap kan niet tot Unix-modus 0700 worden beperkt.", exception);
|
||||
}
|
||||
}
|
||||
|
||||
internal static FileStream CreatePrivateFile(
|
||||
string path,
|
||||
FileMode mode = FileMode.CreateNew,
|
||||
FileAccess access = FileAccess.Write,
|
||||
FileOptions options = FileOptions.None)
|
||||
{
|
||||
var streamOptions = new FileStreamOptions
|
||||
{
|
||||
Mode = mode,
|
||||
Access = access,
|
||||
Share = FileShare.None,
|
||||
Options = options
|
||||
};
|
||||
if (!OperatingSystem.IsWindows())
|
||||
streamOptions.UnixCreateMode = OwnerOnlyFile;
|
||||
|
||||
FileStream? stream = null;
|
||||
try
|
||||
{
|
||||
stream = new FileStream(path, streamOptions);
|
||||
if (!OperatingSystem.IsWindows())
|
||||
{
|
||||
File.SetUnixFileMode(path, OwnerOnlyFile);
|
||||
if (File.GetUnixFileMode(path) != OwnerOnlyFile)
|
||||
throw new IOException("Unix file permissions did not match the requested private mode.");
|
||||
}
|
||||
return stream;
|
||||
}
|
||||
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException or NotSupportedException)
|
||||
{
|
||||
stream?.Dispose();
|
||||
throw new IOException("Het back-upbestand kan niet tot Unix-modus 0600 worden beperkt.", exception);
|
||||
}
|
||||
}
|
||||
|
||||
// Lexical containment alone does not stop a symlink or Windows junction.
|
||||
internal static bool IsSafePath(string root, string candidate)
|
||||
{
|
||||
var fullRoot = Path.GetFullPath(root);
|
||||
var fullCandidate = Path.GetFullPath(candidate);
|
||||
var comparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
|
||||
if (!fullCandidate.Equals(fullRoot, comparison)
|
||||
&& !fullCandidate.StartsWith(fullRoot.TrimEnd(Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar, comparison))
|
||||
return false;
|
||||
for (var current = fullCandidate; current is not null; current = Path.GetDirectoryName(current))
|
||||
{
|
||||
if ((File.Exists(current) || Directory.Exists(current))
|
||||
&& (File.GetAttributes(current) & FileAttributes.ReparsePoint) != 0) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
internal static FileStream AcquireLease(string root)
|
||||
{
|
||||
EnsurePrivateDirectory(root);
|
||||
var path = Path.Combine(root, ".backup.lock");
|
||||
if (!IsSafePath(root, path)) throw new InvalidOperationException("Onveilige back-upopslag.");
|
||||
// Do not unlink the lock: another process could otherwise acquire the old
|
||||
// inode and a third process a new inode. Closing releases the OS lease.
|
||||
return CreatePrivateFile(path, FileMode.OpenOrCreate, FileAccess.ReadWrite);
|
||||
}
|
||||
}
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
using System.Data.Common;
|
||||
using ItWorx.KnowledgeHub.Application.Operations;
|
||||
using Microsoft.EntityFrameworkCore.Diagnostics;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Infrastructure.Operations;
|
||||
|
||||
public sealed class DatabaseMetricsInterceptor(PlatformMetrics metrics) : DbCommandInterceptor
|
||||
{
|
||||
public override DbDataReader ReaderExecuted(DbCommand command, CommandExecutedEventData eventData, DbDataReader result)
|
||||
{ Record(eventData); return result; }
|
||||
public override ValueTask<DbDataReader> ReaderExecutedAsync(DbCommand command, CommandExecutedEventData eventData, DbDataReader result, CancellationToken cancellationToken = default)
|
||||
{ Record(eventData); return ValueTask.FromResult(result); }
|
||||
public override int NonQueryExecuted(DbCommand command, CommandExecutedEventData eventData, int result)
|
||||
{ Record(eventData); return result; }
|
||||
public override ValueTask<int> NonQueryExecutedAsync(DbCommand command, CommandExecutedEventData eventData, int result, CancellationToken cancellationToken = default)
|
||||
{ Record(eventData); return ValueTask.FromResult(result); }
|
||||
public override object? ScalarExecuted(DbCommand command, CommandExecutedEventData eventData, object? result)
|
||||
{ Record(eventData); return result; }
|
||||
public override ValueTask<object?> ScalarExecutedAsync(DbCommand command, CommandExecutedEventData eventData, object? result, CancellationToken cancellationToken = default)
|
||||
{ Record(eventData); return ValueTask.FromResult(result); }
|
||||
public override void CommandFailed(DbCommand command, CommandErrorEventData eventData) => metrics.Record("database", "failure", eventData.Duration.TotalSeconds);
|
||||
public override Task CommandFailedAsync(DbCommand command, CommandErrorEventData eventData, CancellationToken cancellationToken = default)
|
||||
{ CommandFailed(command, eventData); return Task.CompletedTask; }
|
||||
private void Record(CommandExecutedEventData data) => metrics.Record("database", "success", data.Duration.TotalSeconds);
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
using System.Globalization;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using ItWorx.KnowledgeHub.Application.Operations;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Infrastructure.Operations;
|
||||
|
||||
/// <summary>Stores backup evidence beside each backup while authenticating every
|
||||
/// verification receipt with a persistent HMAC key outside the backup tree.</summary>
|
||||
public sealed class FileBackupEvidenceStore(BackupPaths paths, TimeProvider clock) : IBackupEvidenceStore
|
||||
{
|
||||
internal const string ProtectionFile = ".protection.json";
|
||||
internal const string IntegrityFile = ".integrity.json";
|
||||
private const string RestoreFile = ".restore.json";
|
||||
private const int SignatureVersion = 1;
|
||||
private const int IntegrityKeyLength = 32;
|
||||
private const int SignatureHexLength = 64;
|
||||
|
||||
public async Task<BackupEvidence?> GetAsync(
|
||||
string backupId, CancellationToken cancellationToken, string? expectedManifestSha256 = null)
|
||||
{
|
||||
var directory = DirectoryFor(backupId);
|
||||
if (directory is null) return null;
|
||||
var hash = await ManifestHashAsync(directory, cancellationToken);
|
||||
var protection = await ReadAsync<ProtectionRecord>(directory, ProtectionFile, cancellationToken);
|
||||
var integrity = await ReadAsync<SignedVerificationRecord>(directory, IntegrityFile, cancellationToken);
|
||||
var restore = await ReadAsync<SignedVerificationRecord>(directory, RestoreFile, cancellationToken);
|
||||
|
||||
// A malformed protection marker is fail-closed for retention. Verification
|
||||
// markers are fail-closed for trust: without the external key and valid MAC,
|
||||
// editing both the manifest and its adjacent receipt cannot authenticate it.
|
||||
var protectedBackup = protection?.Protected ?? File.Exists(Path.Combine(directory, ProtectionFile));
|
||||
var validIntegrity = integrity is not null
|
||||
&& integrity.BackupId == backupId
|
||||
&& integrity.ManifestSha256 == hash
|
||||
&& (expectedManifestSha256 is null
|
||||
|| string.Equals(integrity.ManifestSha256, expectedManifestSha256, StringComparison.Ordinal))
|
||||
&& ValidTime(integrity.CompletedAt)
|
||||
&& integrity.Success
|
||||
&& await HasValidSignatureAsync(integrity, "integrity", cancellationToken);
|
||||
var validRestore = restore is not null
|
||||
&& restore.BackupId == backupId
|
||||
&& restore.ManifestSha256 == hash
|
||||
&& ValidTime(restore.CompletedAt)
|
||||
&& await HasValidSignatureAsync(restore, "restore", cancellationToken);
|
||||
|
||||
return new BackupEvidence(protectedBackup, protection?.Version ?? 0,
|
||||
validIntegrity ? integrity!.CompletedAt : null,
|
||||
validRestore && restore!.Success ? restore.CompletedAt : null,
|
||||
validRestore ? restore!.Success : null);
|
||||
}
|
||||
|
||||
public async Task RecordRestoreAsync(string backupId, RestoreReport report, CancellationToken cancellationToken)
|
||||
{
|
||||
var directory = DirectoryFor(backupId) ?? throw new InvalidOperationException("Back-up niet gevonden.");
|
||||
if (report.BackupId != backupId) throw new ArgumentException("Herstelbewijs hoort bij een andere back-up.");
|
||||
var currentHash = await ManifestHashAsync(directory, cancellationToken);
|
||||
if (report.ManifestSha256 != currentHash)
|
||||
throw new IOException("Back-upmanifest is gewijzigd; herstelbewijs is niet opgeslagen.");
|
||||
if ((await GetAsync(backupId, cancellationToken))?.IntegrityVerifiedAt is null)
|
||||
throw new IOException("Back-upintegriteit is niet bevestigd; herstelbewijs is niet opgeslagen.");
|
||||
|
||||
var key = await ReadIntegrityKeyAsync(createIfMissing: false, cancellationToken)
|
||||
?? throw new IOException("De sleutel voor back-upintegriteitsbewijs ontbreekt of is ongeldig.");
|
||||
var success = report.Success && report.Tables.All(t => t.Matches) && report.AttachmentsMatch;
|
||||
var record = CreateSignedRecord(key, "restore", backupId, currentHash, clock.GetUtcNow(), success);
|
||||
await WriteAsync(directory, RestoreFile, record, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<BackupProtectionResult> SetProtectionAsync(
|
||||
string backupId, bool protect, int expectedVersion, Guid actorId, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!BackupStorageSafety.SafeId(backupId)) return BackupProtectionResult.NotFound;
|
||||
await using var lease = BackupStorageSafety.AcquireLease(paths.BackupsRoot);
|
||||
var directory = DirectoryFor(backupId);
|
||||
if (directory is null) return BackupProtectionResult.NotFound;
|
||||
var current = await ReadAsync<ProtectionRecord>(directory, ProtectionFile, cancellationToken);
|
||||
if ((current?.Version ?? 0) != expectedVersion) return BackupProtectionResult.Conflict;
|
||||
// A broken marker cannot be cleared through a fabricated version zero.
|
||||
if (current is null && File.Exists(Path.Combine(directory, ProtectionFile))) return BackupProtectionResult.Conflict;
|
||||
await WriteAsync(directory, ProtectionFile, new ProtectionRecord(
|
||||
protect, checked(expectedVersion + 1), actorId, clock.GetUtcNow()), cancellationToken);
|
||||
return BackupProtectionResult.Updated;
|
||||
}
|
||||
|
||||
internal static async Task WriteIntegrityAsync(
|
||||
string directory, string backupId, BackupPaths paths, DateTimeOffset verifiedAt, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!BackupStorageSafety.IsSafePath(paths.BackupsRoot, directory)
|
||||
|| !BackupStorageSafety.IsSafePath(directory, Path.Combine(directory, "manifest.json")))
|
||||
throw new InvalidOperationException("Onveilige back-upopslag.");
|
||||
if (!BackupStorageSafety.SafeId(backupId)) throw new InvalidOperationException("Ongeldig back-up-ID.");
|
||||
|
||||
var store = new FileBackupEvidenceStore(paths, TimeProvider.System);
|
||||
var key = await store.ReadIntegrityKeyAsync(createIfMissing: true, cancellationToken)
|
||||
?? throw new InvalidOperationException("De sleutel voor back-upintegriteitsbewijs ontbreekt of is ongeldig.");
|
||||
var manifestHash = await ManifestHashAsync(directory, cancellationToken);
|
||||
var record = CreateSignedRecord(key, "integrity", backupId, manifestHash, verifiedAt, success: true);
|
||||
await store.WriteAsync(directory, IntegrityFile, record, cancellationToken);
|
||||
}
|
||||
|
||||
private bool ValidTime(DateTimeOffset time) => time > DateTimeOffset.UnixEpoch && time <= clock.GetUtcNow();
|
||||
|
||||
private string? DirectoryFor(string id)
|
||||
{
|
||||
if (!BackupStorageSafety.SafeId(id)) return null;
|
||||
var directory = Path.Combine(paths.BackupsRoot, id);
|
||||
return BackupStorageSafety.IsSafePath(paths.BackupsRoot, directory)
|
||||
&& BackupStorageSafety.IsSafePath(directory, Path.Combine(directory, "manifest.json"))
|
||||
&& File.Exists(Path.Combine(directory, "manifest.json")) ? directory : null;
|
||||
}
|
||||
|
||||
private async Task<bool> HasValidSignatureAsync(
|
||||
SignedVerificationRecord record, string purpose, CancellationToken cancellationToken)
|
||||
{
|
||||
if (record.Version != SignatureVersion
|
||||
|| record.HmacSha256 is not { Length: SignatureHexLength }
|
||||
|| !record.HmacSha256.All(Uri.IsHexDigit))
|
||||
return false;
|
||||
|
||||
var key = await ReadIntegrityKeyAsync(createIfMissing: false, cancellationToken);
|
||||
if (key is null) return false;
|
||||
var expected = CreateSignature(key, purpose, record.BackupId, record.ManifestSha256,
|
||||
record.CompletedAt, record.Success);
|
||||
return CryptographicOperations.FixedTimeEquals(
|
||||
Encoding.ASCII.GetBytes(expected),
|
||||
Encoding.ASCII.GetBytes(record.HmacSha256.ToLowerInvariant()));
|
||||
}
|
||||
|
||||
private async Task<byte[]?> ReadIntegrityKeyAsync(bool createIfMissing, CancellationToken cancellationToken)
|
||||
{
|
||||
var keyPath = paths.IntegrityKeyPath;
|
||||
if (!BackupStorageSafety.IsSafePath(paths.KeysRoot, keyPath)
|
||||
|| BackupStorageSafety.IsSafePath(paths.BackupsRoot, keyPath))
|
||||
throw new InvalidOperationException("De sleutel voor back-upintegriteitsbewijs moet veilig buiten de back-upmap staan.");
|
||||
|
||||
if (!File.Exists(keyPath))
|
||||
{
|
||||
if (!createIfMissing) return null;
|
||||
await CreateIntegrityKeyAsync(keyPath, cancellationToken);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (!BackupStorageSafety.IsSafePath(paths.KeysRoot, keyPath)
|
||||
|| BackupStorageSafety.IsSafePath(paths.BackupsRoot, keyPath))
|
||||
throw new InvalidOperationException("De sleutel voor back-upintegriteitsbewijs staat niet veilig buiten de back-upmap.");
|
||||
|
||||
if (!OperatingSystem.IsWindows())
|
||||
{
|
||||
var ownerOnlyMode = UnixFileMode.UserRead | UnixFileMode.UserWrite;
|
||||
try
|
||||
{
|
||||
if (File.GetUnixFileMode(keyPath) != ownerOnlyMode)
|
||||
File.SetUnixFileMode(keyPath, ownerOnlyMode);
|
||||
if (File.GetUnixFileMode(keyPath) != ownerOnlyMode)
|
||||
return null;
|
||||
}
|
||||
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException or NotSupportedException)
|
||||
{
|
||||
if (createIfMissing)
|
||||
throw new InvalidOperationException("De sleutel voor back-upintegriteitsbewijs kan niet tot Unix-modus 0600 worden beperkt.", exception);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
var key = await File.ReadAllBytesAsync(keyPath, cancellationToken);
|
||||
return key.Length == IntegrityKeyLength ? key : null;
|
||||
}
|
||||
catch (FileNotFoundException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
catch (DirectoryNotFoundException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
catch (IOException) when (!createIfMissing)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
catch (UnauthorizedAccessException) when (!createIfMissing)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task CreateIntegrityKeyAsync(string keyPath, CancellationToken cancellationToken)
|
||||
{
|
||||
var keyDirectory = Path.GetDirectoryName(keyPath)!;
|
||||
Directory.CreateDirectory(keyDirectory);
|
||||
var tempPath = Path.Combine(keyDirectory, ".backup-integrity-key-" + Guid.NewGuid().ToString("N"));
|
||||
var key = RandomNumberGenerator.GetBytes(IntegrityKeyLength);
|
||||
try
|
||||
{
|
||||
var options = new FileStreamOptions
|
||||
{
|
||||
Mode = FileMode.CreateNew,
|
||||
Access = FileAccess.Write,
|
||||
Share = FileShare.None,
|
||||
Options = FileOptions.WriteThrough
|
||||
};
|
||||
if (!OperatingSystem.IsWindows())
|
||||
options.UnixCreateMode = UnixFileMode.UserRead | UnixFileMode.UserWrite;
|
||||
|
||||
await using (var stream = new FileStream(tempPath, options))
|
||||
{
|
||||
await stream.WriteAsync(key, cancellationToken);
|
||||
await stream.FlushAsync(cancellationToken);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
File.Move(tempPath, keyPath);
|
||||
}
|
||||
catch (IOException) when (File.Exists(keyPath))
|
||||
{
|
||||
// Another application process created the same persistent key first.
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
CryptographicOperations.ZeroMemory(key);
|
||||
if (File.Exists(tempPath)) File.Delete(tempPath);
|
||||
}
|
||||
}
|
||||
|
||||
private static SignedVerificationRecord CreateSignedRecord(
|
||||
byte[] key, string purpose, string backupId, string manifestHash, DateTimeOffset completedAt, bool success) =>
|
||||
new(SignatureVersion, backupId, manifestHash, completedAt, success,
|
||||
CreateSignature(key, purpose, backupId, manifestHash, completedAt, success));
|
||||
|
||||
private static string CreateSignature(
|
||||
byte[] key, string purpose, string backupId, string manifestHash, DateTimeOffset completedAt, bool success)
|
||||
{
|
||||
var payload = string.Join('\n',
|
||||
"itworx-knowledge-hub-backup-evidence-v1",
|
||||
purpose,
|
||||
backupId,
|
||||
manifestHash,
|
||||
completedAt.UtcTicks.ToString(CultureInfo.InvariantCulture),
|
||||
success ? "1" : "0");
|
||||
return Convert.ToHexStringLower(HMACSHA256.HashData(key, Encoding.UTF8.GetBytes(payload)));
|
||||
}
|
||||
|
||||
private static async Task<string> ManifestHashAsync(string directory, CancellationToken cancellationToken)
|
||||
{
|
||||
await using var stream = File.OpenRead(Path.Combine(directory, "manifest.json"));
|
||||
return Convert.ToHexStringLower(await SHA256.HashDataAsync(stream, cancellationToken));
|
||||
}
|
||||
|
||||
private static async Task<T?> ReadAsync<T>(string directory, string name, CancellationToken cancellationToken)
|
||||
{
|
||||
var path = Path.Combine(directory, name);
|
||||
if (!BackupStorageSafety.IsSafePath(directory, path) || !File.Exists(path)) return default;
|
||||
try
|
||||
{
|
||||
if (new FileInfo(path).Length > 16_384) return default;
|
||||
await using var stream = File.OpenRead(path);
|
||||
return await JsonSerializer.DeserializeAsync<T>(stream, cancellationToken: cancellationToken);
|
||||
}
|
||||
catch (JsonException) { return default; }
|
||||
}
|
||||
|
||||
private async Task WriteAsync<T>(string directory, string name, T record, CancellationToken cancellationToken)
|
||||
{
|
||||
BackupStorageSafety.EnsurePrivateDirectory(paths.BackupsRoot);
|
||||
BackupStorageSafety.EnsurePrivateDirectory(directory);
|
||||
var path = Path.Combine(directory, name);
|
||||
if (!BackupStorageSafety.IsSafePath(directory, path)) throw new InvalidOperationException("Onveilige back-upopslag.");
|
||||
var temp = Path.Combine(directory, ".evidence-" + Guid.NewGuid().ToString("N"));
|
||||
try
|
||||
{
|
||||
await using (var stream = BackupStorageSafety.CreatePrivateFile(temp, options: FileOptions.Asynchronous))
|
||||
await JsonSerializer.SerializeAsync(stream, record, cancellationToken: cancellationToken);
|
||||
File.Move(temp, path, true);
|
||||
}
|
||||
finally { if (File.Exists(temp)) File.Delete(temp); }
|
||||
}
|
||||
|
||||
private sealed record ProtectionRecord(bool Protected, int Version, Guid ActorId, DateTimeOffset UpdatedAt);
|
||||
private sealed record SignedVerificationRecord(
|
||||
int Version, string BackupId, string ManifestSha256, DateTimeOffset CompletedAt, bool Success, string? HmacSha256);
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
using ItWorx.KnowledgeHub.Application.Jobs;
|
||||
using ItWorx.KnowledgeHub.Application.Operations;
|
||||
using ItWorx.KnowledgeHub.Domain.Ingestion;
|
||||
using ItWorx.KnowledgeHub.Infrastructure.Persistence;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Infrastructure.Operations;
|
||||
|
||||
public sealed class OperationalTelemetryReader(
|
||||
KnowledgeHubDbContext db, IBackgroundJobRepository jobs,
|
||||
IBackupService backups, IBackupEvidenceStore evidence) : IOperationalTelemetryReader
|
||||
{
|
||||
public async Task<OperationalTelemetrySnapshot> ReadAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var queue = await jobs.GetQueueStatisticsAsync(cancellationToken);
|
||||
var pending = db.IngestionProposals.AsNoTracking().Where(p =>
|
||||
p.Status != IngestionProposalStatus.Published && p.Status != IngestionProposalStatus.Rejected);
|
||||
var oldest = await db.IngestionProposalRevisions.AsNoTracking()
|
||||
.Where(r => pending.Any(p => p.Id == EF.Property<Guid>(r, "IngestionProposalId")))
|
||||
.Select(r => (DateTimeOffset?)r.OccurredAt).MinAsync(cancellationToken);
|
||||
var capture = await db.InventorySnapshots.AsNoTracking()
|
||||
.Select(s => (DateTimeOffset?)s.CapturedAt).MaxAsync(cancellationToken);
|
||||
var completed = await backups.ListBackupsAsync(cancellationToken);
|
||||
DateTimeOffset? verifiedBackup = null;
|
||||
if (completed.Count > 0 &&
|
||||
await evidence.GetAsync(completed[0].BackupId, cancellationToken) is { IntegrityVerifiedAt: not null })
|
||||
verifiedBackup = completed[0].CreatedAt;
|
||||
return new(queue.Queued, queue.Running, queue.Failed, queue.OldestQueuedAt,
|
||||
oldest, capture, verifiedBackup);
|
||||
}
|
||||
}
|
||||
+77
-43
@@ -26,24 +26,24 @@ public sealed class PostgresBackupService(
|
||||
int retainedBackupCount = 14) : IBackupService
|
||||
{
|
||||
private const string TempPrefix = ".tmp-";
|
||||
private const string LockFileName = ".backup.lock";
|
||||
private static readonly JsonSerializerOptions ManifestSerializerOptions = new() { WriteIndented = true };
|
||||
|
||||
public async Task<BackupSummary> CreateBackupAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
Directory.CreateDirectory(paths.BackupsRoot);
|
||||
BackupStorageSafety.EnsurePrivateDirectory(paths.BackupsRoot);
|
||||
await using var lease = await AcquireLeaseAsync(cancellationToken);
|
||||
|
||||
var backupId = $"{timeProvider.GetUtcNow():yyyyMMddTHHmmssZ}-{Guid.NewGuid().ToString("N")[..8]}";
|
||||
var tempDir = Path.Combine(paths.BackupsRoot, TempPrefix + backupId);
|
||||
var dbDir = Path.Combine(tempDir, "db");
|
||||
var attachmentsDir = Path.Combine(tempDir, "attachments");
|
||||
Directory.CreateDirectory(dbDir);
|
||||
Directory.CreateDirectory(attachmentsDir);
|
||||
BackupStorageSafety.EnsurePrivateDirectory(tempDir);
|
||||
BackupStorageSafety.EnsurePrivateDirectory(dbDir);
|
||||
BackupStorageSafety.EnsurePrivateDirectory(attachmentsDir);
|
||||
|
||||
try
|
||||
{
|
||||
var tableEntries = await BackupTablesAsync(dbDir, cancellationToken);
|
||||
var (schemaVersion, tableEntries) = await BackupTablesAsync(dbDir, cancellationToken);
|
||||
var attachmentEntries = await BackupAttachmentsAsync(attachmentsDir, cancellationToken);
|
||||
|
||||
var manifest = new BackupManifest(
|
||||
@@ -52,7 +52,7 @@ public sealed class PostgresBackupService(
|
||||
"ItWorx Knowledge Hub",
|
||||
installationDefaults.ProductVersion,
|
||||
Environment.GetEnvironmentVariable("Build__Commit") ?? "development",
|
||||
installationDefaults.SchemaVersion,
|
||||
schemaVersion,
|
||||
tableEntries,
|
||||
attachmentEntries,
|
||||
tableEntries.Sum(t => t.SizeBytes) + attachmentEntries.Sum(a => a.SizeBytes));
|
||||
@@ -61,10 +61,13 @@ public sealed class PostgresBackupService(
|
||||
await VerifyFilesAsync(attachmentsDir, attachmentEntries.Select(a => (a.RelativePath, a.Sha256)), cancellationToken);
|
||||
|
||||
var manifestPath = Path.Combine(tempDir, "manifest.json");
|
||||
await File.WriteAllTextAsync(
|
||||
manifestPath,
|
||||
JsonSerializer.Serialize(manifest, ManifestSerializerOptions),
|
||||
cancellationToken);
|
||||
await using (var manifestStream = BackupStorageSafety.CreatePrivateFile(
|
||||
manifestPath, options: FileOptions.Asynchronous))
|
||||
{
|
||||
await JsonSerializer.SerializeAsync(
|
||||
manifestStream, manifest, ManifestSerializerOptions, cancellationToken);
|
||||
}
|
||||
await FileBackupEvidenceStore.WriteIntegrityAsync(tempDir, backupId, paths, timeProvider.GetUtcNow(), cancellationToken);
|
||||
|
||||
var finalDir = Path.Combine(paths.BackupsRoot, backupId);
|
||||
// Same-volume directory rename: the backup only becomes visible to
|
||||
@@ -123,35 +126,52 @@ public sealed class PostgresBackupService(
|
||||
|
||||
// --- table export ------------------------------------------------------------------
|
||||
|
||||
private async Task<IReadOnlyList<BackupTableManifestEntry>> BackupTablesAsync(string dbDir, CancellationToken cancellationToken)
|
||||
private async Task<(string SchemaVersion, IReadOnlyList<BackupTableManifestEntry> Tables)> BackupTablesAsync(
|
||||
string dbDir, CancellationToken cancellationToken)
|
||||
{
|
||||
await using var connection = new NpgsqlConnection(connectionOptions.ConnectionString);
|
||||
await connection.OpenAsync(cancellationToken);
|
||||
// COPY and row counts for every table share one PostgreSQL snapshot.
|
||||
await using var transaction = await connection.BeginTransactionAsync(System.Data.IsolationLevel.RepeatableRead, cancellationToken);
|
||||
|
||||
var tables = await TableDiscovery.DiscoverBaseTablesAsync(connection, cancellationToken);
|
||||
const string schemaVersionSql = """
|
||||
SELECT migration_id
|
||||
FROM "public"."__EFMigrationsHistory"
|
||||
ORDER BY migration_id DESC
|
||||
LIMIT 1
|
||||
""";
|
||||
await using var schemaVersionCommand = new NpgsqlCommand(schemaVersionSql, connection, transaction);
|
||||
var schemaVersion = (string?)await schemaVersionCommand.ExecuteScalarAsync(cancellationToken)
|
||||
?? throw new InvalidOperationException("Er is geen toegepaste databaseschemamigratie gevonden; de back-up is geweigerd.");
|
||||
var entries = new List<BackupTableManifestEntry>(tables.Count);
|
||||
|
||||
foreach (var table in tables)
|
||||
{
|
||||
// Keep the SQL boundary defensive even though discovery validates catalog
|
||||
// identifiers before returning them.
|
||||
PostgresIdentifierSafety.RequireSafe(table);
|
||||
var fileName = $"{table}.bin";
|
||||
var filePath = Path.Combine(dbDir, fileName);
|
||||
|
||||
await using (var rawCopy = await connection.BeginRawBinaryCopyAsync(
|
||||
$"COPY \"public\".\"{table}\" TO STDOUT (FORMAT BINARY)", cancellationToken))
|
||||
await using (var destination = new FileStream(filePath, FileMode.CreateNew, FileAccess.Write, FileShare.None))
|
||||
await using (var destination = BackupStorageSafety.CreatePrivateFile(
|
||||
filePath, options: FileOptions.Asynchronous))
|
||||
{
|
||||
await rawCopy.CopyToAsync(destination, cancellationToken);
|
||||
}
|
||||
|
||||
var (sizeBytes, sha256) = await ComputeFileHashAsync(filePath, cancellationToken);
|
||||
|
||||
await using var countCommand = new NpgsqlCommand($"SELECT count(*) FROM \"public\".\"{table}\"", connection);
|
||||
await using var countCommand = new NpgsqlCommand($"SELECT count(*) FROM \"public\".\"{table}\"", connection, transaction);
|
||||
var rowCount = (long)(await countCommand.ExecuteScalarAsync(cancellationToken) ?? 0L);
|
||||
|
||||
entries.Add(new BackupTableManifestEntry(table, fileName, rowCount, sizeBytes, sha256));
|
||||
}
|
||||
|
||||
return entries;
|
||||
await transaction.CommitAsync(cancellationToken);
|
||||
return (schemaVersion, entries);
|
||||
}
|
||||
|
||||
// --- attachment export ---------------------------------------------------------------
|
||||
@@ -177,9 +197,17 @@ public sealed class PostgresBackupService(
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!BackupStorageSafety.IsSafePath(paths.AttachmentsRoot, sourcePath))
|
||||
throw new InvalidOperationException("Onveilige bronopslag voor bijlagen.");
|
||||
|
||||
var destinationPath = Path.Combine(attachmentsDir, relative);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!);
|
||||
File.Copy(sourcePath, destinationPath);
|
||||
BackupStorageSafety.EnsurePrivateDirectory(Path.GetDirectoryName(destinationPath)!);
|
||||
await using (var source = new FileStream(sourcePath, FileMode.Open, FileAccess.Read, FileShare.Read))
|
||||
await using (var destination = BackupStorageSafety.CreatePrivateFile(
|
||||
destinationPath, options: FileOptions.Asynchronous))
|
||||
{
|
||||
await source.CopyToAsync(destination, cancellationToken);
|
||||
}
|
||||
|
||||
var (sizeBytes, sha256) = await ComputeFileHashAsync(destinationPath, cancellationToken);
|
||||
entries.Add(new BackupAttachmentManifestEntry(sha256, relative, sizeBytes));
|
||||
@@ -225,7 +253,7 @@ public sealed class PostgresBackupService(
|
||||
|
||||
// --- retention ------------------------------------------------------------------------
|
||||
|
||||
private Task ApplyRetentionAsync(CancellationToken cancellationToken)
|
||||
private async Task ApplyRetentionAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
@@ -240,10 +268,10 @@ public sealed class PostgresBackupService(
|
||||
|
||||
foreach (var stale in completed.Skip(Math.Max(1, retainedBackupCount)))
|
||||
{
|
||||
var evidence = await new FileBackupEvidenceStore(paths, timeProvider).GetAsync(stale.Manifest!.BackupId, cancellationToken);
|
||||
if (evidence?.Protected != false) continue;
|
||||
TryDeleteDirectory(stale.Dir);
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
// --- housekeeping ---------------------------------------------------------------------
|
||||
@@ -257,6 +285,7 @@ public sealed class PostgresBackupService(
|
||||
|
||||
foreach (var dir in Directory.EnumerateDirectories(paths.BackupsRoot))
|
||||
{
|
||||
if (!BackupStorageSafety.IsSafePath(paths.BackupsRoot, dir)) continue;
|
||||
var name = Path.GetFileName(dir);
|
||||
if (name.StartsWith(TempPrefix, StringComparison.Ordinal) || name.StartsWith('.'))
|
||||
{
|
||||
@@ -273,14 +302,36 @@ public sealed class PostgresBackupService(
|
||||
private static BackupManifest? ReadManifest(string backupDirectory)
|
||||
{
|
||||
var manifestPath = Path.Combine(backupDirectory, "manifest.json");
|
||||
if (!File.Exists(manifestPath))
|
||||
if (!BackupStorageSafety.IsSafePath(backupDirectory, manifestPath) || !File.Exists(manifestPath))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
return JsonSerializer.Deserialize<BackupManifest>(File.ReadAllText(manifestPath));
|
||||
if (new FileInfo(manifestPath).Length > 16_777_216) return null;
|
||||
var manifest = JsonSerializer.Deserialize<BackupManifest>(File.ReadAllText(manifestPath));
|
||||
if (manifest is null || manifest.BackupId != Path.GetFileName(backupDirectory)
|
||||
|| manifest.Tables is null || manifest.Attachments is null)
|
||||
return null;
|
||||
|
||||
try
|
||||
{
|
||||
PostgresIdentifierSafety.ValidateUniqueTableNames(manifest.Tables.Select(table => table?.TableName));
|
||||
}
|
||||
catch (InvalidOperationException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (manifest.Tables.Any(t => t is null || string.IsNullOrWhiteSpace(t.FileName)
|
||||
|| Path.GetFileName(t.FileName) != t.FileName
|
||||
|| !BackupStorageSafety.IsSafePath(backupDirectory, Path.Combine(backupDirectory, "db", t.FileName)))
|
||||
|| manifest.Attachments.Any(a => a is null || string.IsNullOrWhiteSpace(a.RelativePath)
|
||||
|| Path.IsPathRooted(a.RelativePath)
|
||||
|| !BackupStorageSafety.IsSafePath(Path.Combine(backupDirectory, "attachments"),
|
||||
Path.Combine(backupDirectory, "attachments", a.RelativePath)))) return null;
|
||||
return manifest;
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
@@ -293,11 +344,11 @@ public sealed class PostgresBackupService(
|
||||
&& backupId.Length <= 100
|
||||
&& backupId.All(c => char.IsAsciiLetterOrDigit(c) || c is '-' or '_');
|
||||
|
||||
private static void TryDeleteDirectory(string path)
|
||||
private void TryDeleteDirectory(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(path))
|
||||
if (BackupStorageSafety.IsSafePath(paths.BackupsRoot, path) && Directory.Exists(path))
|
||||
{
|
||||
Directory.Delete(path, recursive: true);
|
||||
}
|
||||
@@ -310,13 +361,12 @@ public sealed class PostgresBackupService(
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IAsyncDisposable> AcquireLeaseAsync(CancellationToken cancellationToken)
|
||||
private Task<IAsyncDisposable> AcquireLeaseAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var lockPath = Path.Combine(paths.BackupsRoot, LockFileName);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
try
|
||||
{
|
||||
var stream = new FileStream(lockPath, FileMode.Create, FileAccess.Write, FileShare.None);
|
||||
return new LeaseHandle(stream);
|
||||
return Task.FromResult<IAsyncDisposable>(BackupStorageSafety.AcquireLease(paths.BackupsRoot));
|
||||
}
|
||||
catch (IOException exception)
|
||||
{
|
||||
@@ -325,20 +375,4 @@ public sealed class PostgresBackupService(
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class LeaseHandle(FileStream stream) : IAsyncDisposable
|
||||
{
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await stream.DisposeAsync();
|
||||
try
|
||||
{
|
||||
File.Delete(stream.Name);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
// Non-fatal: the next backup attempt will overwrite this lock file via
|
||||
// FileMode.Create regardless.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+346
-43
@@ -1,4 +1,5 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text.Json;
|
||||
using ItWorx.KnowledgeHub.Application.Operations;
|
||||
using Npgsql;
|
||||
|
||||
@@ -13,28 +14,25 @@ namespace ItWorx.KnowledgeHub.Infrastructure.Operations;
|
||||
/// afterwards so the next ordinary insert does not collide with a restored row
|
||||
/// (ADR-016 decision 2).
|
||||
/// </summary>
|
||||
public sealed class PostgresRestoreService(BackupPaths paths, IBackupService backups) : IRestoreService
|
||||
public sealed class PostgresRestoreService(BackupPaths paths, IBackupEvidenceStore evidence) : IRestoreService
|
||||
{
|
||||
private const long MaximumManifestBytes = 16 * 1024 * 1024;
|
||||
private const string SnapshotDirectoryPrefix = ".ikh-restore-";
|
||||
|
||||
public async Task<RestoreReport> RestoreAsync(string backupId, RestoreTarget target, CancellationToken cancellationToken)
|
||||
{
|
||||
var manifest = await backups.GetManifestAsync(backupId, cancellationToken)
|
||||
?? throw new InvalidOperationException($"Back-up '{backupId}' bestaat niet of het manifest is onleesbaar.");
|
||||
|
||||
var backupDir = Path.Combine(paths.BackupsRoot, backupId);
|
||||
var dbDir = Path.Combine(backupDir, "db");
|
||||
var attachmentsDir = Path.Combine(backupDir, "attachments");
|
||||
|
||||
// Verify every source file's hash before touching the target at all —
|
||||
// a corrupted backup must never partially restore.
|
||||
foreach (var table in manifest.Tables)
|
||||
{
|
||||
await VerifyHashAsync(Path.Combine(dbDir, table.FileName), table.Sha256, cancellationToken);
|
||||
}
|
||||
|
||||
foreach (var attachment in manifest.Attachments)
|
||||
{
|
||||
await VerifyHashAsync(Path.Combine(attachmentsDir, attachment.RelativePath), attachment.Sha256, cancellationToken);
|
||||
}
|
||||
if (!BackupStorageSafety.SafeId(backupId)) throw new InvalidOperationException("Ongeldig back-up-ID.");
|
||||
var targetAttachmentsDirectory = ValidateEmptyTargetAttachmentsDirectory(target.AttachmentsDirectory);
|
||||
await using var lease = BackupStorageSafety.AcquireLease(paths.BackupsRoot);
|
||||
var backupDirectory = Path.Combine(paths.BackupsRoot, backupId);
|
||||
var manifestPath = Path.Combine(backupDirectory, "manifest.json");
|
||||
if (!BackupStorageSafety.IsSafePath(paths.BackupsRoot, backupDirectory)
|
||||
|| !BackupStorageSafety.IsSafePath(backupDirectory, manifestPath))
|
||||
throw new InvalidOperationException("Onveilige back-upopslag.");
|
||||
await ValidateManifestIdentifiersAsync(manifestPath, cancellationToken);
|
||||
await using var snapshot = await CreateVerifiedSnapshotAsync(
|
||||
backupId, backupDirectory, manifestPath, targetAttachmentsDirectory, cancellationToken);
|
||||
var manifest = snapshot.Manifest;
|
||||
|
||||
var warnings = new List<string>();
|
||||
var tableReports = new List<RestoreTableReport>();
|
||||
@@ -42,6 +40,7 @@ public sealed class PostgresRestoreService(BackupPaths paths, IBackupService bac
|
||||
await using (var connection = new NpgsqlConnection(target.ConnectionString))
|
||||
{
|
||||
await connection.OpenAsync(cancellationToken);
|
||||
await EnsureTargetSchemaMatchesAsync(connection, manifest.SchemaVersion, cancellationToken);
|
||||
await EnsureTargetIsCleanAsync(connection, cancellationToken);
|
||||
|
||||
await using var transaction = await connection.BeginTransactionAsync(cancellationToken);
|
||||
@@ -57,7 +56,8 @@ public sealed class PostgresRestoreService(BackupPaths paths, IBackupService bac
|
||||
await using (var rawCopy = await connection.BeginRawBinaryCopyAsync(
|
||||
$"COPY \"public\".\"{table.TableName}\" FROM STDIN (FORMAT BINARY)", cancellationToken))
|
||||
await using (var source = new FileStream(
|
||||
Path.Combine(dbDir, table.FileName), FileMode.Open, FileAccess.Read, FileShare.Read))
|
||||
SafeRelativePath(snapshot.DatabaseDirectory, table.FileName, flatOnly: true),
|
||||
FileMode.Open, FileAccess.Read, FileShare.Read))
|
||||
{
|
||||
await source.CopyToAsync(rawCopy, cancellationToken);
|
||||
}
|
||||
@@ -79,7 +79,8 @@ public sealed class PostgresRestoreService(BackupPaths paths, IBackupService bac
|
||||
await transaction.CommitAsync(cancellationToken);
|
||||
}
|
||||
|
||||
var attachmentsRestored = await RestoreAttachmentsAsync(attachmentsDir, manifest, target.AttachmentsDirectory, cancellationToken);
|
||||
var attachmentsRestored = await RestoreAttachmentsAsync(
|
||||
snapshot.AttachmentsDirectory, manifest, target.AttachmentsDirectory, cancellationToken);
|
||||
|
||||
foreach (var mismatch in tableReports.Where(report => !report.Matches))
|
||||
{
|
||||
@@ -88,7 +89,277 @@ public sealed class PostgresRestoreService(BackupPaths paths, IBackupService bac
|
||||
|
||||
var success = tableReports.All(report => report.Matches) && attachmentsRestored == manifest.Attachments.Count;
|
||||
|
||||
return new RestoreReport(backupId, success, tableReports, attachmentsRestored, manifest.Attachments.Count, warnings);
|
||||
var report = new RestoreReport(
|
||||
backupId, success, tableReports, attachmentsRestored, manifest.Attachments.Count, warnings, snapshot.ManifestSha256);
|
||||
try
|
||||
{
|
||||
await evidence.RecordRestoreAsync(backupId, report, cancellationToken);
|
||||
}
|
||||
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
// The restore can finish even if receipt persistence fails after the
|
||||
// storage lease was acquired. The UI must then remain unverified.
|
||||
warnings.Add("Herstel voltooid, maar het herstelbewijs kon niet worden opgeslagen.");
|
||||
}
|
||||
return report;
|
||||
}
|
||||
|
||||
private async Task<VerifiedBackupSnapshot> CreateVerifiedSnapshotAsync(
|
||||
string backupId, string backupDirectory, string manifestPath,
|
||||
string targetAttachmentsDirectory, CancellationToken cancellationToken)
|
||||
{
|
||||
// Stage on the operator-provisioned target attachments volume. This keeps
|
||||
// large restores off the container overlay and makes capacity predictable.
|
||||
var stagingParent = targetAttachmentsDirectory;
|
||||
var snapshotRoot = Path.Combine(stagingParent, SnapshotDirectoryPrefix + Guid.NewGuid().ToString("N"));
|
||||
if (Directory.Exists(snapshotRoot) || File.Exists(snapshotRoot))
|
||||
throw new IOException("Er kon geen privé-herstelmap worden gemaakt.");
|
||||
|
||||
CreatePrivateDirectory(snapshotRoot);
|
||||
try
|
||||
{
|
||||
var sourceManifestBytes = await ReadBoundedFileAsync(manifestPath, cancellationToken);
|
||||
var manifestHash = HashBytes(sourceManifestBytes);
|
||||
var stagedManifestPath = Path.Combine(snapshotRoot, "manifest.json");
|
||||
await WritePrivateFileAsync(stagedManifestPath, sourceManifestBytes, cancellationToken);
|
||||
|
||||
var stagedManifestBytes = await ReadBoundedFileAsync(stagedManifestPath, cancellationToken);
|
||||
if (!string.Equals(HashBytes(stagedManifestBytes), manifestHash, StringComparison.Ordinal))
|
||||
throw new InvalidOperationException("Het herstelmanifest kon niet veilig worden vastgezet.");
|
||||
|
||||
BackupManifest manifest;
|
||||
try
|
||||
{
|
||||
manifest = JsonSerializer.Deserialize<BackupManifest>(stagedManifestBytes)
|
||||
?? throw new InvalidOperationException("Het herstelmanifest is ongeldig.");
|
||||
}
|
||||
catch (JsonException exception)
|
||||
{
|
||||
throw new InvalidOperationException("Het herstelmanifest is ongeldig.", exception);
|
||||
}
|
||||
|
||||
// Bind the evidence lookup to the exact bytes that were staged and parsed.
|
||||
// A source-volume writer cannot swap in a different valid manifest between
|
||||
// the hash check and evidence lookup and make this snapshot trusted.
|
||||
var sourceEvidence = await evidence.GetAsync(backupId, cancellationToken, manifestHash);
|
||||
if (sourceEvidence?.IntegrityVerifiedAt is null
|
||||
|| manifest.BackupId != backupId
|
||||
|| manifest.Tables is null
|
||||
|| manifest.Attachments is null
|
||||
|| manifest.Tables.Count == 0)
|
||||
throw new InvalidOperationException("Back-upintegriteit is niet bevestigd; herstel is geweigerd.");
|
||||
|
||||
// A valid HMAC proves these exact bytes came from a backup, but catalog
|
||||
// identifiers still must be safe before they are used as SQL identifiers.
|
||||
// Reject malformed and duplicate entries before connecting to the target.
|
||||
PostgresIdentifierSafety.ValidateUniqueTableNames(manifest.Tables.Select(table => table?.TableName));
|
||||
|
||||
var sourceDatabaseDirectory = Path.Combine(backupDirectory, "db");
|
||||
var sourceAttachmentsDirectory = Path.Combine(backupDirectory, "attachments");
|
||||
if (!BackupStorageSafety.IsSafePath(backupDirectory, sourceDatabaseDirectory)
|
||||
|| !BackupStorageSafety.IsSafePath(backupDirectory, sourceAttachmentsDirectory))
|
||||
throw new InvalidOperationException("Onveilige back-upopslag.");
|
||||
|
||||
var stagedDatabaseDirectory = Path.Combine(snapshotRoot, "db");
|
||||
var stagedAttachmentsDirectory = Path.Combine(snapshotRoot, "attachments");
|
||||
CreatePrivateDirectory(stagedDatabaseDirectory);
|
||||
CreatePrivateDirectory(stagedAttachmentsDirectory);
|
||||
|
||||
foreach (var table in manifest.Tables)
|
||||
{
|
||||
if (table is null || table.SizeBytes < 0)
|
||||
throw new InvalidOperationException("Het herstelmanifest bevat ongeldige tabelmetadata.");
|
||||
var sourcePath = SafeRelativePath(sourceDatabaseDirectory, table.FileName, flatOnly: true);
|
||||
var stagedPath = SafeRelativePath(stagedDatabaseDirectory, table.FileName, flatOnly: true);
|
||||
await CopyAndVerifyAsync(sourcePath, stagedPath, table.SizeBytes, table.Sha256, cancellationToken);
|
||||
}
|
||||
|
||||
foreach (var attachment in manifest.Attachments)
|
||||
{
|
||||
if (attachment is null || attachment.SizeBytes < 0)
|
||||
throw new InvalidOperationException("Het herstelmanifest bevat ongeldige bijlagemetadata.");
|
||||
if (IsReservedStagingPath(attachment.RelativePath))
|
||||
throw new InvalidOperationException("Het herstelmanifest gebruikt een gereserveerd bijlagepad.");
|
||||
var sourcePath = SafeRelativePath(sourceAttachmentsDirectory, attachment.RelativePath);
|
||||
var stagedPath = SafeRelativePath(stagedAttachmentsDirectory, attachment.RelativePath);
|
||||
var stagedParent = Path.GetDirectoryName(stagedPath)!;
|
||||
CreatePrivateDirectory(stagedParent);
|
||||
await CopyAndVerifyAsync(sourcePath, stagedPath, attachment.SizeBytes, attachment.Sha256, cancellationToken);
|
||||
}
|
||||
|
||||
return new VerifiedBackupSnapshot(snapshotRoot, stagedDatabaseDirectory,
|
||||
stagedAttachmentsDirectory, manifest, manifestHash, stagingParent);
|
||||
}
|
||||
catch
|
||||
{
|
||||
DeleteSnapshotDirectory(snapshotRoot, stagingParent);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task ValidateManifestIdentifiersAsync(string manifestPath, CancellationToken cancellationToken)
|
||||
{
|
||||
// Reject malformed catalog names before creating even a temporary staging
|
||||
// directory under the target attachments root. The exact staged bytes are
|
||||
// validated again after their HMAC is checked to close source-volume races.
|
||||
var manifestBytes = await ReadBoundedFileAsync(manifestPath, cancellationToken);
|
||||
BackupManifest manifest;
|
||||
try
|
||||
{
|
||||
manifest = JsonSerializer.Deserialize<BackupManifest>(manifestBytes)
|
||||
?? throw new InvalidOperationException("Het herstelmanifest is ongeldig.");
|
||||
}
|
||||
catch (JsonException exception)
|
||||
{
|
||||
throw new InvalidOperationException("Het herstelmanifest is ongeldig.", exception);
|
||||
}
|
||||
|
||||
if (manifest.Tables is null || manifest.Tables.Count == 0)
|
||||
throw new InvalidOperationException("Het herstelmanifest bevat geen tabellen.");
|
||||
PostgresIdentifierSafety.ValidateUniqueTableNames(manifest.Tables.Select(table => table?.TableName));
|
||||
}
|
||||
|
||||
private static async Task<byte[]> ReadBoundedFileAsync(string path, CancellationToken cancellationToken)
|
||||
{
|
||||
await using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
if (stream.Length is <= 0 or > MaximumManifestBytes)
|
||||
throw new InvalidOperationException("Het herstelmanifest ontbreekt of is te groot.");
|
||||
|
||||
var bytes = new byte[checked((int)stream.Length)];
|
||||
await stream.ReadExactlyAsync(bytes, cancellationToken);
|
||||
var extra = new byte[1];
|
||||
if (await stream.ReadAsync(extra, cancellationToken) != 0)
|
||||
throw new InvalidOperationException("Het herstelmanifest is gewijzigd tijdens het lezen.");
|
||||
return bytes;
|
||||
}
|
||||
|
||||
private static async Task CopyAndVerifyAsync(
|
||||
string sourcePath, string destinationPath, long expectedSize, string expectedSha256,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (expectedSha256 is not { Length: 64 } || !expectedSha256.All(Uri.IsHexDigit))
|
||||
throw new InvalidOperationException("Het herstelmanifest bevat een ongeldige bestandshash.");
|
||||
|
||||
await using var source = new FileStream(sourcePath, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
await using var destination = CreatePrivateFile(destinationPath);
|
||||
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
|
||||
var buffer = new byte[81920];
|
||||
long total = 0;
|
||||
int read;
|
||||
while ((read = await source.ReadAsync(buffer, cancellationToken)) > 0)
|
||||
{
|
||||
total = checked(total + read);
|
||||
if (total > expectedSize)
|
||||
throw new InvalidOperationException($"Back-upverificatie mislukt voor bestand '{Path.GetFileName(sourcePath)}'.");
|
||||
hash.AppendData(buffer, 0, read);
|
||||
await destination.WriteAsync(buffer.AsMemory(0, read), cancellationToken);
|
||||
}
|
||||
|
||||
await destination.FlushAsync(cancellationToken);
|
||||
RestrictPrivateFile(destinationPath);
|
||||
if (total != expectedSize
|
||||
|| !string.Equals(Convert.ToHexStringLower(hash.GetHashAndReset()), expectedSha256, StringComparison.Ordinal))
|
||||
throw new InvalidOperationException($"Back-upverificatie mislukt voor bestand '{Path.GetFileName(sourcePath)}'.");
|
||||
}
|
||||
|
||||
private static async Task WritePrivateFileAsync(string path, byte[] contents, CancellationToken cancellationToken)
|
||||
{
|
||||
await using (var destination = CreatePrivateFile(path))
|
||||
{
|
||||
await destination.WriteAsync(contents, cancellationToken);
|
||||
await destination.FlushAsync(cancellationToken);
|
||||
}
|
||||
RestrictPrivateFile(path);
|
||||
}
|
||||
|
||||
private static FileStream CreatePrivateFile(string path)
|
||||
{
|
||||
var options = new FileStreamOptions
|
||||
{
|
||||
Mode = FileMode.CreateNew,
|
||||
Access = FileAccess.Write,
|
||||
Share = FileShare.None,
|
||||
Options = FileOptions.Asynchronous | FileOptions.SequentialScan
|
||||
};
|
||||
if (!OperatingSystem.IsWindows())
|
||||
options.UnixCreateMode = UnixFileMode.UserRead | UnixFileMode.UserWrite;
|
||||
return new FileStream(path, options);
|
||||
}
|
||||
|
||||
private static void CreatePrivateDirectory(string path)
|
||||
{
|
||||
const UnixFileMode mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute;
|
||||
if (OperatingSystem.IsWindows())
|
||||
{
|
||||
Directory.CreateDirectory(path);
|
||||
}
|
||||
else
|
||||
{
|
||||
Directory.CreateDirectory(path, mode);
|
||||
File.SetUnixFileMode(path, mode);
|
||||
if (File.GetUnixFileMode(path) != mode)
|
||||
throw new IOException("De privé-herstelmap kan niet tot Unix-modus 0700 worden beperkt.");
|
||||
}
|
||||
|
||||
if ((File.GetAttributes(path) & FileAttributes.ReparsePoint) != 0)
|
||||
throw new IOException("De privé-herstelmap mag geen symbolische koppeling zijn.");
|
||||
}
|
||||
|
||||
private static void RestrictPrivateFile(string path)
|
||||
{
|
||||
if (!OperatingSystem.IsWindows())
|
||||
{
|
||||
const UnixFileMode mode = UnixFileMode.UserRead | UnixFileMode.UserWrite;
|
||||
File.SetUnixFileMode(path, mode);
|
||||
if (File.GetUnixFileMode(path) != mode)
|
||||
throw new IOException("Een herstelbestand kan niet tot Unix-modus 0600 worden beperkt.");
|
||||
}
|
||||
}
|
||||
|
||||
private static string SafeRelativePath(string root, string relativePath, bool flatOnly = false)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(relativePath) || Path.IsPathRooted(relativePath))
|
||||
throw new InvalidOperationException("Het herstelmanifest bevat een onveilig bestandspad.");
|
||||
var segments = relativePath.Replace('\\', '/').Split('/');
|
||||
if (segments.Any(segment => segment.Length == 0 || segment is "." or "..")
|
||||
|| (flatOnly && segments.Length != 1))
|
||||
throw new InvalidOperationException("Het herstelmanifest bevat een onveilig bestandspad.");
|
||||
var candidate = Path.Combine([root, .. segments]);
|
||||
if (!BackupStorageSafety.IsSafePath(root, candidate))
|
||||
throw new InvalidOperationException("Het herstelmanifest bevat een onveilig bestandspad.");
|
||||
return candidate;
|
||||
}
|
||||
|
||||
private static string ValidateEmptyTargetAttachmentsDirectory(string targetAttachmentsDirectory)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(targetAttachmentsDirectory))
|
||||
throw new InvalidOperationException("Een doelmap voor bijlagen is vereist.");
|
||||
var fullPath = Path.GetFullPath(targetAttachmentsDirectory);
|
||||
if (!Directory.Exists(fullPath) || !BackupStorageSafety.IsSafePath(fullPath, fullPath))
|
||||
throw new InvalidOperationException("De doelmap voor bijlagen moet een bestaande, veilige map zijn.");
|
||||
if (Directory.EnumerateFileSystemEntries(fullPath).Any())
|
||||
throw new InvalidOperationException("De doelmap voor bijlagen is niet leeg — herstel vereist een schone doelomgeving.");
|
||||
return fullPath;
|
||||
}
|
||||
|
||||
private static bool IsReservedStagingPath(string relativePath)
|
||||
{
|
||||
var firstSegment = relativePath.Replace('\\', '/').Split('/', 2)[0];
|
||||
return firstSegment.StartsWith(SnapshotDirectoryPrefix, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static string HashBytes(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes));
|
||||
|
||||
private static void DeleteSnapshotDirectory(string snapshotRoot, string stagingParent)
|
||||
{
|
||||
var fullSnapshotRoot = Path.GetFullPath(snapshotRoot);
|
||||
var fullStagingParent = Path.GetFullPath(stagingParent);
|
||||
var comparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
|
||||
if (!Path.GetFileName(fullSnapshotRoot).StartsWith(SnapshotDirectoryPrefix, StringComparison.Ordinal)
|
||||
|| !string.Equals(Path.GetDirectoryName(fullSnapshotRoot), fullStagingParent, comparison)
|
||||
|| !BackupStorageSafety.IsSafePath(fullStagingParent, fullSnapshotRoot))
|
||||
throw new IOException("De privé-herstelmap is gewijzigd; veilige opschoning is geweigerd.");
|
||||
if (Directory.Exists(fullSnapshotRoot)) Directory.Delete(fullSnapshotRoot, recursive: true);
|
||||
}
|
||||
|
||||
private static async Task EnsureTargetIsCleanAsync(NpgsqlConnection connection, CancellationToken cancellationToken)
|
||||
@@ -106,15 +377,30 @@ public sealed class PostgresRestoreService(BackupPaths paths, IBackupService bac
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task EnsureTargetSchemaMatchesAsync(
|
||||
NpgsqlConnection connection, string expectedSchema, CancellationToken cancellationToken)
|
||||
{
|
||||
const string sql = """
|
||||
SELECT migration_id FROM "public"."__EFMigrationsHistory"
|
||||
ORDER BY migration_id DESC LIMIT 1
|
||||
""";
|
||||
await using var command = new NpgsqlCommand(sql, connection);
|
||||
var actualSchema = (string?)await command.ExecuteScalarAsync(cancellationToken);
|
||||
if (string.IsNullOrWhiteSpace(expectedSchema)
|
||||
|| !string.Equals(expectedSchema, actualSchema, StringComparison.Ordinal))
|
||||
throw new InvalidOperationException("Het doelschema komt niet overeen met de back-up; herstel eerst met de bijbehorende applicatieversie.");
|
||||
}
|
||||
|
||||
private static async Task ResetIdentitySequencesAsync(
|
||||
NpgsqlConnection connection, NpgsqlTransaction transaction, CancellationToken cancellationToken)
|
||||
{
|
||||
var identityColumns = await TableDiscovery.DiscoverIdentityColumnsAsync(connection, cancellationToken);
|
||||
foreach (var (table, column) in identityColumns)
|
||||
{
|
||||
// Table/column names come only from information_schema (never user input),
|
||||
// so identifier interpolation here is safe; pg_get_serial_sequence itself
|
||||
// takes plain string literals, not bindable identifier parameters.
|
||||
// Discovery validates catalog names, and repeat the check at the SQL
|
||||
// boundary so every sequence reset remains safe if its caller changes.
|
||||
PostgresIdentifierSafety.RequireSafe(table);
|
||||
PostgresIdentifierSafety.RequireSafe(column);
|
||||
var sql = $"""
|
||||
SELECT setval(
|
||||
pg_get_serial_sequence('public.{table}', '{column}'),
|
||||
@@ -133,15 +419,16 @@ public sealed class PostgresRestoreService(BackupPaths paths, IBackupService bac
|
||||
var restored = 0;
|
||||
foreach (var attachment in manifest.Attachments)
|
||||
{
|
||||
var sourcePath = Path.Combine(backupAttachmentsDir, attachment.RelativePath);
|
||||
var destinationPath = Path.Combine(targetAttachmentsDirectory, attachment.RelativePath);
|
||||
var sourcePath = SafeRelativePath(backupAttachmentsDir, attachment.RelativePath);
|
||||
var destinationPath = SafeRelativePath(targetAttachmentsDirectory, attachment.RelativePath);
|
||||
if (!BackupStorageSafety.IsSafePath(targetAttachmentsDirectory, destinationPath))
|
||||
throw new InvalidOperationException("Onveilige doelopslag voor bijlagen.");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!);
|
||||
await using (var source = new FileStream(sourcePath, FileMode.Open, FileAccess.Read, FileShare.Read))
|
||||
await using (var destination = new FileStream(destinationPath, FileMode.Create, FileAccess.Write, FileShare.None))
|
||||
{
|
||||
await source.CopyToAsync(destination, cancellationToken);
|
||||
}
|
||||
// The verified snapshot is inside this target volume, so moving the
|
||||
// staged blob avoids a second full-size copy at peak restore time.
|
||||
File.Move(sourcePath, destinationPath);
|
||||
|
||||
await VerifyHashAsync(destinationPath, attachment.Sha256, cancellationToken);
|
||||
restored++;
|
||||
}
|
||||
|
||||
@@ -155,20 +442,36 @@ public sealed class PostgresRestoreService(BackupPaths paths, IBackupService bac
|
||||
throw new InvalidOperationException($"Back-upbestand ontbreekt: '{Path.GetFileName(path)}'.");
|
||||
}
|
||||
|
||||
await using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
using var sha256 = SHA256.Create();
|
||||
var buffer = new byte[81920];
|
||||
int read;
|
||||
while ((read = await stream.ReadAsync(buffer, cancellationToken)) > 0)
|
||||
{
|
||||
sha256.TransformBlock(buffer, 0, read, null, 0);
|
||||
}
|
||||
|
||||
sha256.TransformFinalBlock([], 0, 0);
|
||||
var actual = Convert.ToHexString(sha256.Hash!).ToLowerInvariant();
|
||||
var actual = await HashAsync(path, cancellationToken);
|
||||
if (!string.Equals(actual, expectedSha256, StringComparison.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException($"Back-upverificatie mislukt voor bestand '{Path.GetFileName(path)}': hash komt niet overeen.");
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<string> HashAsync(string path, CancellationToken cancellationToken)
|
||||
{
|
||||
await using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
return Convert.ToHexStringLower(await SHA256.HashDataAsync(stream, cancellationToken));
|
||||
}
|
||||
|
||||
private sealed class VerifiedBackupSnapshot(
|
||||
string rootDirectory,
|
||||
string databaseDirectory,
|
||||
string attachmentsDirectory,
|
||||
BackupManifest manifest,
|
||||
string manifestSha256,
|
||||
string stagingParent) : IAsyncDisposable
|
||||
{
|
||||
public string DatabaseDirectory { get; } = databaseDirectory;
|
||||
public string AttachmentsDirectory { get; } = attachmentsDirectory;
|
||||
public BackupManifest Manifest { get; } = manifest;
|
||||
public string ManifestSha256 { get; } = manifestSha256;
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
DeleteSnapshotDirectory(rootDirectory, stagingParent);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -29,9 +29,13 @@ internal static class TableDiscovery
|
||||
await using var reader = await command.ExecuteReaderAsync(cancellationToken);
|
||||
|
||||
var tables = new List<string>();
|
||||
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||||
while (await reader.ReadAsync(cancellationToken))
|
||||
{
|
||||
tables.Add(reader.GetString(0));
|
||||
var table = PostgresIdentifierSafety.RequireSafe(reader.GetString(0));
|
||||
if (!seen.Add(table))
|
||||
throw new InvalidOperationException("De database bevat dubbele tabelnamen.");
|
||||
tables.Add(table);
|
||||
}
|
||||
|
||||
return tables;
|
||||
@@ -55,11 +59,47 @@ internal static class TableDiscovery
|
||||
await using var reader = await command.ExecuteReaderAsync(cancellationToken);
|
||||
|
||||
var columns = new List<(string, string)>();
|
||||
var seen = new HashSet<(string Table, string Column)>();
|
||||
while (await reader.ReadAsync(cancellationToken))
|
||||
{
|
||||
columns.Add((reader.GetString(0), reader.GetString(1)));
|
||||
var table = PostgresIdentifierSafety.RequireSafe(reader.GetString(0));
|
||||
var column = PostgresIdentifierSafety.RequireSafe(reader.GetString(1));
|
||||
if (!seen.Add((table, column)))
|
||||
throw new InvalidOperationException("De database bevat dubbele identiteitskolommen.");
|
||||
columns.Add((table, column));
|
||||
}
|
||||
|
||||
return columns;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The application schema uses PostgreSQL's ordinary ASCII snake-case identifiers.
|
||||
/// Restrict names read from the catalog and backup manifests before they are inserted
|
||||
/// into SQL, rather than trusting a quoted value or a signed manifest to be a valid
|
||||
/// SQL identifier.
|
||||
/// </summary>
|
||||
internal static class PostgresIdentifierSafety
|
||||
{
|
||||
internal static string RequireSafe(string? identifier)
|
||||
{
|
||||
if (identifier is not { Length: > 0 and <= 63 }
|
||||
|| !identifier.All(character => char.IsAsciiLetterOrDigit(character) || character == '_'))
|
||||
{
|
||||
throw new InvalidOperationException("De back-up bevat een onveilige PostgreSQL-identificatie.");
|
||||
}
|
||||
|
||||
return identifier;
|
||||
}
|
||||
|
||||
internal static void ValidateUniqueTableNames(IEnumerable<string?> tableNames)
|
||||
{
|
||||
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||||
foreach (var tableName in tableNames)
|
||||
{
|
||||
var safeName = RequireSafe(tableName);
|
||||
if (!seen.Add(safeName))
|
||||
throw new InvalidOperationException("Het back-upmanifest bevat dubbele tabelnamen.");
|
||||
}
|
||||
}
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
using ItWorx.KnowledgeHub.Domain.Identity;
|
||||
using ItWorx.KnowledgeHub.Domain.Jobs;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Metadata.Builders;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Infrastructure.Persistence.Configurations;
|
||||
|
||||
public sealed class BackgroundJobConfiguration : IEntityTypeConfiguration<BackgroundJob>
|
||||
{
|
||||
public void Configure(EntityTypeBuilder<BackgroundJob> builder)
|
||||
{
|
||||
builder.ToTable("background_jobs", table => table.HasCheckConstraint(
|
||||
"ck_background_jobs_counts",
|
||||
"total >= 0 AND processed >= 0 AND succeeded >= 0 AND skipped >= 0 AND failed >= 0 AND processed <= total"));
|
||||
builder.HasKey(job => job.Id);
|
||||
builder.Property(job => job.Type).HasConversion<string>().HasMaxLength(32).IsRequired();
|
||||
builder.Property(job => job.Status).HasConversion<string>().HasMaxLength(32).IsRequired().IsConcurrencyToken();
|
||||
builder.Property(job => job.LeaseToken).IsConcurrencyToken();
|
||||
builder.Property(job => job.PayloadJson).HasColumnType("jsonb").IsRequired();
|
||||
builder.Property(job => job.ResultJson).HasColumnType("jsonb");
|
||||
builder.Property(job => job.ErrorsJson).HasColumnType("jsonb").IsRequired();
|
||||
builder.Property(job => job.IdempotencyKeyHash).HasMaxLength(64).IsRequired();
|
||||
builder.Property(job => job.PayloadHash).HasMaxLength(64).IsRequired();
|
||||
builder.Property(job => job.CorrelationId).HasMaxLength(128);
|
||||
builder.Property(job => job.AttemptCount).HasDefaultValue(0);
|
||||
builder.Property(job => job.Version).HasDefaultValue(1L).IsConcurrencyToken();
|
||||
builder.HasOne<Workspace>().WithMany().HasForeignKey(job => job.WorkspaceId).OnDelete(DeleteBehavior.Restrict);
|
||||
builder.HasOne<ApplicationUser>().WithMany().HasForeignKey(job => job.ActorId).OnDelete(DeleteBehavior.Restrict);
|
||||
builder.HasIndex(job => new { job.WorkspaceId, job.ActorId, job.IdempotencyKeyHash }).IsUnique()
|
||||
.HasDatabaseName("ux_background_jobs_idempotency");
|
||||
builder.HasIndex(job => new { job.WorkspaceId, job.CreatedAt }).IsDescending(false, true)
|
||||
.HasDatabaseName("ix_background_jobs_workspace_created");
|
||||
builder.HasIndex(job => new { job.Status, job.NextAttemptAt, job.CreatedAt })
|
||||
.HasDatabaseName("ix_background_jobs_queue");
|
||||
builder.HasIndex(job => new { job.Status, job.LeaseUntil })
|
||||
.HasDatabaseName("ix_background_jobs_lease");
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class BackgroundJobItemConfiguration : IEntityTypeConfiguration<BackgroundJobItem>
|
||||
{
|
||||
public void Configure(EntityTypeBuilder<BackgroundJobItem> builder)
|
||||
{
|
||||
builder.ToTable("background_job_items");
|
||||
builder.HasKey(item => item.Ordinal);
|
||||
builder.Property(item => item.Ordinal).UseIdentityByDefaultColumn();
|
||||
builder.Property(item => item.State).HasConversion<string>().HasMaxLength(24).IsRequired();
|
||||
builder.HasOne<BackgroundJob>().WithMany().HasForeignKey(item => item.JobId).OnDelete(DeleteBehavior.Cascade);
|
||||
builder.HasOne<ItWorx.KnowledgeHub.Domain.Knowledge.Page>().WithMany().HasForeignKey(item => item.PageId).OnDelete(DeleteBehavior.Cascade);
|
||||
builder.HasIndex(item => new { item.JobId, item.PageId }).IsUnique()
|
||||
.HasDatabaseName("ux_background_job_items_page");
|
||||
builder.HasIndex(item => new { item.JobId, item.State, item.Ordinal })
|
||||
.HasDatabaseName("ix_background_job_items_queue");
|
||||
}
|
||||
}
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
using ItWorx.KnowledgeHub.Domain.Identity;
|
||||
using ItWorx.KnowledgeHub.Domain.Sources;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Metadata.Builders;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Infrastructure.Persistence.Configurations;
|
||||
|
||||
public sealed class DataSourceConfiguration : IEntityTypeConfiguration<DataSource>
|
||||
{
|
||||
public void Configure(EntityTypeBuilder<DataSource> builder)
|
||||
{
|
||||
builder.ToTable("data_sources");
|
||||
builder.HasKey(source => source.Id);
|
||||
builder.Property(source => source.Id).ValueGeneratedNever();
|
||||
builder.Property(source => source.SourceKey).HasMaxLength(200).IsRequired();
|
||||
builder.Property(source => source.DisplayName).HasMaxLength(120).IsRequired();
|
||||
builder.Property(source => source.Kind).HasConversion<string>().HasMaxLength(30);
|
||||
builder.Property(source => source.LastErrorCode).HasMaxLength(30);
|
||||
builder.Property(source => source.LastCorrelationId).HasMaxLength(100);
|
||||
builder.Property(source => source.Version).IsConcurrencyToken();
|
||||
builder.HasIndex(source => new { source.WorkspaceId, source.SourceKey }).IsUnique();
|
||||
builder.HasOne<Workspace>().WithMany().HasForeignKey(source => source.WorkspaceId).OnDelete(DeleteBehavior.Restrict);
|
||||
}
|
||||
}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
using ItWorx.KnowledgeHub.Domain.Platform;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Metadata.Builders;
|
||||
|
||||
namespace ItWorx.KnowledgeHub.Infrastructure.Persistence.Configurations;
|
||||
|
||||
public sealed class PlatformSettingsConfiguration : IEntityTypeConfiguration<PlatformSettings>
|
||||
{
|
||||
public void Configure(EntityTypeBuilder<PlatformSettings> builder)
|
||||
{
|
||||
builder.ToTable("platform_settings", table => table.HasCheckConstraint(
|
||||
"ck_platform_settings_singleton",
|
||||
"\"id\" = '00000000-0000-0000-0000-000000000002'"));
|
||||
builder.HasKey(settings => settings.Id);
|
||||
|
||||
builder.Property(settings => settings.Id).ValueGeneratedNever();
|
||||
builder.Property(settings => settings.DisplayName).HasMaxLength(100).IsRequired();
|
||||
builder.Property(settings => settings.PrimaryLocale).HasMaxLength(20).IsRequired();
|
||||
builder.Property(settings => settings.TimeZone).HasMaxLength(100).IsRequired();
|
||||
builder.Property(settings => settings.BackupScheduleDescription).HasMaxLength(200).IsRequired();
|
||||
builder.Property(settings => settings.Version).IsConcurrencyToken().IsRequired();
|
||||
builder.Property(settings => settings.UpdatedAt).IsRequired();
|
||||
|
||||
builder.ToTable(table =>
|
||||
{
|
||||
table.HasCheckConstraint("ck_platform_settings_source_count", "maximum_source_count BETWEEN 1 AND 20");
|
||||
table.HasCheckConstraint("ck_platform_settings_source_bytes", "maximum_source_bytes BETWEEN 1 AND 104857600");
|
||||
table.HasCheckConstraint("ck_platform_settings_retry_limit", "failed_retry_limit BETWEEN 1 AND 3");
|
||||
table.HasCheckConstraint("ck_platform_settings_backup_expectation", "backup_expected_interval_hours BETWEEN 1 AND 8760 AND backup_daily_retention_count BETWEEN 1 AND 365 AND backup_weekly_retention_count BETWEEN 1 AND 104 AND backup_monthly_retention_count BETWEEN 1 AND 120 AND protected_backup_count BETWEEN 1 AND 100 AND restore_verification_max_age_days BETWEEN 1 AND 365");
|
||||
table.HasCheckConstraint("ck_platform_settings_inventory_freshness", "inventory_warning_after_hours BETWEEN 1 AND 8760");
|
||||
});
|
||||
}
|
||||
}
|
||||
Loaded 100 of 218 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user