Initial curated RepoPulse source from 2524ccc3c1c27739996dccb2b7992989f88cd26d

This commit is contained in:
Public Source Publisher committed 2026-10-01 01:34:52 +02:00
commit 0adf8ef5d9
90 files changed
+14315

No files matched your search

+19
View File
@@ -0,0 +1,19 @@
FROM python:3.12-slim
WORKDIR /app
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
REPOPULSE_DB=/data/repopulse.db \
REPOPULSE_SCAN_ROOT=/scan
RUN apt-get update \
&& apt-get install -y --no-install-recommends git \
&& rm -rf /var/lib/apt/lists/*
COPY backend/pyproject.toml ./pyproject.toml
COPY backend/app ./app
RUN pip install --no-cache-dir .
VOLUME ["/data", "/scan"]
EXPOSE 8000
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
+1
View File
@@ -0,0 +1 @@
"""RepoPulse backend package."""
+264
View File
@@ -0,0 +1,264 @@
from __future__ import annotations
import os
import re
import shutil
import subprocess
from dataclasses import dataclass
from pathlib import Path
from threading import Lock, RLock
from urllib.parse import urlsplit
from .models import Repository
SAFE_REVISION = re.compile(r"^[0-9a-fA-F]{7,64}$")
SAFE_BRANCH = re.compile(r"^[A-Za-z0-9._/-]+$")
SAFE_JOB_ID = re.compile(r"^[A-Za-z0-9-]+$")
class AcquisitionError(RuntimeError):
def __init__(self, code: str, detail: str) -> None:
super().__init__(detail)
self.code = code
self.detail = detail
@dataclass(frozen=True)
class AcquiredSource:
path: Path
actual_revision: str
branch: str
class GitAcquirer:
"""Maintain bounded bare mirrors and scan detached exact-revision worktrees."""
def __init__(
self,
cache_root: str | Path | None = None,
token: str | None = None,
*,
timeout: int = 180,
allowed_clone_origin: str | None = None,
) -> None:
raw_root = cache_root or os.getenv("REPOPULSE_REPOSITORY_CACHE", "data/repos")
self.cache_root = Path(raw_root).expanduser().resolve()
self.token = token if token is not None else os.getenv("REPOPULSE_GITEA_TOKEN", "")
self.timeout = max(10, timeout)
configured_origin = allowed_clone_origin or os.getenv("REPOPULSE_GITEA_URL", "")
self.allowed_clone_origin = self._origin(configured_origin)
self.mirrors_root = self.cache_root / "mirrors"
self.worktrees_root = self.cache_root / "worktrees"
self.mirrors_root.mkdir(parents=True, exist_ok=True)
self.worktrees_root.mkdir(parents=True, exist_ok=True)
self._lock_guard = Lock()
self._repository_locks: dict[str, RLock] = {}
@staticmethod
def _origin(value: str) -> str | None:
if not value:
return None
try:
parsed = urlsplit(value)
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
return None
port = parsed.port or (443 if parsed.scheme == "https" else 80)
except ValueError:
return None
return f"{parsed.scheme.casefold()}://{parsed.hostname.casefold()}:{port}"
def _repository_lock(self, repository_id: str) -> RLock:
with self._lock_guard:
return self._repository_locks.setdefault(repository_id, RLock())
def _git_env(self) -> dict[str, str]:
env = os.environ.copy()
if self.token:
env.update(
{
"GIT_CONFIG_COUNT": "1",
"GIT_CONFIG_KEY_0": "http.extraHeader",
"GIT_CONFIG_VALUE_0": f"Authorization: token {self.token}",
}
)
return env
def _run(self, args: list[str]) -> str:
try:
result = subprocess.run(
args,
check=True,
capture_output=True,
text=True,
errors="replace",
timeout=self.timeout,
env=self._git_env(),
)
except subprocess.TimeoutExpired as exc:
raise AcquisitionError("timeout", "Git source acquisition timed out") from exc
except (OSError, subprocess.CalledProcessError) as exc:
# Git stderr may contain remote URLs or server-specific details; do
# not persist or expose it because the job API is user-visible.
raise AcquisitionError("git_failed", "Git source acquisition failed") from exc
return result.stdout.strip()
def _mirror_path(self, repository: Repository) -> Path:
identity = f"{repository.provider}-{repository.provider_repository_id}"
safe = re.sub(r"[^A-Za-z0-9._-]", "_", identity)
return self.mirrors_root / f"{safe}.git"
def _worktree_path(self, job_id: str) -> Path:
if not SAFE_JOB_ID.fullmatch(job_id):
raise AcquisitionError("invalid_job", "Scan job identity is invalid")
return self.worktrees_root / job_id
@staticmethod
def _validate_branch(branch: str) -> None:
if (
not SAFE_BRANCH.fullmatch(branch)
or branch.startswith("-")
or ".." in branch
or branch.endswith(("/", "."))
):
raise AcquisitionError("invalid_branch", "Repository default branch is invalid")
def _validate_clone_url(self, clone_url: str | None) -> str:
if not clone_url or "\x00" in clone_url:
raise AcquisitionError("missing_clone_url", "Repository clone URL is unavailable")
try:
parsed = urlsplit(clone_url)
if parsed.username or parsed.password or parsed.query or parsed.fragment:
raise AcquisitionError("unsafe_clone_url", "Repository clone URL contains unsafe credentials or metadata")
if parsed.scheme in {"http", "https"}:
origin = self._origin(clone_url)
if self.token and (self.allowed_clone_origin is None or origin != self.allowed_clone_origin):
raise AcquisitionError("unsafe_clone_url", "Repository clone URL is outside the configured Gitea origin")
elif parsed.scheme != "file":
raise AcquisitionError("unsafe_clone_url", "Repository clone URL uses an unsupported scheme")
except ValueError as exc:
raise AcquisitionError("unsafe_clone_url", "Repository clone URL is invalid") from exc
return clone_url
def acquire(
self,
repository: Repository,
job_id: str,
requested_revision: str | None = None,
) -> AcquiredSource:
with self._repository_lock(repository.id):
return self._acquire(repository, job_id, requested_revision)
def _acquire(
self,
repository: Repository,
job_id: str,
requested_revision: str | None = None,
) -> AcquiredSource:
branch = repository.default_branch or "main"
self._validate_branch(branch)
if requested_revision is not None and not SAFE_REVISION.fullmatch(requested_revision):
raise AcquisitionError("invalid_revision", "Requested revision is not a valid commit identifier")
clone_url = self._validate_clone_url(repository.clone_url)
mirror = self._mirror_path(repository)
if mirror.exists() and not self._mirror_is_usable(mirror):
self._remove_mirror_if_owned(mirror)
if not mirror.exists():
self._run(["git", "clone", "--mirror", clone_url, str(mirror)])
else:
self._run(["git", "--git-dir", str(mirror), "remote", "set-url", "origin", clone_url])
self._run(
[
"git",
"--git-dir",
str(mirror),
"fetch",
"--prune",
"--no-tags",
"origin",
f"+refs/heads/{branch}:refs/remotes/origin/{branch}",
]
)
if requested_revision:
try:
actual_revision = self._run(
["git", "--git-dir", str(mirror), "rev-parse", f"{requested_revision}^{{commit}}"]
)
except AcquisitionError as exc:
raise AcquisitionError("revision_unavailable", "Requested commit was not found in the mirror") from exc
if actual_revision.casefold() != requested_revision.casefold() and len(requested_revision) == 40:
raise AcquisitionError("revision_unavailable", "Requested commit was not found in the mirror")
else:
actual_revision = self._run(
[
"git",
"--git-dir",
str(mirror),
"rev-parse",
f"refs/remotes/origin/{branch}^{{commit}}",
]
)
if not SAFE_REVISION.fullmatch(actual_revision):
raise AcquisitionError("invalid_revision", "Git did not return a usable commit identifier")
worktree = self._worktree_path(job_id)
self._remove_worktree_if_owned(mirror, worktree)
try:
self._run(
[
"git",
"--git-dir",
str(mirror),
"worktree",
"add",
"--detach",
str(worktree),
actual_revision,
]
)
except AcquisitionError:
self._remove_worktree_if_owned(mirror, worktree)
raise
return AcquiredSource(path=worktree, actual_revision=actual_revision, branch=branch)
def cleanup(self, source: AcquiredSource, repository: Repository) -> None:
with self._repository_lock(repository.id):
mirror = self._mirror_path(repository)
self._remove_worktree_if_owned(mirror, source.path)
def _mirror_is_usable(self, mirror: Path) -> bool:
try:
return self._run(["git", "--git-dir", str(mirror), "rev-parse", "--is-bare-repository"]) == "true"
except AcquisitionError:
return False
def _remove_mirror_if_owned(self, mirror: Path) -> None:
try:
relative = mirror.resolve().relative_to(self.mirrors_root.resolve())
except (OSError, RuntimeError, ValueError):
raise AcquisitionError("unsafe_mirror", "Repository mirror path escaped the repository cache")
if not relative.parts:
raise AcquisitionError("unsafe_mirror", "Repository mirror path is invalid")
if mirror.is_dir() and not mirror.is_symlink():
shutil.rmtree(mirror)
elif mirror.exists() or mirror.is_symlink():
mirror.unlink()
def _remove_worktree_if_owned(self, mirror: Path, worktree: Path) -> None:
try:
relative = worktree.resolve().relative_to(self.worktrees_root.resolve())
except (OSError, RuntimeError, ValueError):
raise AcquisitionError("unsafe_worktree", "Worktree path escaped the repository cache")
if not relative.parts:
raise AcquisitionError("unsafe_worktree", "Worktree path is invalid")
if worktree.exists():
try:
self._run(["git", "--git-dir", str(mirror), "worktree", "remove", "--force", str(worktree)])
except AcquisitionError:
# A stale registration must not prevent a safe bounded cleanup.
pass
if worktree.exists():
shutil.rmtree(worktree)
try:
self._run(["git", "--git-dir", str(mirror), "worktree", "prune"])
except AcquisitionError:
pass
+340
View File
@@ -0,0 +1,340 @@
from __future__ import annotations
import ast
import json
import math
import os
import re
from collections import Counter
from pathlib import Path
from .models import ArchitectureEdge, ArchitectureGraph, ArchitectureNode, Hotspot
JS_IMPORT_RE = re.compile(
r"(?:\bfrom\s+|\bimport\s*|\brequire\(\s*|\bimport\(\s*)['\"](?P<target>[^'\"]+)['\"]"
)
JS_EXTENSIONS = (".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs")
SOURCE_ROOTS = {"src", "app", "backend", "frontend", "lib", "packages"}
CONFIG_IGNORED_DIRS = {".git", ".venv", "node_modules", "dist", "build", "coverage", "target", "vendor"}
def _bounded_config_files(root: Path, names: set[str], *, maximum: int = 20) -> list[Path]:
files: list[Path] = []
for current, directories, candidates in os.walk(root, followlinks=False):
directories[:] = [name for name in directories if name.casefold() not in CONFIG_IGNORED_DIRS]
for name in candidates:
if name in names:
files.append(Path(current) / name)
if len(files) >= maximum:
return sorted(files)
return sorted(files)
def _ts_path_aliases(root: Path) -> list[tuple[str, str]]:
"""Read a small local subset of tsconfig paths without resolving packages or URLs."""
aliases: list[tuple[str, str]] = []
root_resolved = root.resolve()
for config in _bounded_config_files(root, {"tsconfig.json", "jsconfig.json"}):
try:
if config.stat().st_size > 256_000:
continue
raw = config.read_text(encoding="utf-8")
# Most project tsconfigs are JSON. Comments are deliberately not
# interpreted as configuration to keep this scanner deterministic.
data = json.loads(raw)
options = data.get("compilerOptions", {})
paths = options.get("paths", {}) if isinstance(options, dict) else {}
if not isinstance(paths, dict):
continue
base_url = options.get("baseUrl", ".") if isinstance(options, dict) else "."
base = (config.parent / str(base_url)).resolve()
if base != root_resolved and root_resolved not in base.parents:
continue
for alias, targets in paths.items():
if not isinstance(alias, str) or not isinstance(targets, list):
continue
for target in targets[:5]:
if not isinstance(target, str) or target.startswith(("/", "\\")):
continue
resolved = (base / target.replace("*", "placeholder")).resolve()
if resolved != root_resolved and root_resolved not in resolved.parents:
continue
aliases.append((alias, str((base / target).relative_to(root_resolved)).replace("\\", "/")))
except (OSError, ValueError, json.JSONDecodeError, TypeError):
continue
return aliases[:100]
def _workspace_packages(root: Path) -> dict[str, Path]:
packages: dict[str, Path] = {}
for manifest in _bounded_config_files(root, {"package.json"}, maximum=100):
try:
if manifest.stat().st_size > 256_000:
continue
data = json.loads(manifest.read_text(encoding="utf-8"))
name = data.get("name") if isinstance(data, dict) else None
if isinstance(name, str) and name and not name.startswith((".", "/")):
packages.setdefault(name, manifest.parent)
except (OSError, ValueError, json.JSONDecodeError, TypeError):
continue
return packages
def _module_names(path: str) -> list[str]:
parts = list(Path(path).with_suffix("").parts)
if parts and parts[-1] == "__init__":
parts = parts[:-1]
names: list[str] = []
if parts:
names.append(".".join(parts))
if parts[0] in SOURCE_ROOTS and len(parts) > 1:
names.append(".".join(parts[1:]))
if len(parts) > 2 and parts[1] in SOURCE_ROOTS:
names.append(".".join(parts[2:]))
return [name for name in names if name]
def _python_module_map(paths: list[str]) -> dict[str, str]:
result: dict[str, str] = {}
for path in paths:
if not path.endswith(".py"):
continue
for module in _module_names(path):
result.setdefault(module, path)
return result
def _resolve_python_target(module: str, module_map: dict[str, str]) -> str | None:
candidate = module
while candidate:
if candidate in module_map:
return module_map[candidate]
if "." not in candidate:
break
candidate = candidate.rsplit(".", 1)[0]
return None
def _python_imports(path: str, text: str, module_map: dict[str, str]) -> set[str]:
try:
tree = ast.parse(text)
except SyntaxError:
return set()
current_names = _module_names(path)
current = current_names[0] if current_names else ""
current_parts = current.split(".") if current else []
if not path.endswith("/__init__.py") and current_parts:
package_parts = current_parts[:-1]
else:
package_parts = current_parts
imports: set[str] = set()
for node in ast.walk(tree):
if isinstance(node, ast.Import):
for alias in node.names:
target = _resolve_python_target(alias.name, module_map)
if target and target != path:
imports.add(target)
elif isinstance(node, ast.ImportFrom):
module = node.module or ""
if node.level:
trim = max(0, node.level - 1)
prefix = package_parts[:-trim] if trim else package_parts
absolute = ".".join([*prefix, *([module] if module else [])])
else:
absolute = module
target = _resolve_python_target(absolute, module_map)
if target and target != path:
imports.add(target)
return imports
def _resolve_js_target(
root: Path,
source_path: str,
specifier: str,
known: set[str],
aliases: list[tuple[str, str]],
workspace_packages: dict[str, Path],
) -> str | None:
if specifier.startswith("."):
source_dir = (root / source_path).parent
base = (source_dir / specifier).resolve()
else:
base = None
for alias, target in aliases:
if alias.endswith("/*") and specifier.startswith(alias[:-1]):
base = (root / target.replace("*", specifier[len(alias) - 1 :])).resolve()
break
if alias == specifier:
base = (root / target).resolve()
break
if base is None:
package_name = next(
(name for name in workspace_packages if specifier == name or specifier.startswith(f"{name}/")),
None,
)
if package_name:
package_root = workspace_packages[package_name]
suffix = specifier.removeprefix(package_name).lstrip("/")
base = package_root / suffix if suffix else package_root / "src" / "index"
if base is None:
return None
try:
relative_base = base.relative_to(root.resolve()).as_posix()
except ValueError:
return None
candidates = [relative_base]
suffix = Path(relative_base).suffix.lower()
if suffix in JS_EXTENSIONS:
extensionless = relative_base[: -len(suffix)]
candidates.extend(extensionless + extension for extension in JS_EXTENSIONS)
candidates.extend(f"{extensionless}/index{extension}" for extension in JS_EXTENSIONS)
elif not suffix:
candidates.extend(relative_base + extension for extension in JS_EXTENSIONS)
candidates.extend(f"{relative_base}/index{extension}" for extension in JS_EXTENSIONS)
return next((candidate for candidate in candidates if candidate in known), None)
def _js_imports(
root: Path,
path: str,
text: str,
known: set[str],
aliases: list[tuple[str, str]],
workspace_packages: dict[str, Path],
) -> set[str]:
imports: set[str] = set()
for match in JS_IMPORT_RE.finditer(text):
target = _resolve_js_target(root, path, match.group("target"), known, aliases, workspace_packages)
if target and target != path:
imports.add(target)
return imports
def build_architecture(
root: Path,
source_texts: dict[str, str],
language_by_path: dict[str, str],
loc_by_path: dict[str, int],
debt_by_path: dict[str, int],
churn_by_path: dict[str, int],
*,
max_nodes: int = 160,
max_edges: int = 600,
) -> tuple[ArchitectureGraph, list[Hotspot]]:
max_nodes = max(1, max_nodes)
max_edges = max(0, max_edges)
known_paths = set(source_texts)
module_map = _python_module_map(list(known_paths))
ts_aliases = _ts_path_aliases(root)
workspace_packages = _workspace_packages(root)
edge_pairs: set[tuple[str, str]] = set()
total_inbound: Counter[str] = Counter()
total_outbound: Counter[str] = Counter()
total_edges = 0
truncated = len(known_paths) > max_nodes
for path in sorted(source_texts):
text = source_texts[path]
language = language_by_path.get(path, "")
if language == "Python":
targets = _python_imports(path, text, module_map)
elif language in {"TypeScript", "JavaScript"}:
targets = _js_imports(root, path, text, known_paths, ts_aliases, workspace_packages)
else:
targets = set()
for target in sorted(targets):
if target not in known_paths:
continue
pair = (path, target)
total_edges += 1
total_outbound[path] += 1
total_inbound[target] += 1
if len(edge_pairs) < max_edges:
edge_pairs.add(pair)
else:
truncated = True
hotspots: list[Hotspot] = []
for path in known_paths:
loc = loc_by_path.get(path, 0)
debt = debt_by_path.get(path, 0)
churn = churn_by_path.get(path, 0)
coupling = total_inbound[path] + total_outbound[path]
score = min(
100,
round(
min(30, loc / 40)
+ min(20, debt * 4)
+ min(30, math.log10(churn + 1) * 10 if churn else 0)
+ min(20, coupling * 2.5)
),
)
reasons: list[str] = []
if loc >= 500:
reasons.append(f"large module ({loc} LOC)")
if debt:
reasons.append(f"{debt} debt markers")
if churn >= 300:
reasons.append(f"high recent churn ({churn} changed lines)")
if coupling >= 8:
reasons.append(f"high coupling ({coupling} local edges)")
if not reasons and score >= 30:
reasons.append("combined size, churn and coupling")
hotspots.append(
Hotspot(
path=path,
language=language_by_path.get(path, "Other"),
lines_of_code=loc,
debt_markers=debt,
churn=churn,
inbound=total_inbound[path],
outbound=total_outbound[path],
score=score,
reasons=reasons,
)
)
hotspots.sort(key=lambda item: (-item.score, -item.churn, -item.lines_of_code, item.path))
selected_paths = {item.path for item in hotspots[:max_nodes]}
graph_edges = [
ArchitectureEdge(source=source, target=target)
for source, target in sorted(edge_pairs)
if source in selected_paths and target in selected_paths
][:max_edges]
visible_inbound: Counter[str] = Counter()
visible_outbound: Counter[str] = Counter()
for edge in graph_edges:
visible_outbound[edge.source] += 1
visible_inbound[edge.target] += 1
hotspot_scores = {item.path: item.score for item in hotspots}
nodes = [
ArchitectureNode(
id=path,
label=Path(path).name,
path=path,
language=language_by_path.get(path, "Other"),
inbound=total_inbound[path],
outbound=total_outbound[path],
visible_inbound=visible_inbound[path],
visible_outbound=visible_outbound[path],
hotspot_score=hotspot_scores[path],
)
for path in sorted(selected_paths)
]
clusters = Counter((Path(path).parts[0] if len(Path(path).parts) > 1 else "root") for path in selected_paths)
return (
ArchitectureGraph(
nodes=nodes,
edges=graph_edges,
clusters=dict(clusters.most_common()),
truncated=truncated,
total_nodes=len(known_paths),
total_edges=total_edges,
),
hotspots[:30],
)
+104
View File
@@ -0,0 +1,104 @@
from __future__ import annotations
import subprocess
from collections import Counter
from datetime import datetime
from pathlib import Path
from .models import GitSignals
def _run(root: Path, *args: str, timeout: int = 8) -> str | None:
try:
result = subprocess.run(
["git", "-C", str(root), *args],
check=True,
capture_output=True,
text=True,
errors="replace",
timeout=timeout,
)
except (OSError, UnicodeError, subprocess.SubprocessError):
return None
return result.stdout
def analyze_git(root: Path) -> tuple[GitSignals, dict[str, int]]:
top_level = _run(root, "rev-parse", "--show-toplevel")
if not top_level:
return GitSignals(), {}
try:
if Path(top_level.strip()).resolve() != root.resolve():
return GitSignals(), {}
except (OSError, RuntimeError, ValueError):
return GitSignals(), {}
inside = _run(root, "rev-parse", "--is-inside-work-tree")
if not inside or inside.strip() != "true":
return GitSignals(), {}
raw_count = _run(root, "rev-list", "--count", "HEAD")
raw_latest = _run(root, "log", "-1", "--format=%aI")
raw_authors = _run(root, "log", "--format=%aN%x00%aE", "-n", "2000")
raw_files = _run(root, "ls-files", "-z")
raw_churn = _run(root, "log", "--numstat", "--format=", "--find-renames=20%", "-n", "200")
raw_renames = _run(root, "log", "--name-status", "--format=", "--find-renames=20%", "-n", "200")
contributors: set[str] = set()
if raw_authors:
for line in raw_authors.splitlines():
if line.strip():
contributors.add(line.strip())
churn: Counter[str] = Counter()
renamed_files: set[str] = set()
rename_map: dict[str, str] = {}
if raw_renames:
for line in raw_renames.splitlines():
parts = line.split("\t")
if len(parts) >= 3 and parts[0].startswith("R"):
old_path, new_path = parts[-2:]
rename_map[old_path] = new_path
renamed_files.add(new_path)
if raw_churn:
for line in raw_churn.splitlines():
parts = line.split("\t", 2)
if len(parts) != 3:
continue
additions, deletions, path = parts
churn_path = _rename_target(path) if " => " in path else rename_map.get(path, path)
if " => " in path:
renamed_files.add(churn_path)
if additions.isdigit():
churn[churn_path] += int(additions)
if deletions.isdigit():
churn[churn_path] += int(deletions)
latest_commit_at = None
if raw_latest and raw_latest.strip():
try:
latest_commit_at = datetime.fromisoformat(raw_latest.strip())
except ValueError:
latest_commit_at = None
return (
GitSignals(
available=True,
commit_count=int(raw_count.strip()) if raw_count and raw_count.strip().isdigit() else None,
contributor_count=len(contributors) if contributors else None,
latest_commit_at=latest_commit_at,
tracked_files=len([item for item in raw_files.split("\0") if item]) if raw_files is not None else None,
renamed_files=sorted(renamed_files),
),
dict(churn),
)
def _rename_target(path: str) -> str:
"""Normalize Git's brace-formatted rename path to the new path."""
if "{" in path and "}" in path:
prefix, remainder = path.split("{", 1)
inner, suffix = remainder.split("}", 1)
if " => " in inner:
return f"{prefix}{inner.split(' => ', 1)[1]}{suffix}"
return path.rsplit(" => ", 1)[-1]
+314
View File
@@ -0,0 +1,314 @@
from __future__ import annotations
import json
import os
import urllib.error
import urllib.parse
import urllib.request
from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime
from typing import Any
from .models import Repository, RepositoryPolicy, RepositorySyncResult
JsonTransport = Callable[[str, dict[str, str]], tuple[int, bytes]]
class GiteaError(RuntimeError):
def __init__(self, code: str, detail: str, *, status_code: int | None = None) -> None:
super().__init__(detail)
self.code = code
self.detail = detail
self.status_code = status_code
def _parse_time(value: object) -> datetime | None:
if not isinstance(value, str) or not value:
return None
try:
parsed = datetime.fromisoformat(value)
except ValueError:
return None
return parsed.replace(tzinfo=UTC) if parsed.tzinfo is None else parsed
class GiteaClient:
"""Small, read-only Gitea API client with bounded pagination and safe errors."""
def __init__(
self,
base_url: str | None = None,
token: str | None = None,
*,
timeout: float = 10.0,
page_size: int = 50,
max_pages: int = 20,
max_repositories: int = 500,
max_branch_workers: int | None = None,
transport: JsonTransport | None = None,
) -> None:
self.base_url = (base_url or os.getenv("REPOPULSE_GITEA_URL", "")).rstrip("/")
self.token = token if token is not None else os.getenv("REPOPULSE_GITEA_TOKEN", "")
self.timeout = max(1.0, timeout)
self.page_size = max(1, min(page_size, 100))
self.max_pages = max(1, max_pages)
self.max_repositories = max(1, max_repositories)
if max_branch_workers is None:
try:
configured_workers = int(os.getenv("REPOPULSE_GITEA_BRANCH_WORKERS", "4"))
except ValueError:
configured_workers = 4
else:
configured_workers = max_branch_workers
self.max_branch_workers = min(8, max(1, configured_workers))
self.transport = transport
self._listing_complete = True
@property
def configured(self) -> bool:
return bool(self.base_url and self.token)
def _request_json(self, path: str, params: dict[str, str | int] | None = None) -> Any:
if not self.base_url:
raise GiteaError("not_configured", "Gitea URL is not configured")
query = urllib.parse.urlencode(params or {})
url = f"{self.base_url}/api/v1/{path.lstrip('/')}"
if query:
url = f"{url}?{query}"
headers = {
"Accept": "application/json",
"User-Agent": "RepoPulse/1.5",
}
if self.token:
headers["Authorization"] = f"token {self.token}"
try:
if self.transport:
status_code, raw = self.transport(url, headers)
if status_code >= 400:
self._raise_for_status(status_code)
else:
request = urllib.request.Request(url, headers=headers, method="GET")
with urllib.request.urlopen(request, timeout=self.timeout) as response:
status_code = int(response.status)
raw = response.read(2_000_001)
if status_code >= 400:
self._raise_for_status(status_code)
except GiteaError:
raise
except urllib.error.HTTPError as exc:
self._raise_for_status(exc.code)
except (urllib.error.URLError, TimeoutError, OSError) as exc:
raise GiteaError("unreachable", "Gitea could not be reached") from exc
if not isinstance(raw, bytes):
raise GiteaError("invalid_response", "Gitea returned an invalid response body")
if len(raw) > 2_000_000:
raise GiteaError("response_too_large", "Gitea response exceeded the configured bound")
try:
return json.loads(raw.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise GiteaError("invalid_response", "Gitea returned invalid JSON") from exc
@staticmethod
def _raise_for_status(status_code: int) -> None:
if status_code in {401, 403}:
raise GiteaError("authentication", "Gitea authentication was rejected", status_code=status_code)
if status_code == 404:
raise GiteaError("not_found", "The requested Gitea resource was not found", status_code=status_code)
if status_code == 429:
raise GiteaError("rate_limited", "Gitea rate-limited the request", status_code=status_code)
if status_code >= 500:
raise GiteaError("upstream", "Gitea returned a server error", status_code=status_code)
raise GiteaError("request_failed", "Gitea rejected the request", status_code=status_code)
def _repository_from_payload(self, payload: object) -> Repository:
if not isinstance(payload, dict):
raise GiteaError("invalid_repository", "Gitea returned an invalid repository record")
raw_id = payload.get("id")
if isinstance(raw_id, bool) or not isinstance(raw_id, (int, str)) or not str(raw_id).isdigit():
raise GiteaError("invalid_repository", "Gitea repository identity is invalid")
owner_payload = payload.get("owner")
owner = owner_payload.get("login") if isinstance(owner_payload, dict) else None
owner = owner or payload.get("owner_name")
name = payload.get("name")
full_name = payload.get("full_name")
if not isinstance(owner, str) or not isinstance(name, str):
if isinstance(full_name, str) and "/" in full_name:
owner, name = full_name.split("/", 1)
else:
raise GiteaError("invalid_repository", "Gitea repository name is invalid")
full_name = full_name if isinstance(full_name, str) else f"{owner}/{name}"
repository_id = str(raw_id)
# Do not trust a provider-supplied clone host when constructing the
# credential-bearing acquisition URL. The configured Gitea origin is
# the only network destination that may receive the server token.
clone_url = (
f"{self.base_url}/{urllib.parse.quote(owner, safe='')}/"
f"{urllib.parse.quote(name, safe='')}.git"
)
html_url = payload.get("html_url")
default_branch = payload.get("default_branch") or "main"
return Repository(
id=f"gitea:{repository_id}",
provider="gitea",
provider_repository_id=repository_id,
owner=owner,
name=name,
full_name=full_name,
description=payload.get("description") if isinstance(payload.get("description"), str) else None,
html_url=html_url if isinstance(html_url, str) else None,
clone_url=clone_url if isinstance(clone_url, str) else None,
default_branch=str(default_branch),
private=bool(payload.get("private", False)),
archived=bool(payload.get("archived", False)),
latest_revision_at=_parse_time(payload.get("updated_at")),
policy=RepositoryPolicy(),
)
def list_repositories(self) -> list[Repository]:
if not self.configured:
raise GiteaError("not_configured", "Gitea connector is not configured")
repositories: list[Repository] = []
self._listing_complete = False
for page in range(1, self.max_pages + 1):
payload = self._request_json(
"user/repos",
{"limit": self.page_size, "page": page, "sort": "updated", "direction": "desc"},
)
if not isinstance(payload, list):
raise GiteaError("invalid_response", "Gitea repository listing was not a list")
for item in payload:
if len(repositories) >= self.max_repositories:
return repositories
repositories.append(self._repository_from_payload(item))
if len(payload) < self.page_size:
self._listing_complete = True
break
return repositories
def branch_revision(self, repository: Repository) -> str:
owner = urllib.parse.quote(repository.owner, safe="")
name = urllib.parse.quote(repository.name, safe="")
branch = urllib.parse.quote(repository.default_branch, safe="")
payload = self._request_json(f"repos/{owner}/{name}/branches/{branch}")
if not isinstance(payload, dict):
raise GiteaError("invalid_response", "Gitea branch response was invalid")
commit = payload.get("commit")
revision = None
if isinstance(commit, dict):
revision = commit.get("id") or commit.get("sha")
if (
not isinstance(revision, str)
or not 7 <= len(revision) <= 64
or not all(char in "0123456789abcdefABCDEF" for char in revision)
):
raise GiteaError("invalid_revision", "Gitea did not return a usable branch revision")
return revision
def discover_with_errors(self) -> tuple[list[Repository], list[tuple[Repository, str]], list[str]]:
"""Resolve branch tips independently so one repository cannot abort a catalog sync."""
repositories = self.list_repositories()
def resolve(repository: Repository) -> tuple[Repository, str | None, str | None]:
try:
return repository, self.branch_revision(repository), None
except GiteaError as exc:
return repository, None, f"{repository.full_name}: {exc.code}: {exc.detail}"
with ThreadPoolExecutor(max_workers=min(self.max_branch_workers, len(repositories) or 1)) as executor:
outcomes = list(executor.map(resolve, repositories))
discovered = [
(repository.model_copy(update={"latest_revision": revision}), revision)
for repository, revision, error in outcomes
if revision is not None and error is None
]
errors = [error for _, _, error in outcomes if error]
return repositories, discovered, errors
def discover(self) -> list[tuple[Repository, str]]:
"""Compatibility helper for callers needing only successfully resolved repositories."""
_, discovered, _ = self.discover_with_errors()
return discovered
def sync(self, store) -> RepositorySyncResult:
synced_at = datetime.now(UTC)
try:
listed, discovered, branch_errors = self.discover_with_errors()
except GiteaError as exc:
error = f"{exc.code}: {exc.detail}"
for existing in store.list_repositories(limit=1000):
if existing.provider == "gitea" and existing.available:
store.update_repository_sync(existing, synced_at=synced_at, error=error)
result = RepositorySyncResult(
provider="gitea",
errors=[error],
synced_at=synced_at,
)
store.record_sync_result(result)
return result
created = 0
updated = 0
listed_ids = {repository.id for repository in listed}
seen_ids: set[str] = set()
for repository, revision in discovered:
seen_ids.add(repository.id)
before = store.get_repository(repository.id)
merged = repository.model_copy(
update={
"available": True,
"last_synced_at": synced_at,
"sync_error": None,
"latest_revision": revision,
"policy": before.policy if before else RepositoryPolicy(),
}
)
store.upsert_repository(merged)
if before is None:
created += 1
else:
updated += 1
for repository in listed:
if repository.id in seen_ids:
continue
before = store.get_repository(repository.id)
error = next((item for item in branch_errors if item.startswith(f"{repository.full_name}:")), None)
merged = repository.model_copy(
update={
"available": True,
"last_synced_at": synced_at,
"sync_error": error or "branch: revision could not be resolved",
"latest_revision": before.latest_revision if before else None,
"policy": before.policy if before else RepositoryPolicy(),
}
)
store.upsert_repository(merged)
if before is None:
created += 1
else:
updated += 1
unavailable = 0
if self._listing_complete:
for existing in store.list_repositories(limit=1000):
if existing.provider == "gitea" and existing.id not in listed_ids and existing.available:
store.update_repository_sync(
existing.model_copy(update={"available": False}),
synced_at=synced_at,
error="not_discovered",
)
unavailable += 1
result = RepositorySyncResult(
provider="gitea",
discovered=len(discovered),
created=created,
updated=updated,
unavailable=unavailable,
errors=branch_errors,
partial=bool(branch_errors) or not self._listing_complete,
branch_failures=len(branch_errors),
listing_complete=self._listing_complete,
synced_at=synced_at,
)
store.record_sync_result(result)
return result
+330
View File
@@ -0,0 +1,330 @@
from __future__ import annotations
import logging
import os
from datetime import UTC, datetime
from threading import Event, Lock, Thread, current_thread
from time import monotonic
from .acquisition import AcquisitionError, GitAcquirer
from .gitea import GiteaClient
from .models import (
Repository,
RepositorySyncResult,
ScanJob,
ScanJobStatus,
ScanSource,
ScanTrigger,
SourceOrigin,
)
from .scanner import scan_repository
from .store import MaintenanceModeError, QueueFullError, ScanStore, job_lease_seconds
LOGGER = logging.getLogger(__name__)
def _positive_int(name: str, default: int, *, maximum: int | None = None) -> int:
try:
value = max(1, int(os.getenv(name, str(default))))
except ValueError:
value = default
return min(value, maximum) if maximum else value
def _positive_float(name: str, default: float) -> float:
try:
return max(0.2, float(os.getenv(name, str(default))))
except ValueError:
return default
class ScanJobRunner:
"""Single-process durable worker and bounded Gitea scheduler."""
def __init__(
self,
store: ScanStore,
*,
gitea: GiteaClient | None = None,
acquirer: GitAcquirer | None = None,
poll_seconds: float = 0.5,
sync_interval_seconds: float | None = None,
max_queue: int | None = None,
) -> None:
self.store = store
self.gitea = gitea or GiteaClient()
self.acquirer = acquirer or GitAcquirer()
self.poll_seconds = max(0.2, poll_seconds)
self.sync_interval_seconds = sync_interval_seconds or _positive_float(
"REPOPULSE_GITEA_SYNC_INTERVAL_SECONDS", 300.0
)
self.max_queue = max_queue or _positive_int("REPOPULSE_MAX_QUEUED_SCANS", 100, maximum=1000)
self._stop = Event()
self._thread: Thread | None = None
self._last_sync = float("-inf")
self._sync_lock = Lock()
@property
def started(self) -> bool:
return self._thread is not None and self._thread.is_alive()
def start(self) -> None:
if self.started:
return
self._stop.clear()
self._thread = Thread(target=self._run, name="repopulse-scan-worker", daemon=True)
self._thread.start()
def stop(self) -> None:
self._stop.set()
thread = self._thread
if thread and thread is not current_thread():
thread.join(timeout=5)
if thread is None or not thread.is_alive():
self._thread = None
def _run(self) -> None:
while not self._stop.is_set():
try:
job = self._run_iteration()
except MaintenanceModeError:
job = None
except Exception:
self._last_sync = monotonic()
LOGGER.exception("RepoPulse worker loop recovered from an unexpected failure")
job = None
if job is None:
self._stop.wait(self.poll_seconds)
def _run_iteration(self) -> ScanJob | None:
"""Perform one scheduler pass unless durable maintenance has paused work."""
if self.store.maintenance_status()["enabled"]:
return None
if not self.store.touch_worker_heartbeat_if_active(datetime.now(UTC).isoformat()):
return None
if self.gitea.configured and monotonic() - self._last_sync >= self.sync_interval_seconds:
self.sync_now()
if self.store.maintenance_status()["enabled"]:
return None
return self.process_one()
def sync_now(self) -> RepositorySyncResult:
with self._sync_lock:
if self.store.maintenance_status()["enabled"]:
raise MaintenanceModeError("RepoPulse is in maintenance mode")
if not self.gitea.configured:
result = RepositorySyncResult(
provider="gitea",
errors=["not_configured: Gitea connector is not configured"],
synced_at=datetime.now(UTC),
)
self.store.record_sync_result(result)
self._last_sync = monotonic()
return result
try:
result = self.gitea.sync(self.store)
except Exception:
self._last_sync = monotonic()
raise
self._last_sync = monotonic()
if not result.errors:
self._schedule_changed_repositories()
return result
def _schedule_changed_repositories(self) -> None:
if self.store.maintenance_status()["enabled"]:
return
for summary in self.store.list_repositories(limit=1000):
if summary.provider != "gitea" or summary.archived or not summary.latest_revision:
continue
if not summary.policy.auto_scan:
continue
if summary.last_scan_revision and summary.last_scan_revision == summary.latest_revision:
continue
try:
self.store.enqueue_job(
summary.id,
summary.latest_revision,
ScanTrigger.SCHEDULED,
max_queue=self.max_queue,
)
except (MaintenanceModeError, QueueFullError, KeyError):
continue
def set_maintenance_mode(self, enabled: bool) -> dict[str, object]:
"""Serialize maintenance changes with catalog reconciliation and scan scheduling."""
with self._sync_lock:
return self.store.set_maintenance_mode(enabled)
def enqueue(
self,
repository: Repository,
*,
force: bool = False,
trigger: ScanTrigger = ScanTrigger.MANUAL,
) -> ScanJob:
requested_revision = repository.latest_revision
return self.store.enqueue_job(
repository.id,
requested_revision,
trigger,
force=force,
max_queue=self.max_queue,
)
def process_one(self) -> ScanJob | None:
job = self.store.claim_next_job()
if job is None:
return None
try:
self._process(job)
except Exception:
LOGGER.exception("RepoPulse scan job failed outside the isolated job handler")
try:
self.store.update_job(
job.id,
status=ScanJobStatus.FAILED,
phase="failed",
progress=100,
error_code="internal_error",
error_detail="Unexpected repository analysis failure",
message="Repository analysis failed",
)
except Exception:
LOGGER.exception("RepoPulse could not persist an unexpected job failure")
return self.store.get_job(job.id)
class _LeaseHeartbeat:
def __init__(self, store: ScanStore, job_id: str) -> None:
self.store = store
self.job_id = job_id
self.stop_event = Event()
self.thread = Thread(target=self._run, name=f"repopulse-lease-{job_id[:8]}", daemon=True)
def start(self) -> None:
self.thread.start()
def stop(self) -> None:
self.stop_event.set()
self.thread.join(timeout=2)
def _run(self) -> None:
interval = max(1.0, job_lease_seconds() / 3)
while not self.stop_event.wait(interval):
try:
self.store.renew_job_lease(self.job_id)
except Exception:
LOGGER.warning("RepoPulse could not renew a scan-job lease", exc_info=True)
def _process(self, job: ScanJob) -> None:
repository = self.store.get_repository(job.repository_id)
if repository is None:
self.store.update_job(
job.id,
status=ScanJobStatus.FAILED,
phase="failed",
progress=100,
error_code="repository_not_found",
error_detail="Repository catalog entry no longer exists",
)
return
source_path = None
acquired = None
lease_heartbeat = self._LeaseHeartbeat(self.store, job.id)
lease_heartbeat.start()
try:
if repository.provider != "gitea":
source_path = self.store.get_repository_source_path(repository.id)
if not source_path:
raise AcquisitionError("source_unavailable", "Local repository source is unavailable")
actual_revision = job.requested_revision
branch = repository.default_branch
else:
acquired = self.acquirer.acquire(repository, job.id, job.requested_revision)
source_path = str(acquired.path)
actual_revision = acquired.actual_revision
branch = acquired.branch
self.store.update_job(
job.id,
status=ScanJobStatus.SCANNING,
phase="scanning",
progress=30,
actual_revision=actual_revision,
message="Running deterministic repository analysis",
)
source = ScanSource(
origin=SourceOrigin.GITEA if repository.provider == "gitea" else SourceOrigin.LOCAL,
provider=repository.provider,
repository_id=repository.id,
repository_full_name=repository.full_name,
branch=branch,
requested_revision=job.requested_revision,
actual_revision=actual_revision,
trigger=job.trigger,
)
report = scan_repository(source_path, source=source, display_name=repository.name)
self.store.update_job(
job.id,
status=ScanJobStatus.ANALYZING,
phase="analyzing",
progress=80,
message="Persisting report and comparison baseline",
)
self.store.update_job(
job.id,
status=ScanJobStatus.PERSISTING,
phase="persisting",
progress=90,
message="Saving report and revision history",
)
saved = self.store.save(report)
self.store.upsert_repository(repository.model_copy(update={"policy": saved.policy}))
self.store.update_job(
job.id,
status=ScanJobStatus.COMPLETED,
phase="completed",
progress=100,
actual_revision=saved.source.actual_revision,
scan_id=saved.id,
message="Scan completed",
)
except AcquisitionError as exc:
self.store.update_job(
job.id,
status=ScanJobStatus.FAILED,
phase="failed",
progress=100,
error_code=exc.code,
error_detail=exc.detail,
message="Repository source acquisition failed",
)
except (ValueError, PermissionError, OSError) as exc:
self.store.update_job(
job.id,
status=ScanJobStatus.FAILED,
phase="failed",
progress=100,
error_code="scan_failed",
error_detail=str(exc) if isinstance(exc, ValueError) else "Repository analysis failed",
message="Repository analysis failed",
)
except Exception:
LOGGER.exception("RepoPulse scan job failed without exposing job details")
self.store.update_job(
job.id,
status=ScanJobStatus.FAILED,
phase="failed",
progress=100,
error_code="internal_error",
error_detail="Unexpected repository analysis failure",
message="Repository analysis failed",
)
finally:
lease_heartbeat.stop()
if acquired is not None and repository is not None:
try:
self.acquirer.cleanup(acquired, repository)
except AcquisitionError:
LOGGER.warning("Unable to clean one RepoPulse worktree")
+491
View File
@@ -0,0 +1,491 @@
from __future__ import annotations
import asyncio
import hashlib
import hmac
import json
import os
from contextlib import asynccontextmanager
from pathlib import Path
from threading import BoundedSemaphore, Lock
from fastapi import FastAPI, HTTPException, Query, Request, Response, status
from fastapi.concurrency import run_in_threadpool
from fastapi.responses import JSONResponse
from fastapi.staticfiles import StaticFiles
from .acquisition import GitAcquirer
from .gitea import GiteaClient
from .jobs import ScanJobRunner
from .models import (
RepositoryDetail,
RepositoryScanRequest,
RepositorySummary,
RepositorySyncResult,
ScanComparison,
ScanJob,
ScanReport,
ScanRequest,
ScanSummary,
ScanTrigger,
TrendPoint,
WebhookResponse,
)
from .reporting import content_disposition, render_html_report, safe_report_filename
from .scanner import scan_repository
from .store import MaintenanceModeError, QueueFullError, ScanStore
VERSION = "1.5.0"
store = ScanStore()
SCAN_ROOT = os.getenv("REPOPULSE_SCAN_ROOT")
def _positive_int_env(name: str, default: int) -> int:
try:
return max(1, int(os.getenv(name, str(default))))
except ValueError:
return default
# The durable queue has one real worker. Keep direct legacy path scans at the
# same capacity rather than advertising concurrency that cannot be delivered.
REQUESTED_MAX_CONCURRENT_SCANS = _positive_int_env("REPOPULSE_MAX_CONCURRENT_SCANS", 1)
MAX_CONCURRENT_SCANS = 1
SCAN_SLOTS = BoundedSemaphore(MAX_CONCURRENT_SCANS)
gitea_client = GiteaClient()
repository_acquirer = GitAcquirer()
job_runner: ScanJobRunner | None = None
_api_mutation_lock = Lock()
_MAINTENANCE_ENTER_PATH = "/api/v1/operator/maintenance/enter"
_MAINTENANCE_EXIT_PATH = "/api/v1/operator/maintenance/exit"
def _get_runner() -> ScanJobRunner:
global job_runner
if job_runner is None or job_runner.store is not store:
job_runner = ScanJobRunner(
store,
gitea=gitea_client,
acquirer=repository_acquirer,
)
return job_runner
@asynccontextmanager
async def lifespan(_app: FastAPI):
runner = _get_runner()
runner.start()
try:
yield
finally:
runner.stop()
app = FastAPI(title="RepoPulse API", version=VERSION, lifespan=lifespan)
@app.middleware("http")
async def maintenance_mutation_gate(request: Request, call_next):
"""Serialize external API writes with maintenance transitions and reject writes while paused."""
if not request.url.path.startswith("/api/v1/") or request.method in {"GET", "HEAD", "OPTIONS"}:
return await call_next(request)
if request.url.path in {_MAINTENANCE_ENTER_PATH, _MAINTENANCE_EXIT_PATH}:
return await call_next(request)
await run_in_threadpool(_api_mutation_lock.acquire)
try:
if store.maintenance_status()["enabled"]:
return JSONResponse(
status_code=status.HTTP_423_LOCKED,
content={"detail": "RepoPulse is in maintenance mode"},
headers={"Retry-After": "30"},
)
return await call_next(request)
finally:
_api_mutation_lock.release()
def _maintenance_snapshot() -> dict[str, object]:
mode = store.maintenance_status()
stats = store.queue_stats()
queued = int(stats["queued"])
active = int(stats["active"])
return {
**mode,
"queue_depth": queued,
"active_jobs": active,
"active_drained": active == 0,
"drained": queued == 0 and active == 0,
}
def _require_mutation_authorization(request: Request, *, strict: bool = False) -> None:
"""Use an operator token when configured; deployment perimeter remains explicit otherwise."""
token = os.getenv("REPOPULSE_OPERATOR_TOKEN", "")
if strict:
token = os.getenv("REPOPULSE_MAINTENANCE_TOKEN", "") or token
if not token:
if strict:
raise HTTPException(status_code=503, detail="Operator token is not configured")
return
supplied = request.headers.get("x-repopulse-token", "")
if not hmac.compare_digest(supplied, token):
raise HTTPException(status_code=401, detail="Operator authorization is required")
def _resolve_scan_path(raw_path: str) -> Path:
supplied = Path(raw_path).expanduser()
if SCAN_ROOT:
root = Path(SCAN_ROOT).expanduser().resolve()
candidate = (root / supplied).resolve() if not supplied.is_absolute() else supplied.resolve()
if candidate != root and root not in candidate.parents:
raise ValueError(f"Path must be inside configured scan root {root}")
return candidate
return supplied.resolve()
@app.get("/health")
def health() -> dict[str, str]:
recovery = store.database_recovery_status()
status = "degraded" if recovery.get("status") == "degraded_recovered" else "ok"
return {"status": status}
@app.get("/api/v1/meta")
def meta() -> dict[str, object]:
stats = store.queue_stats()
repository_states = store.repository_state_counts()
last_sync = store.get_state("last_gitea_sync")
parsed_sync: object = None
if last_sync:
try:
parsed_sync = json.loads(last_sync)
except json.JSONDecodeError:
parsed_sync = last_sync
return {
"name": "RepoPulse",
"version": VERSION,
"stage": "v1.5.0",
"scan_root_restricted": bool(SCAN_ROOT),
"scan_root": SCAN_ROOT,
"max_concurrent_scans": MAX_CONCURRENT_SCANS,
"requested_max_concurrent_scans": REQUESTED_MAX_CONCURRENT_SCANS,
"repository_count": store.repository_count(),
"repository_states": repository_states,
"current_repositories": repository_states.get("current", 0),
"stale_repositories": repository_states.get("stale", 0),
"queue_depth": stats["queued"],
"active_jobs": stats["active"],
"completed_jobs": stats["completed"],
"failed_jobs": stats["failed"],
"worker_heartbeat": stats["worker_heartbeat"],
"gitea": {
"configured": gitea_client.configured,
"url": gitea_client.base_url or None,
"last_sync": parsed_sync,
},
"storage": store.storage_stats(),
"database_recovery": store.database_recovery_status(),
"maintenance": _maintenance_snapshot(),
"mutation_authentication": "operator_token" if os.getenv("REPOPULSE_OPERATOR_TOKEN") else "perimeter_required",
}
@app.get("/api/v1/operator/maintenance")
def get_maintenance_status(request: Request) -> dict[str, object]:
_require_mutation_authorization(request, strict=True)
return _maintenance_snapshot()
@app.post("/api/v1/operator/maintenance/enter")
def enter_maintenance(request: Request) -> dict[str, object]:
_require_mutation_authorization(request, strict=True)
with _api_mutation_lock:
if store.maintenance_status()["enabled"]:
raise HTTPException(status_code=423, detail="RepoPulse is already in maintenance mode")
_get_runner().set_maintenance_mode(True)
return _maintenance_snapshot()
@app.post("/api/v1/operator/maintenance/exit")
def exit_maintenance(request: Request) -> dict[str, object]:
_require_mutation_authorization(request, strict=True)
with _api_mutation_lock:
_get_runner().set_maintenance_mode(False)
return _maintenance_snapshot()
@app.post("/api/v1/scans", response_model=ScanReport, status_code=status.HTTP_201_CREATED)
async def create_scan(payload: ScanRequest, request: Request) -> ScanReport:
_require_mutation_authorization(request)
try:
repository_path = _resolve_scan_path(payload.path)
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
if not SCAN_SLOTS.acquire(blocking=False):
raise HTTPException(
status_code=429,
detail="Repository scan capacity is currently in use; retry after the active scan completes",
headers={"Retry-After": "2"},
)
try:
report = await asyncio.to_thread(scan_repository, str(repository_path))
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
except PermissionError as exc:
raise HTTPException(status_code=403, detail="Repository path cannot be read") from exc
except OSError as exc:
raise HTTPException(status_code=422, detail="Repository path cannot be scanned") from exc
finally:
SCAN_SLOTS.release()
return store.save(report)
@app.get("/api/v1/repositories", response_model=list[RepositorySummary])
def list_repositories(
limit: int = Query(default=100, ge=1, le=1000),
offset: int = Query(default=0, ge=0, le=100_000),
) -> list[RepositorySummary]:
return store.list_repositories(limit, offset)
@app.post("/api/v1/repositories/sync", response_model=RepositorySyncResult)
async def sync_repositories(request: Request) -> RepositorySyncResult:
_require_mutation_authorization(request)
if not gitea_client.configured:
raise HTTPException(status_code=503, detail="Gitea connector is not configured")
try:
result = await asyncio.to_thread(_get_runner().sync_now)
except MaintenanceModeError as exc:
raise HTTPException(status_code=423, detail=str(exc), headers={"Retry-After": "30"}) from exc
if result.errors and result.branch_failures == 0:
raise HTTPException(status_code=502, detail=result.errors[0])
return result
@app.get("/api/v1/repositories/{repository_id}", response_model=RepositoryDetail)
def get_repository(repository_id: str) -> RepositoryDetail:
repository = store.get_repository_detail(repository_id)
if repository is None:
raise HTTPException(status_code=404, detail="Repository not found")
return repository
@app.get("/api/v1/repositories/{repository_id}/scans", response_model=list[ScanSummary])
def list_repository_scans(
repository_id: str,
limit: int = Query(default=50, ge=1, le=200),
) -> list[ScanSummary]:
if store.get_repository(repository_id) is None:
raise HTTPException(status_code=404, detail="Repository not found")
return store.repository_scans(repository_id, limit)
@app.get("/api/v1/repositories/{repository_id}/trend", response_model=list[TrendPoint])
def repository_trend(
repository_id: str,
limit: int = Query(default=50, ge=1, le=200),
) -> list[TrendPoint]:
if store.get_repository(repository_id) is None:
raise HTTPException(status_code=404, detail="Repository not found")
return store.trend(repository_id, limit)
@app.post("/api/v1/repositories/{repository_id}/scans", response_model=ScanJob, status_code=202)
async def create_repository_scan(
request: Request,
repository_id: str,
payload: RepositoryScanRequest | None = None,
) -> ScanJob:
_require_mutation_authorization(request)
repository = store.get_repository(repository_id)
if repository is None:
raise HTTPException(status_code=404, detail="Repository not found")
if repository.archived:
raise HTTPException(status_code=409, detail="Archived repositories are not scanned")
if not repository.available:
raise HTTPException(status_code=409, detail="Repository is no longer available to the Gitea connector")
if repository.provider == "gitea" and not gitea_client.configured:
raise HTTPException(status_code=503, detail="Gitea connector is not configured")
try:
return _get_runner().enqueue(repository, force=bool(payload and payload.force))
except MaintenanceModeError as exc:
raise HTTPException(status_code=423, detail=str(exc), headers={"Retry-After": "30"}) from exc
except QueueFullError as exc:
raise HTTPException(
status_code=429,
detail=str(exc),
headers={"Retry-After": "5"},
) from exc
@app.get("/api/v1/scan-jobs/{job_id}", response_model=ScanJob)
def get_scan_job(job_id: str) -> ScanJob:
job = store.get_job(job_id)
if job is None:
raise HTTPException(status_code=404, detail="Scan job not found")
return job
@app.get("/api/v1/repositories/{repository_id}/scans/{scan_id}", response_model=ScanReport)
def get_repository_scan(repository_id: str, scan_id: str) -> ScanReport:
report = store.get(scan_id)
if report is None or report.source.repository_id != repository_id:
raise HTTPException(status_code=404, detail="Repository scan not found")
return report
@app.post("/api/v1/webhooks/gitea", response_model=WebhookResponse)
async def gitea_webhook(request: Request) -> WebhookResponse:
secret = os.getenv("REPOPULSE_GITEA_WEBHOOK_SECRET", "")
if not secret:
raise HTTPException(status_code=404, detail="Gitea webhook integration is not configured")
body = await request.body()
if len(body) > 2_000_000:
raise HTTPException(status_code=413, detail="Webhook payload is too large")
signature = request.headers.get("x-gitea-signature", "").removeprefix("sha256=")
expected = hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest()
if not signature or not hmac.compare_digest(signature.casefold(), expected.casefold()):
raise HTTPException(status_code=401, detail="Invalid webhook signature")
try:
payload = json.loads(body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise HTTPException(status_code=400, detail="Webhook payload is invalid JSON") from exc
if not isinstance(payload, dict):
raise HTTPException(status_code=400, detail="Webhook payload must be an object")
repository_payload = payload.get("repository")
if not isinstance(repository_payload, dict):
raise HTTPException(status_code=400, detail="Webhook repository is missing")
raw_id = repository_payload.get("id")
repository_id = f"gitea:{raw_id}" if str(raw_id).isdigit() else None
repository = store.get_repository(repository_id) if repository_id else None
if repository is None:
return WebhookResponse(
accepted=False,
message="Repository is not present in the synchronized catalog",
)
event = request.headers.get("x-gitea-event", "push").casefold()
if event != "push":
return WebhookResponse(
accepted=True,
repository_id=repository.id,
message="Non-push webhook acknowledged without scheduling a scan",
)
ref = payload.get("ref")
if ref != f"refs/heads/{repository.default_branch}":
return WebhookResponse(
accepted=True,
repository_id=repository.id,
message="Non-default branch push acknowledged without scheduling a scan",
)
after = payload.get("after")
if (
not isinstance(after, str)
or not 7 <= len(after) <= 64
or not all(char in "0123456789abcdefABCDEF" for char in after)
or not set(after) - {"0"}
):
return WebhookResponse(
accepted=True,
repository_id=repository.id,
message="Push acknowledged without a usable commit revision",
)
if not gitea_client.configured:
raise HTTPException(status_code=503, detail="Gitea connector is not configured")
updated = repository.model_copy(update={"latest_revision": after, "sync_error": None})
store.upsert_repository(updated)
try:
job = _get_runner().enqueue(updated, trigger=ScanTrigger.WEBHOOK)
except MaintenanceModeError as exc:
raise HTTPException(status_code=423, detail=str(exc), headers={"Retry-After": "30"}) from exc
except QueueFullError as exc:
raise HTTPException(status_code=429, detail=str(exc), headers={"Retry-After": "5"}) from exc
return WebhookResponse(
accepted=True,
repository_id=repository.id,
job_id=job.id,
message="Push accepted and scan scheduled",
)
@app.post("/api/v1/operator/database-recovery/acknowledge", status_code=status.HTTP_204_NO_CONTENT)
def acknowledge_database_recovery(request: Request) -> Response:
_require_mutation_authorization(request, strict=True)
store.acknowledge_database_recovery()
return Response(status_code=status.HTTP_204_NO_CONTENT)
@app.get("/api/v1/scans", response_model=list[ScanReport])
def list_scans(limit: int = Query(default=20, ge=1, le=100)) -> list[ScanReport]:
return store.recent(limit)
@app.get("/api/v1/scans/{scan_id}", response_model=ScanReport)
def get_scan(scan_id: str) -> ScanReport:
report = store.get(scan_id)
if report is None:
raise HTTPException(status_code=404, detail="Scan not found")
return report
@app.get("/api/v1/scans/{scan_id}/comparison", response_model=ScanComparison)
def get_scan_comparison(scan_id: str) -> ScanComparison:
report = store.get(scan_id)
if report is None:
raise HTTPException(status_code=404, detail="Scan not found")
return store.compare(report)
@app.get("/api/v1/scans/{scan_id}/delta")
def get_scan_delta(scan_id: str) -> dict[str, int | None]:
report = store.get(scan_id)
if report is None:
raise HTTPException(status_code=404, detail="Scan not found")
previous = store.previous_for_repository(
report.source.repository_id or report.repository_path,
report.id,
)
return {
"previous_health_score": previous.health_score if previous else None,
"health_score_delta": report.health_score - previous.health_score if previous else None,
"previous_finding_count": len(previous.findings) if previous else None,
"finding_count_delta": len(report.findings) - len(previous.findings) if previous else None,
}
@app.get("/api/v1/trend", response_model=list[TrendPoint])
def trend(path: str, limit: int = Query(default=30, ge=1, le=100)) -> list[TrendPoint]:
try:
resolved = _resolve_scan_path(path)
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
return store.trend(str(resolved), limit)
@app.get("/api/v1/scans/{scan_id}/export")
def export_scan(scan_id: str, format: str = Query(default="html", pattern="^(html|json)$")) -> Response:
report = store.get(scan_id)
if report is None:
raise HTTPException(status_code=404, detail="Scan not found")
if format == "json":
filename = safe_report_filename(report.repository_name, "json")
return Response(
report.model_dump_json(indent=2),
media_type="application/json",
headers={"Content-Disposition": content_disposition(filename)},
)
filename = safe_report_filename(report.repository_name, "html")
return Response(
render_html_report(report),
media_type="text/html",
headers={"Content-Disposition": content_disposition(filename)},
)
static_dir = Path(os.getenv("REPOPULSE_STATIC_DIR", "/app/static"))
if static_dir.is_dir():
app.mount("/", StaticFiles(directory=static_dir, html=True), name="frontend")
+397
View File
@@ -0,0 +1,397 @@
from __future__ import annotations
import re
from datetime import datetime
from enum import StrEnum
from hashlib import sha256
from pydantic import BaseModel, Field
class Severity(StrEnum):
INFO = "info"
WARNING = "warning"
BLOCKER = "blocker"
class GateState(StrEnum):
PASS = "pass"
WARN = "warn"
BLOCK = "block"
class SourceOrigin(StrEnum):
LOCAL = "local"
GITEA = "gitea"
class ScanTrigger(StrEnum):
MANUAL = "manual"
SCHEDULED = "scheduled"
WEBHOOK = "webhook"
LOCAL = "local"
class ScanJobStatus(StrEnum):
QUEUED = "queued"
ACQUIRING_SOURCE = "acquiring_source"
SCANNING = "scanning"
ANALYZING = "analyzing"
PERSISTING = "persisting"
COMPLETED = "completed"
FAILED = "failed"
CANCELLED = "cancelled"
SUPERSEDED = "superseded"
class RepositoryState(StrEnum):
NEVER_SCANNED = "never_scanned"
CURRENT = "current"
STALE = "stale"
QUEUED = "queued"
SCANNING = "scanning"
FAILED = "failed"
UNAVAILABLE = "unavailable"
ARCHIVED = "archived"
class RepositoryPolicy(BaseModel):
auto_scan: bool = True
min_health_score: int = Field(default=0, ge=0, le=100)
max_blockers: int = Field(default=0, ge=0)
require_tests: bool = False
require_ci: bool = False
suppressions: list[str] = Field(default_factory=list)
class RemediationSignal(BaseModel):
"""A compact, redacted finding summary safe for catalog-level triage."""
code: str
category: str
severity: Severity
message: str
count: int = Field(default=1, ge=1)
guidance: FindingGuidance | None = None
class ScanSource(BaseModel):
origin: SourceOrigin = SourceOrigin.LOCAL
provider: str | None = None
repository_id: str | None = None
repository_full_name: str | None = None
branch: str | None = None
requested_revision: str | None = None
actual_revision: str | None = None
trigger: ScanTrigger = ScanTrigger.MANUAL
class Repository(BaseModel):
id: str
provider: str
provider_repository_id: str
owner: str
name: str
full_name: str
description: str | None = None
html_url: str | None = None
clone_url: str | None = None
default_branch: str = "main"
private: bool = False
archived: bool = False
available: bool = True
latest_revision: str | None = None
latest_revision_at: datetime | None = None
last_synced_at: datetime | None = None
sync_error: str | None = None
state: RepositoryState = RepositoryState.NEVER_SCANNED
stale: bool = False
active_job_id: str | None = None
last_scan_id: str | None = None
last_scan_at: datetime | None = None
last_scan_health: int | None = None
last_scan_readiness: str | None = None
last_scan_revision: str | None = None
policy: RepositoryPolicy = Field(default_factory=RepositoryPolicy)
class RepositorySummary(Repository):
"""Catalog row intentionally contains no full scan report."""
last_scan_blockers: int = 0
last_scan_warnings: int = 0
last_scan_primary_signal: RemediationSignal | None = None
last_scan_health_delta: int | None = None
last_scan_added_findings: int = 0
last_scan_resolved_findings: int = 0
last_scan_priority: int = 0
last_scan_priority_rationale: str | None = None
class RepositoryDetail(RepositorySummary):
latest_scan: ScanReport | None = None
class ScanJob(BaseModel):
id: str
repository_id: str
status: ScanJobStatus
trigger: ScanTrigger
requested_revision: str | None = None
actual_revision: str | None = None
phase: str = "queued"
progress: int = Field(default=0, ge=0, le=100)
message: str | None = None
created_at: datetime
started_at: datetime | None = None
updated_at: datetime
finished_at: datetime | None = None
duration_ms: float | None = None
scan_id: str | None = None
error_code: str | None = None
error_detail: str | None = None
attempt: int = 0
deduplicated: bool = False
class RepositoryScanRequest(BaseModel):
force: bool = False
class RepositorySyncResult(BaseModel):
provider: str
discovered: int = 0
created: int = 0
updated: int = 0
unavailable: int = 0
errors: list[str] = Field(default_factory=list)
partial: bool = False
branch_failures: int = 0
listing_complete: bool = True
synced_at: datetime
class WebhookResponse(BaseModel):
accepted: bool
repository_id: str | None = None
job_id: str | None = None
message: str
class FindingGuidance(BaseModel):
"""Backend-owned, safe-to-display explanation of a deterministic rule."""
remediation_class: str
confidence: str
why: str
recommended_action: str
priority: int = Field(default=0, ge=0, le=1000)
priority_rationale: str
class Finding(BaseModel):
code: str
category: str
severity: Severity
message: str
path: str | None = None
line: int | None = None
deduction: int = Field(default=0, ge=0)
fingerprint: str | None = None
suppressed: bool = False
guidance: FindingGuidance | None = None
def finding_fingerprint(finding: Finding) -> str:
"""Build a stable semantic identity that ignores line movement and counters."""
normalized_message = re.sub(r"\b\d+(?:\.\d+)?\b", "#", finding.message.casefold())
payload = "\0".join(
(
finding.code,
finding.category,
finding.path or "",
normalized_message,
)
)
return sha256(payload.encode("utf-8")).hexdigest()[:32]
class RepositoryMetrics(BaseModel):
files: int = 0
source_files: int = 0
lines_of_code: int = 0
test_files: int = 0
todo: int = 0
fixme: int = 0
hack: int = 0
xxx: int = 0
dependency_manifests: int = 0
declared_dependencies: int = 0
large_files: int = 0
large_source_files: int = 0
class TestSignals(BaseModel):
frameworks: list[str] = Field(default_factory=list)
coverage_artifacts: list[str] = Field(default_factory=list)
ci_workflows: list[str] = Field(default_factory=list)
ci_validation_workflows: list[str] = Field(default_factory=list)
ci_build_workflows: list[str] = Field(default_factory=list)
ci_deploy_only_workflows: list[str] = Field(default_factory=list)
coverage_evidence: list[str] = Field(default_factory=list)
test_source_ratio: float | None = None
class GitSignals(BaseModel):
available: bool = False
commit_count: int | None = None
contributor_count: int | None = None
latest_commit_at: datetime | None = None
tracked_files: int | None = None
renamed_files: list[str] = Field(default_factory=list)
class DependencySignals(BaseModel):
manifests: list[str] = Field(default_factory=list)
lockfiles: list[str] = Field(default_factory=list)
declared_dependencies: int = 0
unbounded_dependencies: int = 0
unpinned_dependencies: int = 0
duplicate_dependencies: list[str] = Field(default_factory=list)
included_requirements: list[str] = Field(default_factory=list)
class ArchitectureNode(BaseModel):
id: str
label: str
path: str
language: str
inbound: int = 0
outbound: int = 0
visible_inbound: int = 0
visible_outbound: int = 0
hotspot_score: int = Field(default=0, ge=0, le=100)
class ArchitectureEdge(BaseModel):
source: str
target: str
kind: str = "import"
class ArchitectureGraph(BaseModel):
nodes: list[ArchitectureNode] = Field(default_factory=list)
edges: list[ArchitectureEdge] = Field(default_factory=list)
clusters: dict[str, int] = Field(default_factory=dict)
truncated: bool = False
total_nodes: int = 0
total_edges: int = 0
class Hotspot(BaseModel):
path: str
language: str
lines_of_code: int
debt_markers: int = 0
churn: int = 0
inbound: int = 0
outbound: int = 0
score: int = Field(ge=0, le=100)
reasons: list[str] = Field(default_factory=list)
class GateCheck(BaseModel):
key: str
label: str
state: GateState
detail: str
class ReleaseGate(BaseModel):
status: str
checks: list[GateCheck]
class ScanLimits(BaseModel):
max_files: int = Field(default=10_000, ge=1, le=100_000)
max_text_bytes_per_file: int = Field(default=2_000_000, ge=1_024, le=10_000_000)
max_total_text_bytes: int = Field(default=50_000_000, ge=1_000_000, le=500_000_000)
max_findings: int = Field(default=2_000, ge=10, le=20_000)
max_secret_findings: int = Field(default=20, ge=1, le=500)
class ScanCoverage(BaseModel):
limits: ScanLimits
discovered_files: int = 0
scanned_files: int = 0
skipped_files_by_budget: int = 0
skipped_files_by_size: int = 0
skipped_files_by_text_budget: int = 0
text_bytes_scanned: int = 0
truncated: bool = False
notes: list[str] = Field(default_factory=list)
class ScanReport(BaseModel):
id: str
repository_path: str
repository_name: str
source: ScanSource = Field(default_factory=ScanSource)
policy: RepositoryPolicy = Field(default_factory=RepositoryPolicy)
scanned_at: datetime
duration_ms: float
metrics: RepositoryMetrics
languages: dict[str, int]
category_scores: dict[str, int]
health_score: int
release_readiness: str
findings: list[Finding]
tests: TestSignals = Field(default_factory=TestSignals)
git: GitSignals = Field(default_factory=GitSignals)
dependencies: DependencySignals = Field(default_factory=DependencySignals)
architecture: ArchitectureGraph = Field(default_factory=ArchitectureGraph)
hotspots: list[Hotspot] = Field(default_factory=list)
release_gate: ReleaseGate | None = None
coverage: ScanCoverage | None = None
compacted: bool = False
class ScanSummary(BaseModel):
id: str
repository_id: str
scanned_at: datetime
branch: str | None = None
requested_revision: str | None = None
actual_revision: str | None = None
trigger: ScanTrigger = ScanTrigger.MANUAL
health_score: int
release_readiness: str
findings: int
duration_ms: float
class ScanRequest(BaseModel):
path: str = Field(min_length=1)
class ScanComparison(BaseModel):
current_scan_id: str
previous_scan_id: str | None = None
health_score_delta: int | None = None
category_score_deltas: dict[str, int] = Field(default_factory=dict)
metric_deltas: dict[str, int] = Field(default_factory=dict)
added_findings: list[Finding] = Field(default_factory=list)
resolved_findings: list[Finding] = Field(default_factory=list)
class TrendPoint(BaseModel):
scan_id: str
scanned_at: datetime
health_score: int
findings: int
blockers: int
warnings: int
debt_markers: int
RepositoryDetail.model_rebuild()
+98
View File
@@ -0,0 +1,98 @@
from __future__ import annotations
from pathlib import Path
from .models import RepositoryPolicy
POLICY_NAMES = (".repopulse.yml", ".repopulse.yaml")
MAX_POLICY_BYTES = 64_000
def _scalar(value: str) -> bool | int | str:
normalized = value.strip().strip('"\'')
lowered = normalized.casefold()
if lowered in {"true", "yes", "on"}:
return True
if lowered in {"false", "no", "off"}:
return False
try:
return int(normalized)
except ValueError:
return normalized
def load_policy(root: Path) -> RepositoryPolicy:
policy_path = next((root / name for name in POLICY_NAMES if (root / name).is_file()), None)
if policy_path is None:
return RepositoryPolicy()
try:
if policy_path.stat().st_size > MAX_POLICY_BYTES:
return RepositoryPolicy()
lines = policy_path.read_text(encoding="utf-8", errors="replace").splitlines()
except OSError:
return RepositoryPolicy()
values: dict[str, object] = {}
suppressions: list[str] = []
in_suppressions = False
for raw_line in lines:
line = raw_line.split("#", 1)[0].rstrip()
if not line.strip():
continue
stripped = line.strip()
if stripped == "suppressions:":
in_suppressions = True
continue
if in_suppressions and stripped.startswith("-"):
value = stripped[1:].strip().strip('"\'')
if value:
suppressions.append(value)
continue
in_suppressions = False
if ":" not in stripped:
continue
key, value = stripped.split(":", 1)
key = key.strip()
if key in {"auto_scan", "min_health_score", "max_blockers", "require_tests", "require_ci"}:
values[key] = _scalar(value)
values["suppressions"] = suppressions
try:
return RepositoryPolicy.model_validate(values)
except ValueError:
return RepositoryPolicy()
def apply_policy(report, policy: RepositoryPolicy):
findings = [
item.model_copy(
update={
"suppressed": item.suppressed or item.code in set(policy.suppressions),
}
)
for item in report.findings
]
active_findings = [item for item in findings if not item.suppressed]
scores = {category: 100 for category in report.category_scores}
for finding in active_findings:
scores[finding.category] = max(0, scores[finding.category] - finding.deduction)
health_score = round(sum(scores.values()) / len(scores)) if scores else 100
from .release_gate import build_release_gate
release_gate = build_release_gate(
health_score=health_score,
findings=findings,
metrics=report.metrics,
tests=report.tests,
dependencies=report.dependencies,
policy=policy,
)
return report.model_copy(
update={
"findings": findings,
"category_scores": scores,
"health_score": health_score,
"release_readiness": release_gate.status,
"release_gate": release_gate,
"policy": policy,
}
)
+147
View File
@@ -0,0 +1,147 @@
from __future__ import annotations
from .models import (
DependencySignals,
Finding,
GateCheck,
GateState,
ReleaseGate,
RepositoryMetrics,
RepositoryPolicy,
ScanCoverage,
Severity,
TestSignals,
)
def build_release_gate(
*,
health_score: int,
findings: list[Finding],
metrics: RepositoryMetrics,
tests: TestSignals,
dependencies: DependencySignals,
coverage: ScanCoverage | None = None,
policy: RepositoryPolicy | None = None,
) -> ReleaseGate:
policy = policy or RepositoryPolicy()
active_findings = [item for item in findings if not item.suppressed]
blockers = [item for item in active_findings if item.severity == Severity.BLOCKER]
warnings = [item for item in active_findings if item.severity == Severity.WARNING]
dependency_issues = dependencies.unbounded_dependencies + dependencies.unpinned_dependencies
checks = [
GateCheck(
key="security",
label="Security basics",
state=GateState.BLOCK if len(blockers) > policy.max_blockers else GateState.PASS,
detail=(
f"{len(blockers)} blocker finding(s), policy allows {policy.max_blockers}"
if blockers
else "No blocker-level secret/environment findings"
),
),
GateCheck(
key="tests",
label="Tests" + (" (required)" if policy.require_tests else ""),
state=(
GateState.BLOCK
if policy.require_tests and not metrics.test_files
else GateState.PASS
if metrics.test_files
else GateState.WARN
),
detail=(
f"{metrics.test_files} test file(s) detected"
if metrics.test_files
else "Tests are required by repository policy"
if policy.require_tests
else "No test files detected"
),
),
GateCheck(
key="ci",
label="Continuous integration" + (" (required)" if policy.require_ci else ""),
state=(
GateState.BLOCK
if policy.require_ci and not tests.ci_validation_workflows
else GateState.PASS
if tests.ci_validation_workflows
else GateState.WARN
),
detail=(
f"{len(tests.ci_validation_workflows)} validation workflow(s) detected"
if tests.ci_validation_workflows
else f"{len(tests.ci_deploy_only_workflows)} deploy-only workflow(s) do not provide validation evidence"
if tests.ci_deploy_only_workflows
else "CI is required by repository policy"
if policy.require_ci
else "No validation workflow detected"
),
),
GateCheck(
key="coverage",
label="Coverage evidence",
state=(
GateState.PASS
if tests.coverage_evidence
else GateState.WARN
if metrics.test_files
else GateState.PASS
),
detail=(
f"{len(tests.coverage_evidence)} coverage evidence source(s) detected"
if tests.coverage_evidence
else "Tests exist but no coverage-producing command or retained coverage output was detected"
if metrics.test_files
else "No test suite is present to produce coverage evidence"
),
),
GateCheck(
key="analysis_coverage",
label="Analysis coverage",
state=GateState.WARN if coverage and coverage.truncated else GateState.PASS,
detail=(
"; ".join(coverage.notes) if coverage and coverage.truncated else "Analysis completed within configured limits"
),
),
GateCheck(
key="dependencies",
label="Dependency hygiene",
state=GateState.WARN if dependency_issues else GateState.PASS,
detail=(
f"{dependencies.unbounded_dependencies} unbounded and {dependencies.unpinned_dependencies} unpinned declaration(s)"
if dependency_issues
else f"{dependencies.declared_dependencies} dependency declaration(s), no unbounded or unpinned versions"
),
),
GateCheck(
key="health",
label="Repository health",
state=(
GateState.PASS
if health_score >= max(85, policy.min_health_score)
else GateState.WARN
if health_score >= max(65, policy.min_health_score)
else GateState.BLOCK
),
detail=(
f"Health score {health_score}/100"
if not policy.min_health_score
else f"Health score {health_score}/100; policy minimum {policy.min_health_score}"
),
),
GateCheck(
key="warnings",
label="Engineering warnings",
state=GateState.WARN if warnings else GateState.PASS,
detail=f"{len(warnings)} warning finding(s)" if warnings else "No warning-level findings",
),
]
if any(check.state == GateState.BLOCK for check in checks):
status = "BLOCKED"
elif any(check.state == GateState.WARN for check in checks):
status = "READY WITH WARNINGS"
else:
status = "READY"
return ReleaseGate(status=status, checks=checks)
+75
View File
@@ -0,0 +1,75 @@
from __future__ import annotations
import re
import unicodedata
from html import escape
from urllib.parse import quote
from .models import ScanReport
INVALID_FILENAME_CHARS = re.compile(r'[\x00-\x1f\x7f<>:"/\\|?*]')
REPORT_MARK = """<svg class=\"brand-mark\" viewBox=\"0 0 32 32\" aria-hidden=\"true\" xmlns=\"http://www.w3.org/2000/svg\"><rect width=\"32\" height=\"32\" rx=\"8\" fill=\"#123245\"/><path d=\"M7 9.5h5.1l2.2 2.2H25M7 22.5h5.1l2.2-2.2H25M12.8 16h2.1l1.55-3.1 2.15 6.2 1.35-3.1H25\" fill=\"none\" stroke=\"#91ebe0\" stroke-width=\"1.8\" stroke-linecap=\"round\" stroke-linejoin=\"round\"/><circle cx=\"7\" cy=\"9.5\" r=\"2\" fill=\"#91ebe0\"/><circle cx=\"7\" cy=\"22.5\" r=\"2\" fill=\"#91ebe0\"/><circle cx=\"25\" cy=\"16\" r=\"2\" fill=\"#91ebe0\"/></svg>"""
def safe_report_filename(repository_name: str, extension: str) -> str:
cleaned = INVALID_FILENAME_CHARS.sub("_", repository_name).strip(" .")
cleaned = cleaned[:100].rstrip(" .") or "repository"
return f"repopulse-{cleaned}.{extension}"
def content_disposition(filename: str) -> str:
fallback = unicodedata.normalize("NFKD", filename).encode("ascii", "ignore").decode("ascii")
fallback = INVALID_FILENAME_CHARS.sub("_", fallback).strip(" .") or "repopulse-report"
return f"attachment; filename=\"{fallback}\"; filename*=UTF-8''{quote(filename, safe='')}"
def render_html_report(report: ScanReport) -> str:
category_rows = "".join(
f"<tr><td>{escape(name)}</td><td>{score}/100</td></tr>"
for name, score in report.category_scores.items()
)
finding_rows = "".join(
"<tr>"
f"<td>{escape(item.severity.value)}</td>"
f"<td>{escape(item.category)}</td>"
f"<td>{escape(item.message)}</td>"
f"<td>{escape(item.path or '')}{':' + str(item.line) if item.line else ''}</td>"
"</tr>"
for item in report.findings
)
hotspot_rows = "".join(
"<tr>"
f"<td>{escape(item.path)}</td><td>{escape(item.language)}</td>"
f"<td>{item.score}</td><td>{item.lines_of_code}</td><td>{item.churn}</td>"
"</tr>"
for item in report.hotspots[:25]
)
return f"""<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<title>RepoPulse report — {escape(report.repository_name)}</title>
<style>
*{{box-sizing:border-box}}body{{font-family:Inter,ui-sans-serif,system-ui,sans-serif;margin:42px auto;max-width:1120px;color:#172638;line-height:1.45}}
.brand{{display:flex;align-items:center;gap:10px;margin-bottom:34px}}.brand-mark{{width:34px;height:34px;flex:0 0 auto}}.brand strong{{display:block;font-size:15px;letter-spacing:-.03em}}.brand small{{display:block;color:#66808d;font-size:8px;font-weight:700;letter-spacing:.13em;text-transform:uppercase}}
.hero{{display:flex;align-items:end;justify-content:space-between;gap:20px;padding:23px 25px;border:1px solid #dce8e8;border-radius:18px;background:linear-gradient(120deg,#effafa,#f9fcfc)}}h1{{margin:0 0 4px;font-size:30px;letter-spacing:-.04em}}h2{{margin-top:36px;font-size:18px;letter-spacing:-.02em}}.muted{{color:#607588}}.score{{color:#176c70;font-size:54px;font-weight:750;letter-spacing:-.08em}}.readiness{{color:#176c70;font-size:11px;font-weight:750;letter-spacing:.08em;text-transform:uppercase}}
table{{width:100%;border-collapse:collapse;margin:14px 0 34px}}th,td{{text-align:left;padding:10px 9px;border-bottom:1px solid #e4ebef;vertical-align:top}}th{{color:#607588;font-size:10px;letter-spacing:.06em;text-transform:uppercase}}.grid{{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:1px;overflow:hidden;border:1px solid #dce8e8;border-radius:13px;background:#dce8e8}}.card{{padding:15px;background:#fff}}.card strong{{color:#176c70;font-size:24px}}
@media print{{body{{margin:18mm;max-width:none}}.hero{{background:#fff}}}}@media(max-width:760px){{.grid{{grid-template-columns:1fr 1fr}}body{{margin:22px}}.hero{{align-items:start;flex-direction:column}}}}
</style>
</head>
<body>
<div class="brand">{REPORT_MARK}<span><strong>RepoPulse</strong><small>Repository intelligence</small></span></div>
<section class="hero"><div><p class="muted">Deterministic repository intelligence report</p><h1>{escape(report.repository_name)}</h1><p class="muted">{escape(report.repository_path)} · {escape(report.scanned_at.isoformat())}</p></div><div><div class="score">{report.health_score}</div><div class="readiness">{escape(report.release_readiness)}</div></div></section>
<div class="grid">
<div class="card"><strong>{report.metrics.source_files}</strong><br>source files</div>
<div class="card"><strong>{report.metrics.lines_of_code}</strong><br>LOC</div>
<div class="card"><strong>{report.metrics.test_files}</strong><br>test files</div>
<div class="card"><strong>{len(report.findings)}</strong><br>findings</div>
</div>
<h2>Category scores</h2><table><tbody>{category_rows}</tbody></table>
<h2>Hotspots</h2><table><thead><tr><th>Path</th><th>Language</th><th>Score</th><th>LOC</th><th>Churn</th></tr></thead><tbody>{hotspot_rows}</tbody></table>
<h2>Findings</h2><table><thead><tr><th>Severity</th><th>Category</th><th>Finding</th><th>Location</th></tr></thead><tbody>{finding_rows}</tbody></table>
</body>
</html>"""
+83
View File
@@ -0,0 +1,83 @@
from .models import Finding, FindingGuidance, Severity
_RULES: dict[str, tuple[str, str, str, str]] = {
"security.secret-assignment": (
"security remediation",
"A credential-shaped assignment was found in tracked content.",
"Move the value to a secret store or runtime environment and rotate it if it was genuine.",
"high",
),
"security.env-committed": (
"security remediation",
"A live environment file is tracked, so its contents require deliberate review.",
"Remove the live file from version control and retain a sanitized .env.example template.",
"high",
),
"testing.none-detected": (
"test coverage",
"No conventionally named test source was detected for a repository with source files.",
"Add a focused deterministic smoke test, or record a repository policy that explains why tests do not apply.",
"medium",
),
"testing.no-ci": (
"delivery evidence",
"No repository workflow was found that contains a validation command.",
"Add a CI validation workflow that runs the repository's deterministic checks.",
"medium",
),
"dependency.no-lockfile": (
"dependency reproducibility",
"A JavaScript manifest was found without a lockfile in its package or workspace root.",
"Commit the package-manager lockfile used by the build.",
"high",
),
"hygiene.missing-gitignore": (
"repository hygiene",
"The repository has no root ignore policy.",
"Add a minimal .gitignore for generated files, local settings and build outputs.",
"high",
),
}
def _default_rule(finding: Finding) -> tuple[str, str, str, str]:
if finding.code.startswith("dependency."):
return (
"dependency hygiene",
"The dependency declaration has a deterministic hygiene signal.",
"Review the declared version and lockfile semantics, then make the smallest reproducible correction.",
"medium",
)
if finding.code.startswith("architecture."):
return (
"architecture review",
"The bounded static graph has a structural signal.",
"Inspect the affected module boundary before planning a contained refactor.",
"medium",
)
return (
"engineering hygiene",
"A deterministic repository rule produced this signal.",
"Review the retained evidence and make the smallest safe correction.",
"medium",
)
def enrich_finding(finding: Finding) -> Finding:
remediation_class, why, action, confidence = _RULES.get(finding.code, _default_rule(finding))
severity_weight = {Severity.BLOCKER: 700, Severity.WARNING: 400, Severity.INFO: 100}[finding.severity]
priority = min(1000, severity_weight + finding.deduction * 5)
rationale = f"{finding.severity.value.title()} signal; {finding.deduction} health-point impact."
guidance = FindingGuidance(
remediation_class=remediation_class,
confidence=confidence,
why=why,
recommended_action=action,
priority=priority,
priority_rationale=rationale,
)
return finding.model_copy(update={"guidance": guidance})
def enrich_findings(findings: list[Finding]) -> list[Finding]:
return [enrich_finding(item) for item in findings]
+695
View File
@@ -0,0 +1,695 @@
from __future__ import annotations
import ast
import fnmatch
import os
import re
import subprocess
from collections import Counter
from datetime import UTC, datetime
from pathlib import Path
from time import perf_counter
from uuid import uuid4
from .architecture import build_architecture
from .git_analysis import analyze_git
from .models import (
Finding,
RepositoryMetrics,
RepositoryPolicy,
ScanCoverage,
ScanLimits,
ScanReport,
ScanSource,
Severity,
finding_fingerprint,
)
from .policy import load_policy
from .release_gate import build_release_gate
from .rules import enrich_findings
from .signals import analyze_dependencies, detect_test_signals
LANGUAGES = {
".py": "Python",
".ts": "TypeScript",
".tsx": "TypeScript",
".js": "JavaScript",
".jsx": "JavaScript",
".mjs": "JavaScript",
".cjs": "JavaScript",
".cs": "C#",
".java": "Java",
".go": "Go",
".rs": "Rust",
".cpp": "C++",
".cc": "C++",
".c": "C",
".h": "C/C++ Header",
".hpp": "C++ Header",
".php": "PHP",
".rb": "Ruby",
".kt": "Kotlin",
".swift": "Swift",
".vue": "Vue",
".svelte": "Svelte",
".html": "HTML",
".css": "CSS",
".scss": "SCSS",
".sh": "Shell",
".ps1": "PowerShell",
".sql": "SQL",
}
IGNORED_DIRS = {
".git",
".idea",
".vscode",
".venv",
"venv",
"node_modules",
"dist",
"build",
"coverage",
".next",
".nuxt",
".pytest_cache",
"__pycache__",
"vendor",
"target",
"bin",
"obj",
}
MAX_TEXT_BYTES = 2_000_000
MAX_SECRET_FINDINGS = 20
DEBT_PATTERN = re.compile(r"\b(TODO|FIXME|HACK|XXX)\b")
SECRET_PATTERN = re.compile(
r"""(?ix)
(?:
^\s*(?:export\s+)?(?:(?:const|let|var|final|static|private|public|protected|internal)\s+)?(?:[A-Za-z_]\w*\.)*
|[,{]\s*
)
[\"']?(?P<name>api[_-]?key|secret|password|passwd|token|access[_-]?key)[\"']?
\s*[:=]\s*[\"']?(?P<value>[^\s\"']{8,})
"""
)
PLACEHOLDER_SECRET_MARKERS = ("example", "placeholder", "dummy", "changeme", "not-a-real", "replace-me", "your-", "fake", "test-")
SECRET_REFERENCE_MARKERS = (
"os.environ",
"os.getenv",
"getenv",
"process.env",
"env[",
"config.",
"settings.",
)
SECRET_SCAN_CONFIG_SUFFIXES = {".json", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".conf", ".md", ".txt"}
CATEGORY_NAMES = (
"Architecture",
"Testing",
"Maintainability",
"Documentation",
"Dependencies",
"Repository hygiene",
"Security basics",
)
def _ignored(relative: str) -> bool:
return any(part.casefold() in IGNORED_DIRS for part in Path(relative).parts)
def _is_env_template(name: str) -> bool:
return name == ".env.example" or (name.startswith(".env.") and name.endswith(".example"))
def _is_secret_scan_candidate(path_obj: Path, relative: str, language: str | None) -> bool:
return language is not None or path_obj.suffix.casefold() in SECRET_SCAN_CONFIG_SUFFIXES or path_obj.name.casefold().startswith(".env")
def _bounded_env_int(name: str, default: int, *, minimum: int, maximum: int) -> int:
try:
value = int(os.getenv(name, str(default)))
except ValueError:
value = default
return min(maximum, max(minimum, value))
def scan_limits_from_environment() -> ScanLimits:
"""Return bounded analysis limits; input cannot turn a scan into an unbounded job."""
return ScanLimits(
max_files=_bounded_env_int("REPOPULSE_SCAN_MAX_FILES", 10_000, minimum=1, maximum=100_000),
max_text_bytes_per_file=_bounded_env_int(
"REPOPULSE_SCAN_MAX_TEXT_BYTES", MAX_TEXT_BYTES, minimum=1_024, maximum=10_000_000
),
max_total_text_bytes=_bounded_env_int(
"REPOPULSE_SCAN_MAX_TOTAL_TEXT_BYTES", 50_000_000, minimum=1_000_000, maximum=500_000_000
),
max_findings=_bounded_env_int("REPOPULSE_SCAN_MAX_FINDINGS", 2_000, minimum=10, maximum=20_000),
max_secret_findings=_bounded_env_int(
"REPOPULSE_SCAN_MAX_SECRET_FINDINGS", MAX_SECRET_FINDINGS, minimum=1, maximum=500
),
)
class _GitIgnoreMatcher:
"""Bounded root/nested .gitignore support for non-Git directory scans."""
def __init__(self, root: Path) -> None:
self.rules: list[tuple[str, str, bool, bool, bool]] = []
for current, dirs, names in os.walk(root, followlinks=False):
dirs[:] = [
name
for name in dirs
if name.casefold() not in IGNORED_DIRS and not (Path(current) / name).is_symlink()
]
if ".gitignore" not in names:
continue
ignore_file = Path(current) / ".gitignore"
try:
lines = ignore_file.read_text(encoding="utf-8", errors="replace").splitlines()
base_path = Path(current).relative_to(root)
base = "" if str(base_path) == "." else base_path.as_posix()
except (OSError, ValueError):
continue
for raw in lines:
pattern = raw.strip()
if not pattern or pattern.startswith("#"):
continue
negated = pattern.startswith("!")
if negated:
pattern = pattern[1:]
pattern = pattern.replace("\\", "/")
anchored = pattern.startswith("/")
pattern = pattern.lstrip("/")
dir_only = pattern.endswith("/")
pattern = pattern.rstrip("/")
if pattern:
self.rules.append((base, pattern, negated, dir_only, anchored))
def is_ignored(self, relative: str, *, is_dir: bool) -> bool:
normalized = relative.replace("\\", "/").strip("/")
ignored = False
for base, pattern, negated, dir_only, anchored in self.rules:
if dir_only and not is_dir:
continue
if base:
if normalized == base:
target = ""
elif normalized.startswith(f"{base}/"):
target = normalized[len(base) + 1 :]
else:
continue
else:
target = normalized
if not target:
continue
if "/" not in pattern:
matched = (
fnmatch.fnmatch(target, pattern)
if anchored
else any(fnmatch.fnmatch(part, pattern) for part in target.split("/")
)
)
else:
matched = fnmatch.fnmatch(target, pattern)
if not matched and pattern.startswith("**/"):
matched = fnmatch.fnmatch(target, pattern[3:])
if matched:
ignored = not negated
return ignored
def _git_worktree_root(root: Path) -> Path | None:
try:
result = subprocess.run(
["git", "-C", str(root), "rev-parse", "--show-toplevel"],
check=True,
capture_output=True,
timeout=5,
)
except (OSError, subprocess.SubprocessError):
return None
try:
return Path(result.stdout.decode("utf-8", errors="surrogateescape").strip()).resolve()
except (OSError, ValueError):
return None
def _safe_file(root: Path, candidate: Path) -> Path | None:
"""Return a resolved regular file only when every path component is safe."""
try:
relative = candidate.relative_to(root)
if _ignored(str(relative)):
return None
current = root
for part in Path(relative).parts:
current /= part
if current.is_symlink():
return None
resolved = candidate.resolve(strict=True)
root_resolved = root.resolve()
if resolved == root_resolved or root_resolved not in resolved.parents:
return None
if not resolved.is_file():
return None
except (OSError, RuntimeError, ValueError):
return None
return resolved
def _tracked_files(root: Path) -> list[Path] | None:
if _git_worktree_root(root) != root:
return None
try:
result = subprocess.run(
["git", "-C", str(root), "ls-files", "-z"],
check=True,
capture_output=True,
timeout=5,
)
except (OSError, subprocess.SubprocessError):
return None
paths: list[Path] = []
for raw in result.stdout.split(b"\0"):
if not raw:
continue
relative = raw.decode("utf-8", errors="surrogateescape")
relative_path = Path(relative)
if relative_path.is_absolute() or _ignored(relative):
continue
safe = _safe_file(root, root / relative_path)
if safe is not None:
paths.append(safe)
return paths
def _walk_files(root: Path) -> list[Path]:
files: list[Path] = []
gitignore = _GitIgnoreMatcher(root)
for current, dirs, names in os.walk(root, followlinks=False):
current_path = Path(current)
dirs[:] = [
name
for name in dirs
if name.casefold() not in IGNORED_DIRS
and not (current_path / name).is_symlink()
and not gitignore.is_ignored(
str((current_path / name).relative_to(root)),
is_dir=True,
)
]
for name in names:
candidate = current_path / name
try:
relative = str(candidate.relative_to(root))
except (OSError, ValueError):
continue
if _ignored(relative):
continue
if gitignore.is_ignored(relative, is_dir=False):
continue
safe = _safe_file(root, candidate)
if safe is not None:
files.append(safe)
return files
def _read_text(path: Path, max_bytes: int = MAX_TEXT_BYTES) -> str | None:
try:
if path.stat().st_size > max_bytes:
return None
raw = path.read_bytes()
except OSError:
return None
if b"\0" in raw[:8192]:
return None
return raw.decode("utf-8", errors="replace")
def _is_test_file(relative: str) -> bool:
normalized = relative.replace("\\", "/").lower()
name = Path(normalized).name
return (
"/tests/" in f"/{normalized}"
or "/test/" in f"/{normalized}"
or name.startswith("test_")
or ".test." in name
or ".spec." in name
or name.endswith(("tests.cs", "test.cs", "_test.go"))
)
def _python_function_findings(relative: str, text: str) -> list[Finding]:
try:
tree = ast.parse(text)
except SyntaxError:
return []
findings: list[Finding] = []
branch_types = (ast.If, ast.For, ast.AsyncFor, ast.While, ast.Try, ast.Match, ast.BoolOp)
for node in ast.walk(tree):
if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
continue
end_line = getattr(node, "end_lineno", node.lineno)
length = max(1, end_line - node.lineno + 1)
branches = sum(1 for child in ast.walk(node) if isinstance(child, branch_types))
if length >= 140:
findings.append(
Finding(
code="maintainability.long-function",
category="Maintainability",
severity=Severity.WARNING,
message=f"Function {node.name} spans {length} lines",
path=relative,
line=node.lineno,
deduction=4,
)
)
if branches >= 20:
findings.append(
Finding(
code="maintainability.branch-heavy-function",
category="Maintainability",
severity=Severity.WARNING,
message=f"Function {node.name} contains {branches} branch constructs",
path=relative,
line=node.lineno,
deduction=4,
)
)
return findings[:12]
def _is_placeholder_secret(match: re.Match[str]) -> bool:
value = match.group("value").casefold().strip("`$(){}[]<>")
exact_placeholder = value in {"", "none", "null", "undefined", "xxx", "xxxxx"}
obvious_placeholder = any(value.startswith(marker) for marker in PLACEHOLDER_SECRET_MARKERS)
return exact_placeholder or obvious_placeholder or any(
value.startswith(marker) for marker in SECRET_REFERENCE_MARKERS
)
def scan_repository(
path: str,
*,
source: ScanSource | None = None,
display_name: str | None = None,
policy: RepositoryPolicy | None = None,
limits: ScanLimits | None = None,
) -> ScanReport:
started = perf_counter()
root = Path(path).expanduser().resolve()
if not root.exists() or not root.is_dir():
raise ValueError("Repository path does not exist or is not a directory")
limits = limits or scan_limits_from_environment()
tracked_files = _tracked_files(root)
all_files = _walk_files(root) if tracked_files is None else tracked_files
all_files.sort(key=lambda item: str(item.relative_to(root)).replace("\\", "/").casefold())
files = all_files[: limits.max_files]
coverage = ScanCoverage(
limits=limits,
discovered_files=len(all_files),
skipped_files_by_budget=max(0, len(all_files) - len(files)),
truncated=len(all_files) > len(files),
notes=(
[f"File analysis limited to {limits.max_files} of {len(all_files)} discovered safe files."]
if len(all_files) > len(files)
else []
),
)
languages: Counter[str] = Counter()
findings: list[Finding] = []
metrics = RepositoryMetrics(files=len(all_files))
files_by_name: dict[str, list[Path]] = {}
root_entries = {entry.name.lower() for entry in root.iterdir()}
text_by_path: dict[str, str] = {}
source_texts: dict[str, str] = {}
language_by_path: dict[str, str] = {}
loc_by_path: dict[str, int] = {}
debt_by_path: dict[str, int] = {}
secret_findings = 0
secret_findings_truncated = False
text_budget_exhausted = False
for path_obj in files:
try:
relative = str(path_obj.relative_to(root)).replace("\\", "/")
size = path_obj.stat().st_size
except (OSError, ValueError):
continue
files_by_name.setdefault(path_obj.name, []).append(path_obj)
suffix = path_obj.suffix.lower()
language = LANGUAGES.get(suffix)
if size >= 1_000_000:
metrics.large_files += 1
findings.append(
Finding(
code="hygiene.large-file",
category="Repository hygiene",
severity=Severity.INFO,
message=f"Large tracked file ({size / 1_000_000:.1f} MB)",
path=relative,
deduction=1,
)
)
name = path_obj.name.casefold()
if name == ".env" or (name.startswith(".env.") and not _is_env_template(name)):
findings.append(
Finding(
code="security.env-committed",
category="Security basics",
severity=Severity.BLOCKER,
message="Environment file is tracked; review it for credentials",
path=relative,
deduction=30,
)
)
if size > limits.max_text_bytes_per_file:
coverage.skipped_files_by_size += 1
continue
if coverage.text_bytes_scanned + size > limits.max_total_text_bytes:
coverage.skipped_files_by_text_budget += 1
text_budget_exhausted = True
continue
text = _read_text(path_obj, limits.max_text_bytes_per_file)
if text is None:
continue
coverage.scanned_files += 1
coverage.text_bytes_scanned += len(text.encode("utf-8", errors="replace"))
text_by_path[relative] = text
if language:
loc = sum(1 for line in text.splitlines() if line.strip())
metrics.source_files += 1
metrics.lines_of_code += loc
languages[language] += loc
source_texts[relative] = text
language_by_path[relative] = language
loc_by_path[relative] = loc
if loc > 1000:
metrics.large_source_files += 1
findings.append(
Finding(
code="maintainability.large-source",
category="Maintainability",
severity=Severity.WARNING,
message=f"Source file has {loc} non-empty lines",
path=relative,
deduction=5,
)
)
if language == "Python":
findings.extend(_python_function_findings(relative, text))
if _is_test_file(relative):
metrics.test_files += 1
debt = Counter(DEBT_PATTERN.findall(text))
metrics.todo += debt["TODO"]
metrics.fixme += debt["FIXME"]
metrics.hack += debt["HACK"]
metrics.xxx += debt["XXX"]
debt_by_path[relative] = sum(debt.values())
if _is_secret_scan_candidate(path_obj, relative, language) and secret_findings < limits.max_secret_findings:
for line_number, line in enumerate(text.splitlines(), start=1):
match = SECRET_PATTERN.search(line)
if match is None or _is_placeholder_secret(match):
continue
findings.append(
Finding(
code="security.secret-assignment",
category="Security basics",
severity=Severity.BLOCKER,
message="Potential credential assignment detected; value intentionally redacted",
path=relative,
line=line_number,
deduction=20,
)
)
secret_findings += 1
if secret_findings >= limits.max_secret_findings:
secret_findings_truncated = True
break
if text_budget_exhausted:
coverage.truncated = True
coverage.notes.append(
f"Text analysis limited to {limits.max_total_text_bytes} bytes across repository files."
)
if secret_findings_truncated:
findings.append(
Finding(
code="security.secret-findings-truncated",
category="Security basics",
severity=Severity.INFO,
message=f"Potential credential findings are capped at {limits.max_secret_findings}",
deduction=0,
)
)
if not any(name.startswith("readme") for name in root_entries):
findings.append(Finding(code="docs.missing-readme", category="Documentation", severity=Severity.WARNING, message="No root README detected", deduction=20))
if not any(name.startswith(("license", "copying")) for name in root_entries):
findings.append(Finding(code="docs.missing-license", category="Documentation", severity=Severity.INFO, message="No root license file detected", deduction=5))
if ".gitignore" not in root_entries:
findings.append(Finding(code="hygiene.missing-gitignore", category="Repository hygiene", severity=Severity.WARNING, message="No root .gitignore detected", deduction=15))
if not any(_is_env_template(name) for name in root_entries) and any(name.startswith(".env") for name in root_entries):
findings.append(Finding(code="hygiene.missing-env-example", category="Repository hygiene", severity=Severity.INFO, message="Environment configuration exists without a root .env.example", deduction=3))
test_signals = detect_test_signals(
[str(path.relative_to(root)).replace("\\", "/") for path in files],
metrics.source_files,
metrics.test_files,
text_by_path,
)
if metrics.source_files and metrics.test_files == 0:
findings.append(Finding(code="testing.none-detected", category="Testing", severity=Severity.WARNING, message="No test files detected", deduction=30))
if not test_signals.ci_workflows:
findings.append(Finding(code="testing.no-ci", category="Testing", severity=Severity.WARNING, message="No recognized CI workflow detected", deduction=15))
if metrics.test_files and not test_signals.coverage_artifacts:
findings.append(Finding(code="testing.no-coverage-artifact", category="Testing", severity=Severity.INFO, message="Tests are present but no coverage artifact is tracked", deduction=2))
common_source_roots = {"src", "app", "backend", "frontend", "lib", "packages"}
if metrics.source_files >= 10 and not (root_entries & common_source_roots):
findings.append(Finding(code="architecture.flat-layout", category="Architecture", severity=Severity.INFO, message="No conventional source root detected in a non-trivial repository", deduction=8))
debt_total = metrics.todo + metrics.fixme + metrics.hack + metrics.xxx
if debt_total:
findings.append(Finding(code="maintainability.debt-markers", category="Maintainability", severity=Severity.INFO, message=f"Detected {debt_total} TODO/FIXME/HACK/XXX markers", deduction=min(20, max(1, debt_total // 5))))
dependency_signals, dependency_findings = analyze_dependencies(root, files_by_name)
metrics.dependency_manifests = len(dependency_signals.manifests)
metrics.declared_dependencies = dependency_signals.declared_dependencies
findings.extend(dependency_findings)
git_signals, churn_by_path = analyze_git(root)
architecture, hotspots = build_architecture(
root,
source_texts,
language_by_path,
loc_by_path,
debt_by_path,
churn_by_path,
)
for hotspot in hotspots[:5]:
if hotspot.score >= 70:
findings.append(
Finding(
code="maintainability.hotspot",
category="Maintainability",
severity=Severity.WARNING,
message=f"Engineering hotspot score {hotspot.score}/100: {', '.join(hotspot.reasons) or 'combined risk signals'}",
path=hotspot.path,
deduction=4,
)
)
if architecture.truncated:
findings.append(
Finding(
code="architecture.graph-truncated",
category="Architecture",
severity=Severity.INFO,
message="Architecture graph was bounded to keep analysis responsive",
deduction=0,
)
)
severity_rank = {Severity.BLOCKER: 0, Severity.WARNING: 1, Severity.INFO: 2}
if len(findings) > limits.max_findings:
discarded = len(findings) - limits.max_findings
findings.sort(key=lambda item: (severity_rank[item.severity], item.category, item.path or "", item.line or 0))
findings = findings[: limits.max_findings]
findings.append(
Finding(
code="analysis.findings-truncated",
category="Repository hygiene",
severity=Severity.INFO,
message=f"{discarded} lower-priority findings were omitted by the analysis finding budget",
deduction=0,
)
)
coverage.truncated = True
coverage.notes.append(f"Finding output limited to {limits.max_findings} deterministic signals.")
policy = policy or load_policy(root)
suppressed_codes = set(policy.suppressions)
findings = [
item.model_copy(update={"suppressed": item.suppressed or item.code in suppressed_codes})
for item in findings
]
scores = {category: 100 for category in CATEGORY_NAMES}
for finding in findings:
if not finding.suppressed:
scores[finding.category] = max(0, scores[finding.category] - finding.deduction)
health_score = round(sum(scores.values()) / len(scores))
release_gate = build_release_gate(
health_score=health_score,
findings=findings,
metrics=metrics,
tests=test_signals,
dependencies=dependency_signals,
coverage=coverage,
policy=policy,
)
findings.sort(key=lambda item: (severity_rank[item.severity], item.category, item.path or "", item.line or 0))
findings = [
item.model_copy(update={"fingerprint": item.fingerprint or finding_fingerprint(item)})
for item in findings
]
findings = enrich_findings(findings)
return ScanReport(
id=str(uuid4()),
repository_path=str(root),
repository_name=display_name or root.name,
source=source or ScanSource(),
policy=policy,
scanned_at=datetime.now(UTC),
duration_ms=round((perf_counter() - started) * 1000, 2),
metrics=metrics,
languages=dict(languages.most_common()),
category_scores=scores,
health_score=health_score,
release_readiness=release_gate.status,
findings=findings,
tests=test_signals,
git=git_signals,
dependencies=dependency_signals,
architecture=architecture,
hotspots=hotspots,
release_gate=release_gate,
coverage=coverage,
)
+494
View File
@@ -0,0 +1,494 @@
from __future__ import annotations
import json
import re
import tomllib
import xml.etree.ElementTree as ET
from collections import defaultdict
from pathlib import Path
from .models import DependencySignals, Finding, Severity, TestSignals
LOCKFILES = {
"package-lock.json",
"pnpm-lock.yaml",
"yarn.lock",
"bun.lock",
"bun.lockb",
"uv.lock",
"poetry.lock",
"Pipfile.lock",
"Cargo.lock",
"go.sum",
}
MANIFEST_MAX_BYTES = 2_000_000
COVERAGE_NAMES = {
"coverage.xml",
"lcov.info",
"cobertura.xml",
".coverage",
"jacoco.xml",
}
COVERAGE_NAMES_CASEFOLD = {item.casefold() for item in COVERAGE_NAMES}
def _python_dependency_name(declaration: str) -> str:
return re.split(r"[<>=!~ @;\[]", declaration, maxsplit=1)[0].strip()
def _python_constraint_kind(declaration: str) -> str:
"""Classify the useful subset of Python version constraints without penalising bounded ranges."""
value = declaration.split(";", maxsplit=1)[0].strip().lower()
if " @ " in value or "==" in value:
return "pinned"
if "~=" in value or "^" in value:
return "bounded"
has_lower = ">" in value
has_upper = "<" in value
if has_lower and has_upper:
return "bounded"
return "unbounded"
def _python_dependency_finding(declaration: str, relative: str) -> Finding | None:
if _python_constraint_kind(declaration) != "unbounded":
return None
return Finding(
code="dependency.python-unbounded",
category="Dependencies",
severity=Severity.WARNING,
message=f"Python dependency has no bounded version range: {declaration.split(';')[0]}",
path=relative,
deduction=3,
)
def detect_test_signals(
relative_paths: list[str],
source_files: int,
test_files: int,
text_by_path: dict[str, str],
) -> TestSignals:
frameworks: set[str] = set()
coverage: list[str] = []
ci: list[str] = []
ci_validation: list[str] = []
ci_build: list[str] = []
ci_deploy_only: list[str] = []
coverage_evidence: list[str] = []
for path in relative_paths:
normalized = path.replace("\\", "/")
name = Path(normalized).name
name_lower = name.casefold()
lower = normalized.lower()
if name_lower in COVERAGE_NAMES_CASEFOLD or "/coverage/" in f"/{lower}":
coverage.append(normalized)
if lower.startswith((".github/workflows/", ".gitea/workflows/")) or name_lower in {
".gitlab-ci.yml",
"jenkinsfile",
"azure-pipelines.yml",
}:
ci.append(normalized)
if (name_lower.startswith("test_") or "/tests/" in f"/{lower}") and name_lower.endswith(".py"):
frameworks.add("pytest/unittest")
if (".test." in name_lower or ".spec." in name_lower) and name_lower.endswith(
(".ts", ".tsx", ".js", ".jsx")
):
frameworks.add("JavaScript test runner")
if name_lower.endswith("_test.go"):
frameworks.add("go test")
if name_lower.endswith(("tests.cs", "test.cs")):
frameworks.add(".NET test")
for path, text in text_by_path.items():
name = Path(path).name
normalized = path.replace("\\", "/")
lowered_text = text.casefold()
if normalized in ci:
validates = any(
token in lowered_text
for token in ("pytest", "vitest", "jest", "npm test", "pnpm test", "go test", "dotnet test", "ruff", "eslint", "typecheck", "py_compile")
)
builds = any(
token in lowered_text
for token in ("npm run build", "pnpm build", "docker build", "vite build", "cargo build", "go build", "dotnet build")
)
deploys = any(token in lowered_text for token in ("deploy", "release", "publish", "kubectl apply"))
if validates:
ci_validation.append(normalized)
if builds:
ci_build.append(normalized)
if deploys and not validates:
ci_deploy_only.append(normalized)
if any(token in lowered_text for token in ("--cov", "coverage", "lcov", "codecov")):
coverage_evidence.append(normalized)
if name.casefold() in COVERAGE_NAMES_CASEFOLD:
coverage_evidence.append(normalized)
if name.casefold() == "package.json":
try:
data = json.loads(text)
except (TypeError, json.JSONDecodeError):
continue
if not isinstance(data, dict):
continue
deps: dict[object, object] = {}
for section_name in (
"dependencies",
"devDependencies",
"peerDependencies",
"optionalDependencies",
):
section = data.get(section_name, {})
if isinstance(section, dict):
deps.update(section)
if "vitest" in deps:
frameworks.add("Vitest")
if "jest" in deps:
frameworks.add("Jest")
if "@playwright/test" in deps:
frameworks.add("Playwright")
if "cypress" in deps:
frameworks.add("Cypress")
elif name.casefold() == "pyproject.toml":
lowered = text.lower()
if "pytest" in lowered:
frameworks.add("pytest")
ratio = round((test_files / source_files) * 100, 2) if source_files else None
return TestSignals(
frameworks=sorted(frameworks),
coverage_artifacts=sorted(set(coverage))[:30],
ci_workflows=sorted(set(ci))[:30],
ci_validation_workflows=sorted(set(ci_validation))[:30],
ci_build_workflows=sorted(set(ci_build))[:30],
ci_deploy_only_workflows=sorted(set(ci_deploy_only))[:30],
coverage_evidence=sorted(set(coverage_evidence))[:30],
test_source_ratio=ratio,
)
def analyze_dependencies(
root: Path,
files_by_name: dict[str, list[Path]],
) -> tuple[DependencySignals, list[Finding]]:
findings: list[Finding] = []
manifests: list[str] = []
lockfiles: list[str] = []
dependency_origins: dict[str, set[str]] = defaultdict(set)
declared = 0
unbounded = 0
unpinned = 0
for lock_name in sorted(LOCKFILES):
for path in files_by_name.get(lock_name, []):
lockfiles.append(str(path.relative_to(root)).replace("\\", "/"))
for package_json in files_by_name.get("package.json", []):
relative = str(package_json.relative_to(root)).replace("\\", "/")
manifests.append(relative)
try:
if package_json.stat().st_size > MANIFEST_MAX_BYTES:
raise ValueError("package.json is too large to parse")
data = json.loads(package_json.read_text(encoding="utf-8"))
if not isinstance(data, dict):
raise TypeError("package.json root must be an object")
except (OSError, UnicodeDecodeError, json.JSONDecodeError, TypeError):
findings.append(
Finding(
code="dependency.invalid-package-json",
category="Dependencies",
severity=Severity.WARNING,
message="package.json could not be parsed safely",
path=relative,
deduction=8,
)
)
continue
deps: dict[str, object] = {}
invalid_sections = False
for section_name in ("dependencies", "devDependencies", "peerDependencies", "optionalDependencies"):
section = data.get(section_name, {})
if section is None:
continue
if not isinstance(section, dict):
invalid_sections = True
continue
deps.update(section)
if invalid_sections:
findings.append(
Finding(
code="dependency.invalid-package-json",
category="Dependencies",
severity=Severity.WARNING,
message="One or more package dependency sections are not objects",
path=relative,
deduction=8,
)
)
declared += len(deps)
for name, version in deps.items():
dependency_name = str(name)
dependency_origins[dependency_name.casefold()].add(relative)
raw = str(version).strip().lower()
if raw in {"*", "latest", "x"} and not raw.startswith("workspace:"):
unbounded += 1
findings.append(
Finding(
code="dependency.unbounded",
category="Dependencies",
severity=Severity.WARNING,
message=f"Dependency {dependency_name} uses an unbounded version",
path=relative,
deduction=5,
)
)
parent = package_json.parent
package_roots = [parent, *[ancestor for ancestor in parent.parents if ancestor == root or root in ancestor.parents]]
if not any(
(package_root / lock).exists()
for package_root in package_roots
for lock in ("package-lock.json", "pnpm-lock.yaml", "yarn.lock", "bun.lock", "bun.lockb")
):
findings.append(Finding(code="dependency.no-lockfile", category="Dependencies", severity=Severity.WARNING, message="JavaScript package has no recognized lockfile", path=relative, deduction=8))
requirement_paths = [
path
for name, values in files_by_name.items()
if name.casefold().startswith("requirements") and name.casefold().endswith(".txt")
for path in values
]
included_requirements: list[str] = []
processed_requirements: set[Path] = set()
pending_requirements = list(requirement_paths)
while pending_requirements:
requirements = pending_requirements.pop(0)
try:
requirements = requirements.resolve(strict=True)
except OSError:
continue
if requirements in processed_requirements or requirements != root and root not in requirements.parents:
continue
processed_requirements.add(requirements)
relative = str(requirements.relative_to(root)).replace("\\", "/")
manifests.append(relative)
try:
lines = requirements.read_text(encoding="utf-8").splitlines()
except (OSError, UnicodeDecodeError):
findings.append(
Finding(
code="dependency.unreadable-requirements",
category="Dependencies",
severity=Severity.WARNING,
message="Python requirements file could not be read",
path=relative,
deduction=8,
)
)
continue
for line in lines:
stripped = line.strip()
if not stripped or stripped.startswith("#"):
continue
if stripped.startswith(("-r ", "--requirement ")):
include = stripped.split(maxsplit=1)[1].strip()
candidate = (requirements.parent / include).resolve()
if candidate == root or root in candidate.parents:
pending_requirements.append(candidate)
included_requirements.append(str(candidate.relative_to(root)).replace("\\", "/"))
continue
if stripped.startswith("-"):
continue
declared += 1
name = _python_dependency_name(stripped)
if name:
dependency_origins[name.casefold()].add(relative)
finding = _python_dependency_finding(stripped, relative)
if finding:
unbounded += 1
findings.append(finding)
for pyproject in files_by_name.get("pyproject.toml", []):
relative = str(pyproject.relative_to(root)).replace("\\", "/")
manifests.append(relative)
try:
if pyproject.stat().st_size > MANIFEST_MAX_BYTES:
raise ValueError("pyproject.toml is too large to parse")
data = tomllib.loads(pyproject.read_text(encoding="utf-8"))
if not isinstance(data, dict):
raise TypeError("pyproject.toml root must be a table")
except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError, TypeError):
findings.append(
Finding(
code="dependency.invalid-pyproject",
category="Dependencies",
severity=Severity.WARNING,
message="pyproject.toml could not be parsed safely",
path=relative,
deduction=8,
)
)
continue
project = data.get("project", {})
if not isinstance(project, dict):
findings.append(
Finding(
code="dependency.invalid-pyproject",
category="Dependencies",
severity=Severity.WARNING,
message="pyproject.toml project section is not a table",
path=relative,
deduction=8,
)
)
project = {}
project_deps = project.get("dependencies", [])
if isinstance(project_deps, list):
declared += len(project_deps)
for dependency in project_deps:
declaration = str(dependency)
name = _python_dependency_name(declaration)
if name:
dependency_origins[name.casefold()].add(relative)
finding = _python_dependency_finding(declaration, relative)
if finding:
unbounded += 1
findings.append(finding)
elif project_deps is not None:
findings.append(
Finding(
code="dependency.invalid-pyproject",
category="Dependencies",
severity=Severity.WARNING,
message="pyproject.toml project.dependencies is not a list",
path=relative,
deduction=8,
)
)
optional_deps = project.get("optional-dependencies", {})
if isinstance(optional_deps, dict):
for dependencies in optional_deps.values():
if not isinstance(dependencies, list):
continue
declared += len(dependencies)
for dependency in dependencies:
declaration = str(dependency)
name = _python_dependency_name(declaration)
if name:
dependency_origins[name.casefold()].add(relative)
finding = _python_dependency_finding(declaration, relative)
if finding:
unbounded += 1
findings.append(finding)
elif optional_deps is not None:
findings.append(
Finding(
code="dependency.invalid-pyproject",
category="Dependencies",
severity=Severity.WARNING,
message="pyproject.toml project.optional-dependencies is not a table",
path=relative,
deduction=8,
)
)
for go_mod in files_by_name.get("go.mod", []):
relative = str(go_mod.relative_to(root)).replace("\\", "/")
manifests.append(relative)
try:
text = go_mod.read_text(encoding="utf-8")
except (OSError, UnicodeDecodeError):
findings.append(
Finding(
code="dependency.unreadable-go-mod",
category="Dependencies",
severity=Severity.WARNING,
message="go.mod could not be read",
path=relative,
deduction=8,
)
)
continue
matches = list(re.finditer(r"(?m)^\s*([\w./-]+)\s+v\d[^\s]*", text))
if "require" in text and not matches:
findings.append(
Finding(
code="dependency.invalid-go-mod",
category="Dependencies",
severity=Severity.WARNING,
message="go.mod contains no parseable versioned dependencies",
path=relative,
deduction=8,
)
)
for match in matches:
declared += 1
dependency_origins[match.group(1).casefold()].add(relative)
for cargo in files_by_name.get("Cargo.toml", []):
relative = str(cargo.relative_to(root)).replace("\\", "/")
manifests.append(relative)
try:
data = tomllib.loads(cargo.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError):
findings.append(
Finding(
code="dependency.invalid-cargo-toml",
category="Dependencies",
severity=Severity.WARNING,
message="Cargo.toml could not be parsed safely",
path=relative,
deduction=8,
)
)
continue
cargo_deps = data.get("dependencies", {})
if isinstance(cargo_deps, dict):
declared += len(cargo_deps)
for name in cargo_deps:
dependency_origins[str(name).casefold()].add(relative)
for pom in files_by_name.get("pom.xml", []):
relative = str(pom.relative_to(root)).replace("\\", "/")
manifests.append(relative)
try:
xml_root = ET.parse(pom).getroot()
except (OSError, UnicodeError, ET.ParseError):
findings.append(
Finding(
code="dependency.invalid-pom",
category="Dependencies",
severity=Severity.WARNING,
message="pom.xml could not be parsed safely",
path=relative,
deduction=8,
)
)
continue
dependencies = [element for element in xml_root.iter() if element.tag.split("}")[-1] == "dependency"]
declared += len(dependencies)
for dependency in dependencies:
artifact = next((child.text for child in dependency if child.tag.split("}")[-1] == "artifactId"), None)
if artifact:
dependency_origins[artifact.casefold()].add(relative)
duplicates = sorted(name for name, origins in dependency_origins.items() if len(origins) > 1)
for name in duplicates[:25]:
origins = sorted(dependency_origins[name])
findings.append(Finding(code="dependency.duplicate-manifest", category="Dependencies", severity=Severity.INFO, message=f"Dependency {name} is declared across multiple manifests", path=origins[0], deduction=1))
return (
DependencySignals(
manifests=sorted(set(manifests)),
lockfiles=sorted(set(lockfiles)),
declared_dependencies=declared,
unbounded_dependencies=unbounded,
unpinned_dependencies=unpinned,
duplicate_dependencies=duplicates[:50],
included_requirements=sorted(set(included_requirements))[:50],
),
findings,
)
+1527
View File
File diff suppressed because it is too large. Load diff
+23
View File
@@ -0,0 +1,23 @@
[project]
name = "repopulse-backend"
version = "1.5.0"
description = "RepoPulse repository intelligence API and deterministic analysis engine"
requires-python = ">=3.12"
dependencies = [
"fastapi>=0.116,<1",
"uvicorn[standard]>=0.35,<1",
"pydantic>=2.11,<3",
]
[project.optional-dependencies]
dev = [
"pytest>=8.4,<9",
"ruff>=0.12,<1",
"httpx>=0.28,<1",
]
[tool.pytest.ini_options]
pythonpath = ["."]
[tool.ruff]
line-length = 100
+122
View File
@@ -0,0 +1,122 @@
from pathlib import Path
from threading import BoundedSemaphore
import pytest
from fastapi.testclient import TestClient
from app import main as main_module
from app.store import ScanStore
def _repository(root: Path) -> Path:
repository = root / "repo"
repository.mkdir()
(repository / "README.md").write_text("# Demo\n", encoding="utf-8")
(repository / "LICENSE").write_text("MIT\n", encoding="utf-8")
(repository / ".gitignore").write_text(".env\n", encoding="utf-8")
(repository / "app.py").write_text("print('hello')\n", encoding="utf-8")
return repository
def test_scan_api_respects_root_and_exports_reports(tmp_path: Path) -> None:
repository = _repository(tmp_path)
main_module.SCAN_ROOT = str(tmp_path)
main_module.store = ScanStore(str(tmp_path / "history.db"))
main_module.SCAN_SLOTS = BoundedSemaphore(1)
client = TestClient(main_module.app)
first_response = client.post("/api/v1/scans", json={"path": "repo"})
assert first_response.status_code == 201
first = first_response.json()
assert first["repository_path"] == str(repository.resolve())
html = client.get(f"/api/v1/scans/{first['id']}/export?format=html")
assert html.status_code == 200
assert "RepoPulse report" in html.text
assert "repo" in html.text
blocked = client.post("/api/v1/scans", json={"path": str(tmp_path.parent)})
assert blocked.status_code == 422
def test_scan_api_comparison_delta_and_trend(tmp_path: Path) -> None:
repository = _repository(tmp_path)
main_module.SCAN_ROOT = str(tmp_path)
main_module.store = ScanStore(str(tmp_path / "history.db"))
main_module.SCAN_SLOTS = BoundedSemaphore(1)
client = TestClient(main_module.app)
first = client.post("/api/v1/scans", json={"path": "repo"}).json()
(repository / "tests").mkdir()
(repository / "tests" / "test_app.py").write_text("def test_ok():\n assert True\n", encoding="utf-8")
second = client.post("/api/v1/scans", json={"path": "repo"}).json()
comparison = client.get(f"/api/v1/scans/{second['id']}/comparison")
assert comparison.status_code == 200
assert comparison.json()["previous_scan_id"] == first["id"]
delta = client.get(f"/api/v1/scans/{second['id']}/delta")
assert delta.status_code == 200
assert delta.json()["previous_health_score"] == first["health_score"]
assert delta.json()["previous_finding_count"] == len(first["findings"])
trend = client.get(f"/api/v1/trend?path={repository}")
assert trend.status_code == 200
assert [item["scan_id"] for item in trend.json()] == [first["id"], second["id"]]
def test_scan_api_rejects_when_scan_capacity_is_busy(tmp_path: Path) -> None:
_repository(tmp_path)
main_module.SCAN_ROOT = str(tmp_path)
main_module.store = ScanStore(str(tmp_path / "history.db"))
main_module.SCAN_SLOTS = BoundedSemaphore(1)
client = TestClient(main_module.app)
assert main_module.SCAN_SLOTS.acquire(blocking=False)
try:
response = client.post("/api/v1/scans", json={"path": "repo"})
finally:
main_module.SCAN_SLOTS.release()
assert response.status_code == 429
assert response.headers["retry-after"] == "2"
def test_scan_api_rejects_symlinked_repository_outside_root(tmp_path: Path) -> None:
outside = tmp_path.parent / f"{tmp_path.name}-outside-repository"
outside.mkdir()
(outside / "app.py").write_text("print('outside')\n", encoding="utf-8")
linked = tmp_path / "linked-repository"
try:
linked.symlink_to(outside, target_is_directory=True)
except (NotImplementedError, OSError):
pytest.skip("directory symlinks are unavailable in this environment")
main_module.SCAN_ROOT = str(tmp_path)
main_module.store = ScanStore(str(tmp_path / "history.db"))
main_module.SCAN_SLOTS = BoundedSemaphore(1)
client = TestClient(main_module.app)
response = client.post("/api/v1/scans", json={"path": "linked-repository"})
assert response.status_code == 422
def test_mutating_api_requires_operator_token_only_when_configured(tmp_path: Path, monkeypatch) -> None:
_repository(tmp_path)
main_module.SCAN_ROOT = str(tmp_path)
main_module.store = ScanStore(str(tmp_path / "history.db"))
main_module.SCAN_SLOTS = BoundedSemaphore(1)
monkeypatch.setenv("REPOPULSE_OPERATOR_TOKEN", "test-operator-token")
client = TestClient(main_module.app)
assert client.get("/api/v1/meta").status_code == 200
denied = client.post("/api/v1/scans", json={"path": "repo"})
assert denied.status_code == 401
allowed = client.post(
"/api/v1/scans",
json={"path": "repo"},
headers={"X-RepoPulse-Token": "test-operator-token"},
)
assert allowed.status_code == 201
+130
View File
@@ -0,0 +1,130 @@
from __future__ import annotations
import hashlib
import hmac
import json
from fastapi.testclient import TestClient
from app import main as main_module
from app.gitea import GiteaClient
from app.models import Repository
from app.store import ScanStore
def test_repository_catalog_and_durable_scan_job_contract(tmp_path) -> None:
main_module.store = ScanStore(str(tmp_path / "history.db"))
main_module.job_runner = None
main_module.gitea_client = GiteaClient("http://gitea.local", "test-token", transport=lambda *_: (500, b""))
repository = Repository(
id="gitea:64",
provider="gitea",
provider_repository_id="64",
owner="Jens",
name="RepoPulse",
full_name="Jens/RepoPulse",
default_branch="main",
latest_revision="0123456789abcdef0123456789abcdef01234567",
)
main_module.store.upsert_repository(repository)
client = TestClient(main_module.app)
catalog = client.get("/api/v1/repositories")
assert catalog.status_code == 200
assert catalog.json()[0]["id"] == "gitea:64"
assert "latest_scan" not in catalog.json()[0]
queued = client.post("/api/v1/repositories/gitea%3A64/scans", json={"force": False})
assert queued.status_code == 202
job = queued.json()
assert job["repository_id"] == "gitea:64"
assert job["status"] == "queued"
duplicate = client.post("/api/v1/repositories/gitea%3A64/scans", json={"force": False})
assert duplicate.status_code == 202
assert duplicate.json()["id"] == job["id"]
assert duplicate.json()["deduplicated"] is True
detail = client.get("/api/v1/repositories/gitea%3A64")
assert detail.status_code == 200
assert detail.json()["latest_scan"] is None
assert client.get(f"/api/v1/scan-jobs/{job['id']}").json()["status"] == "queued"
def test_signed_gitea_webhook_only_schedules_default_branch_push(
tmp_path, monkeypatch
) -> None:
main_module.store = ScanStore(str(tmp_path / "history.db"))
main_module.job_runner = None
main_module.gitea_client = GiteaClient("http://gitea.local", "test-token", transport=lambda *_: (500, b""))
repository = Repository(
id="gitea:64",
provider="gitea",
provider_repository_id="64",
owner="Jens",
name="RepoPulse",
full_name="Jens/RepoPulse",
default_branch="main",
latest_revision="0123456789abcdef0123456789abcdef01234567",
)
main_module.store.upsert_repository(repository)
monkeypatch.setenv("REPOPULSE_GITEA_WEBHOOK_SECRET", "webhook-secret")
payload = {
"repository": {"id": 64},
"ref": "refs/heads/main",
"after": "abcdef0123456789abcdef0123456789abcdef01",
}
body = json.dumps(payload).encode()
signature = hmac.new(b"webhook-secret", body, hashlib.sha256).hexdigest()
client = TestClient(main_module.app)
response = client.post(
"/api/v1/webhooks/gitea",
content=body,
headers={"X-Gitea-Signature": signature, "X-Gitea-Event": "push"},
)
assert response.status_code == 200
assert response.json()["job_id"]
assert main_module.store.get_repository("gitea:64").latest_revision == payload["after"]
wrong_signature = client.post(
"/api/v1/webhooks/gitea",
content=body,
headers={"X-Gitea-Signature": "0" * 64, "X-Gitea-Event": "push"},
)
assert wrong_signature.status_code == 401
feature_payload = payload | {"ref": "refs/heads/feature"}
feature_body = json.dumps(feature_payload).encode()
feature_signature = hmac.new(b"webhook-secret", feature_body, hashlib.sha256).hexdigest()
feature = client.post(
"/api/v1/webhooks/gitea",
content=feature_body,
headers={"X-Gitea-Signature": feature_signature, "X-Gitea-Event": "push"},
)
assert feature.status_code == 200
assert feature.json()["job_id"] is None
def test_meta_exposes_operational_counts_without_connector_token(tmp_path) -> None:
main_module.store = ScanStore(str(tmp_path / "history.db"))
main_module.job_runner = None
main_module.gitea_client = GiteaClient("http://gitea.local", "never-return-this-token")
main_module.store.upsert_repository(
Repository(
id="gitea:64",
provider="gitea",
provider_repository_id="64",
owner="Jens",
name="RepoPulse",
full_name="Jens/RepoPulse",
latest_revision="0123456789abcdef0123456789abcdef01234567",
)
)
response = TestClient(main_module.app).get("/api/v1/meta")
assert response.status_code == 200
assert response.json()["repository_count"] == 1
assert response.json()["repository_states"]["never_scanned"] == 1
assert "never-return-this-token" not in response.text
+98
View File
@@ -0,0 +1,98 @@
from pathlib import Path
from app.architecture import build_architecture
def test_architecture_resolves_python_and_relative_typescript_imports(tmp_path: Path) -> None:
(tmp_path / "src").mkdir()
(tmp_path / "web").mkdir()
(tmp_path / "src" / "core.py").write_text("VALUE = 1\n", encoding="utf-8")
(tmp_path / "src" / "app.py").write_text("import src.core\n", encoding="utf-8")
(tmp_path / "web" / "util.ts").write_text("export const value = 1\n", encoding="utf-8")
(tmp_path / "web" / "app.ts").write_text("import { value } from './util'\n", encoding="utf-8")
(tmp_path / "web" / "side-effect.ts").write_text("import './util.js'\n", encoding="utf-8")
source_texts = {
"src/core.py": "VALUE = 1\n",
"src/app.py": "import src.core\n",
"web/util.ts": "export const value = 1\n",
"web/app.ts": "import { value } from './util'\n",
"web/side-effect.ts": "import './util.js'\n",
}
languages = {
"src/core.py": "Python",
"src/app.py": "Python",
"web/util.ts": "TypeScript",
"web/app.ts": "TypeScript",
"web/side-effect.ts": "TypeScript",
}
loc = {path: 1 for path in source_texts}
graph, hotspots = build_architecture(
tmp_path,
source_texts,
languages,
loc,
{path: 0 for path in source_texts},
{"src/core.py": 100, "src/app.py": 200},
)
pairs = {(edge.source, edge.target) for edge in graph.edges}
assert ("src/app.py", "src/core.py") in pairs
assert ("web/app.ts", "web/util.ts") in pairs
assert ("web/side-effect.ts", "web/util.ts") in pairs
assert {node.id for node in graph.nodes} == set(source_texts)
assert hotspots[0].path in {"src/app.py", "src/core.py"}
def test_architecture_edge_bound_is_explicit(tmp_path: Path) -> None:
source_texts = {
"src/a.py": "import src.b\n",
"src/b.py": "import src.c\n",
"src/c.py": "VALUE = 1\n",
}
languages = {path: "Python" for path in source_texts}
graph, _ = build_architecture(
tmp_path,
source_texts,
languages,
{path: 1 for path in source_texts},
{path: 0 for path in source_texts},
{},
max_edges=1,
)
assert len(graph.edges) == 1
assert graph.truncated is True
def test_architecture_resolves_jsconfig_aliases_and_local_workspace_packages(tmp_path: Path) -> None:
(tmp_path / "src" / "components").mkdir(parents=True)
(tmp_path / "packages" / "shared" / "src").mkdir(parents=True)
(tmp_path / "jsconfig.json").write_text(
'{"compilerOptions":{"baseUrl":".","paths":{"@/*":["src/*"]}}}',
encoding="utf-8",
)
(tmp_path / "package.json").write_text('{"name":"workspace-root"}', encoding="utf-8")
(tmp_path / "packages" / "shared" / "package.json").write_text(
'{"name":"@workspace/shared"}', encoding="utf-8"
)
source_texts = {
"src/app.ts": "import { Button } from '@/components/Button'\nimport { shared } from '@workspace/shared'\n",
"src/components/Button.ts": "export const Button = () => null\n",
"packages/shared/src/index.ts": "export const shared = true\n",
}
graph, _ = build_architecture(
tmp_path,
source_texts,
{path: "TypeScript" for path in source_texts},
{path: 1 for path in source_texts},
{path: 0 for path in source_texts},
{},
)
pairs = {(edge.source, edge.target) for edge in graph.edges}
assert ("src/app.ts", "src/components/Button.ts") in pairs
assert ("src/app.ts", "packages/shared/src/index.ts") in pairs
+203
View File
@@ -0,0 +1,203 @@
from __future__ import annotations
import json
from pathlib import Path
import pytest
from app.gitea import GiteaClient, GiteaError
from app.store import ScanStore
def _transport(url: str, headers: dict[str, str]) -> tuple[int, bytes]:
assert headers["Authorization"] == "token test-token"
if "/user/repos" in url:
return 200, json.dumps(
[
{
"id": 64,
"name": "RepoPulse",
"full_name": "Jens/RepoPulse",
"owner": {"login": "Jens"},
"description": "Repository intelligence",
"default_branch": "main",
"clone_url": "https://evil.example/Jens/RepoPulse.git",
"private": True,
}
]
).encode()
if "/branches/main" in url:
return 200, b'{"commit":{"id":"0123456789abcdef0123456789abcdef01234567"}}'
raise AssertionError(url)
def test_gitea_discovery_is_bounded_and_preserves_stable_identity() -> None:
client = GiteaClient(
"http://gitea.local",
"test-token",
transport=_transport,
)
discovered = client.discover()
assert len(discovered) == 1
repository, revision = discovered[0]
assert repository.id == "gitea:64"
assert repository.provider_repository_id == "64"
assert repository.full_name == "Jens/RepoPulse"
assert repository.clone_url == "http://gitea.local/Jens/RepoPulse.git"
assert repository.latest_revision == revision
def test_gitea_auth_errors_are_sanitized() -> None:
def rejected(_url: str, _headers: dict[str, str]) -> tuple[int, bytes]:
return 401, b"secret-bearing upstream response"
client = GiteaClient("http://gitea.local", "test-token", transport=rejected)
try:
client.list_repositories()
except GiteaError as exc:
assert exc.code == "authentication"
assert "test-token" not in str(exc)
assert "secret-bearing" not in str(exc)
else:
raise AssertionError("authentication error was not raised")
@pytest.mark.parametrize("status_code, error_code", [(401, "authentication"), (403, "authentication"), (404, "not_found"), (429, "rate_limited"), (500, "upstream")])
def test_gitea_http_statuses_are_classified_without_upstream_body(status_code: int, error_code: str) -> None:
client = GiteaClient(
"http://gitea.local",
"test-token",
transport=lambda *_: (status_code, b"token=test-token upstream body"),
)
with pytest.raises(GiteaError) as raised:
client.list_repositories()
assert raised.value.code == error_code
assert "test-token" not in str(raised.value)
assert "upstream body" not in str(raised.value)
def test_gitea_discovery_paginates_until_a_short_page() -> None:
calls: list[str] = []
def paginated(url: str, headers: dict[str, str]) -> tuple[int, bytes]:
calls.append(url)
assert headers["Authorization"] == "token test-token"
if "/user/repos" in url:
if "page=1" in url:
payload = [{"id": 64, "owner": {"login": "Jens"}, "name": "One", "default_branch": "main"}]
elif "page=2" in url:
payload = [{"id": 65, "owner": {"login": "Jens"}, "name": "Two", "default_branch": "trunk"}]
else:
payload = []
return 200, json.dumps(payload).encode()
if "/branches/main" in url:
return 200, b'{"commit":{"id":"0123456789abcdef0123456789abcdef01234567"}}'
if "/branches/trunk" in url:
return 200, b'{"commit":{"sha":"abcdef0123456789abcdef0123456789abcdef01"}}'
raise AssertionError(url)
client = GiteaClient("http://gitea.local", "test-token", page_size=1, max_pages=3, transport=paginated)
discovered = client.discover()
assert [repository.id for repository, _ in discovered] == ["gitea:64", "gitea:65"]
assert len([call for call in calls if "/user/repos" in call]) == 3
@pytest.mark.parametrize(
"transport, error_code",
[
(lambda *_: (200, b"not-json"), "invalid_response"),
(lambda *_: (_ for _ in ()).throw(TimeoutError()), "unreachable"),
],
)
def test_gitea_malformed_and_timeout_responses_are_safe(transport, error_code: str) -> None:
client = GiteaClient("http://gitea.local", "test-token", transport=transport)
with pytest.raises(GiteaError) as raised:
client.list_repositories()
assert raised.value.code == error_code
def test_gitea_sync_creates_catalog_without_full_reports(tmp_path: Path) -> None:
store = ScanStore(str(tmp_path / "history.db"))
client = GiteaClient("http://gitea.local", "test-token", transport=_transport)
result = client.sync(store)
assert result.discovered == 1
assert result.created == 1
repository = store.get_repository("gitea:64")
assert repository is not None
assert repository.latest_revision == "0123456789abcdef0123456789abcdef01234567"
assert repository.description == "Repository intelligence"
assert store.repository_scans(repository.id) == []
def test_gitea_outage_preserves_catalog_and_marks_sync_error(tmp_path: Path) -> None:
store = ScanStore(str(tmp_path / "history.db"))
healthy = GiteaClient("http://gitea.local", "test-token", transport=_transport)
healthy.sync(store)
unavailable = GiteaClient(
"http://gitea.local",
"test-token",
transport=lambda *_: (503, b"upstream details must not escape"),
)
result = unavailable.sync(store)
assert result.errors == ["upstream: Gitea returned a server error"]
repository = store.get_repository("gitea:64")
assert repository is not None
assert repository.available is True
assert repository.latest_revision == "0123456789abcdef0123456789abcdef01234567"
assert repository.sync_error == "upstream: Gitea returned a server error"
def test_complete_sync_marks_missing_repository_unavailable_without_deleting_history(tmp_path: Path) -> None:
store = ScanStore(str(tmp_path / "history.db"))
GiteaClient("http://gitea.local", "test-token", transport=_transport).sync(store)
missing = GiteaClient("http://gitea.local", "test-token", transport=lambda *_: (200, b"[]"))
result = missing.sync(store)
assert result.unavailable == 1
repository = store.get_repository("gitea:64")
assert repository is not None
assert repository.available is False
assert store.repository_scans(repository.id) == []
def test_branch_resolution_failure_is_partial_and_preserves_other_catalog_entries(tmp_path: Path) -> None:
def partial_transport(url: str, _headers: dict[str, str]) -> tuple[int, bytes]:
if "/user/repos" in url:
return 200, json.dumps(
[
{"id": 1, "owner": {"login": "Jens"}, "name": "Healthy", "default_branch": "main"},
{"id": 2, "owner": {"login": "Jens"}, "name": "Broken", "default_branch": "main"},
]
).encode()
if "/repos/Jens/Healthy/branches/main" in url:
return 200, b'{"commit":{"id":"0123456789abcdef0123456789abcdef01234567"}}'
if "/repos/Jens/Broken/branches/main" in url:
return 500, b"upstream details must remain private"
raise AssertionError(url)
store = ScanStore(str(tmp_path / "history.db"))
result = GiteaClient("http://gitea.local", "test-token", transport=partial_transport).sync(store)
assert result.discovered == 1
assert result.partial is True
assert result.branch_failures == 1
assert result.listing_complete is True
assert "Jens/Broken: upstream: Gitea returned a server error" in result.errors
assert store.get_repository("gitea:1").latest_revision
broken = store.get_repository("gitea:2")
assert broken is not None and broken.available is True
assert broken.sync_error == "Jens/Broken: upstream: Gitea returned a server error"
+232
View File
@@ -0,0 +1,232 @@
from __future__ import annotations
import subprocess
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime, timedelta
from pathlib import Path
import pytest
from app.acquisition import AcquisitionError, GitAcquirer
from app.gitea import GiteaClient
from app.jobs import ScanJobRunner
from app.models import Repository, ScanJobStatus, ScanSource, ScanTrigger, SourceOrigin
from app.scanner import scan_repository
from app.store import ScanStore
def _git_repository(root: Path) -> tuple[Path, str]:
root.mkdir()
(root / "README.md").write_text("# Demo\n", encoding="utf-8")
(root / "app.py").write_text("print('hello')\n", encoding="utf-8")
subprocess.run(["git", "init", "--quiet", str(root)], check=True)
subprocess.run(["git", "-C", str(root), "config", "user.email", "test@example.com"], check=True)
subprocess.run(["git", "-C", str(root), "config", "user.name", "RepoPulse Test"], check=True)
subprocess.run(["git", "-C", str(root), "add", "."], check=True)
subprocess.run(["git", "-C", str(root), "commit", "--quiet", "-m", "initial"], check=True)
revision = subprocess.run(
["git", "-C", str(root), "rev-parse", "HEAD"],
check=True,
capture_output=True,
text=True,
).stdout.strip()
return root, revision
def _catalog_repository(source: Path, revision: str) -> Repository:
return Repository(
id="gitea:64",
provider="gitea",
provider_repository_id="64",
owner="Jens",
name="Demo",
full_name="Jens/Demo",
clone_url=source.as_uri(),
default_branch="master",
latest_revision=revision,
)
def test_acquirer_scans_detached_exact_revision_and_cleans_up(tmp_path: Path) -> None:
source, revision = _git_repository(tmp_path / "source")
repository = _catalog_repository(source, revision)
acquirer = GitAcquirer(tmp_path / "cache", timeout=30)
acquired = acquirer.acquire(repository, "job-1", revision)
checked_out = subprocess.run(
["git", "-C", str(acquired.path), "rev-parse", "HEAD"],
check=True,
capture_output=True,
text=True,
).stdout.strip()
assert checked_out == revision
assert (acquired.path / "app.py").read_text(encoding="utf-8") == "print('hello')\n"
acquirer.cleanup(acquired, repository)
assert not acquired.path.exists()
def test_acquirer_recovers_a_corrupt_mirror_and_rejects_missing_sha(tmp_path: Path) -> None:
source, revision = _git_repository(tmp_path / "source")
repository = _catalog_repository(source, revision)
acquirer = GitAcquirer(tmp_path / "cache", timeout=30)
mirror = acquirer._mirror_path(repository)
mirror.parent.mkdir(parents=True, exist_ok=True)
mirror.write_text("corrupt mirror", encoding="utf-8")
with pytest.raises(AcquisitionError, match="Requested commit was not found"):
acquirer.acquire(repository, "job-missing", "abcdef0123456789abcdef0123456789abcdef01")
acquired = acquirer.acquire(repository, "job-recovered", revision)
assert acquired.actual_revision == revision
acquirer.cleanup(acquired, repository)
def test_durable_runner_completes_and_deduplicates_revision(tmp_path: Path) -> None:
source, revision = _git_repository(tmp_path / "source")
store = ScanStore(str(tmp_path / "history.db"))
repository = _catalog_repository(source, revision)
store.upsert_repository(repository)
runner = ScanJobRunner(
store,
gitea=GiteaClient("http://gitea.local", "test-token", transport=lambda *_: (500, b"")),
acquirer=GitAcquirer(tmp_path / "cache", timeout=30),
)
first = runner.enqueue(repository)
completed = runner.process_one()
reused = runner.enqueue(repository)
assert completed is not None
assert completed.status == ScanJobStatus.COMPLETED
assert completed.scan_id is not None
assert reused.id == first.id
assert reused.deduplicated is True
report = store.get(completed.scan_id)
assert report is not None
assert report.source.actual_revision == revision
assert report.source.trigger == ScanTrigger.MANUAL
assert completed.duration_ms is not None
def test_expired_job_lease_is_recovered_for_another_claim(tmp_path: Path) -> None:
store = ScanStore(str(tmp_path / "history.db"))
repository = _catalog_repository(tmp_path, "0123456789abcdef0123456789abcdef01234567")
store.upsert_repository(repository)
queued = store.enqueue_job(repository.id, repository.latest_revision, ScanTrigger.SCHEDULED)
store.update_job(
queued.id,
status=ScanJobStatus.SCANNING,
phase="scanning",
lease_until=datetime.now(UTC) - timedelta(seconds=5),
)
claimed = store.claim_next_job()
assert claimed is not None
assert claimed.id == queued.id
assert claimed.status == ScanJobStatus.ACQUIRING_SOURCE
assert claimed.attempt == 1
assert claimed.message == "Acquiring exact repository revision"
def test_active_job_lease_renews_but_terminal_jobs_are_not_resurrected(tmp_path: Path) -> None:
store = ScanStore(str(tmp_path / "history.db"))
repository = _catalog_repository(tmp_path, "0123456789abcdef0123456789abcdef01234567")
store.upsert_repository(repository)
queued = store.enqueue_job(repository.id, repository.latest_revision, ScanTrigger.SCHEDULED)
claimed = store.claim_next_job()
assert claimed is not None
with store._connection() as db:
initial_lease = db.execute("SELECT lease_until FROM scan_jobs WHERE id = ?", (queued.id,)).fetchone()[0]
assert initial_lease is not None
renewed = store.renew_job_lease(queued.id)
assert renewed is not None
with store._connection() as db:
renewed_lease = db.execute("SELECT lease_until FROM scan_jobs WHERE id = ?", (queued.id,)).fetchone()[0]
assert renewed_lease is not None and renewed_lease > initial_lease
completed = store.update_job(queued.id, status=ScanJobStatus.COMPLETED, phase="completed", progress=100)
assert completed is not None
after_completion = store.renew_job_lease(queued.id)
assert after_completion is not None
with store._connection() as db:
terminal_lease = db.execute("SELECT lease_until FROM scan_jobs WHERE id = ?", (queued.id,)).fetchone()[0]
assert terminal_lease is None
def test_atomic_claim_allows_only_one_worker_to_own_a_queued_job(tmp_path: Path) -> None:
store = ScanStore(str(tmp_path / "history.db"))
repository = _catalog_repository(tmp_path, "0123456789abcdef0123456789abcdef01234567")
store.upsert_repository(repository)
store.enqueue_job(repository.id, repository.latest_revision, ScanTrigger.SCHEDULED)
with ThreadPoolExecutor(max_workers=2) as workers:
claims = list(workers.map(lambda _unused: store.claim_next_job(), range(2)))
successful = [claim for claim in claims if claim is not None]
assert len(successful) == 1
assert successful[0].status == ScanJobStatus.ACQUIRING_SOURCE
def test_scheduler_queues_unscanned_and_changed_repositories_only(tmp_path: Path) -> None:
source, revision = _git_repository(tmp_path / "source")
store = ScanStore(str(tmp_path / "history.db"))
repository = _catalog_repository(source, revision)
store.upsert_repository(repository)
runner = ScanJobRunner(store, gitea=GiteaClient())
runner._schedule_changed_repositories()
first = store.claim_next_job()
assert first is not None
assert first.requested_revision == revision
store.update_job(first.id, status=ScanJobStatus.SUPERSEDED)
report = scan_repository(
str(source),
source=ScanSource(
origin=SourceOrigin.GITEA,
provider="gitea",
repository_id=repository.id,
repository_full_name=repository.full_name,
branch=repository.default_branch,
requested_revision=revision,
actual_revision=revision,
trigger=ScanTrigger.MANUAL,
),
)
store.save(report)
runner._schedule_changed_repositories()
assert store.queue_stats()["queued"] == 0
changed = "abcdef0123456789abcdef0123456789abcdef01"
store.upsert_repository(repository.model_copy(update={"latest_revision": changed}))
runner._schedule_changed_repositories()
assert store.queue_stats()["queued"] == 1
disabled = repository.model_copy(
update={
"id": "gitea:65",
"provider_repository_id": "65",
"latest_revision": "1234567890abcdef1234567890abcdef12345678",
"policy": repository.policy.model_copy(update={"auto_scan": False}),
}
)
store.upsert_repository(disabled)
runner._schedule_changed_repositories()
assert store.queue_stats()["queued"] == 1
def test_acquisition_rejects_token_bearing_clone_outside_gitea_origin(tmp_path: Path) -> None:
repository = _catalog_repository(tmp_path, "0123456789abcdef0123456789abcdef01234567").model_copy(
update={"clone_url": "https://evil.example/Jens/Demo.git"}
)
acquirer = GitAcquirer(
tmp_path / "cache",
token="test-token",
allowed_clone_origin="https://gitea.local",
)
with pytest.raises(AcquisitionError, match="outside the configured Gitea origin"):
acquirer.acquire(repository, "job-1", repository.latest_revision)
+175
View File
@@ -0,0 +1,175 @@
from __future__ import annotations
from pathlib import Path
import pytest
from fastapi.testclient import TestClient
from app import main as main_module
from app.gitea import GiteaClient
from app.jobs import ScanJobRunner
from app.models import Repository, ScanJobStatus, ScanTrigger
from app.store import MaintenanceModeError, ScanStore
def _gitea_repository() -> Repository:
return Repository(
id="gitea:64",
provider="gitea",
provider_repository_id="64",
owner="Jens",
name="RepoPulse",
full_name="Jens/RepoPulse",
default_branch="main",
latest_revision="0123456789abcdef0123456789abcdef01234567",
)
def test_maintenance_mode_is_durable_authenticated_and_exposes_drain_state(tmp_path: Path, monkeypatch) -> None:
database = tmp_path / "history.db"
main_module.store = ScanStore(str(database))
main_module.job_runner = None
main_module.store.upsert_repository(_gitea_repository())
queued = main_module.store.enqueue_job(
"gitea:64",
_gitea_repository().latest_revision,
ScanTrigger.MANUAL,
)
monkeypatch.delenv("REPOPULSE_OPERATOR_TOKEN", raising=False)
monkeypatch.delenv("REPOPULSE_MAINTENANCE_TOKEN", raising=False)
client = TestClient(main_module.app)
assert client.post("/api/v1/operator/maintenance/enter").status_code == 503
monkeypatch.setenv("REPOPULSE_MAINTENANCE_TOKEN", "operator-secret")
assert client.get("/api/v1/meta").json()["mutation_authentication"] == "perimeter_required"
assert client.post("/api/v1/operator/maintenance/enter").status_code == 401
assert client.get("/api/v1/operator/maintenance").status_code == 401
headers = {"X-RepoPulse-Token": "operator-secret"}
entered = client.post("/api/v1/operator/maintenance/enter", headers=headers)
assert entered.status_code == 200
assert entered.json()["enabled"] is True
assert entered.json()["queue_depth"] == 1
assert entered.json()["active_jobs"] == 0
assert entered.json()["active_drained"] is True
assert entered.json()["drained"] is False
changed_at = entered.json()["changed_at"]
assert client.post("/api/v1/operator/maintenance/enter", headers=headers).status_code == 423
assert main_module.store.maintenance_status()["changed_at"] == changed_at
assert client.get("/api/v1/operator/maintenance", headers=headers).json()["enabled"] is True
assert client.get("/api/v1/meta").json()["maintenance"]["enabled"] is True
restarted = ScanStore(str(database))
assert restarted.maintenance_status()["enabled"] is True
assert restarted.get_job(queued.id).status == ScanJobStatus.QUEUED
assert client.post("/api/v1/operator/maintenance/exit").status_code == 401
exited = client.post("/api/v1/operator/maintenance/exit", headers=headers)
assert exited.status_code == 200
assert exited.json()["enabled"] is False
assert ScanStore(str(database)).maintenance_status()["enabled"] is False
def test_paused_mode_rejects_api_mutations_but_keeps_read_endpoints_available(tmp_path: Path, monkeypatch) -> None:
main_module.store = ScanStore(str(tmp_path / "history.db"))
main_module.job_runner = None
repository = _gitea_repository()
main_module.store.upsert_repository(repository)
queued = main_module.store.enqueue_job(
repository.id,
repository.latest_revision,
ScanTrigger.MANUAL,
)
monkeypatch.delenv("REPOPULSE_OPERATOR_TOKEN", raising=False)
monkeypatch.setenv("REPOPULSE_MAINTENANCE_TOKEN", "operator-secret")
client = TestClient(main_module.app)
headers = {"X-RepoPulse-Token": "operator-secret"}
assert client.post("/api/v1/operator/maintenance/enter", headers=headers).status_code == 200
mutation_requests = [
("/api/v1/scans", {}),
("/api/v1/repositories/sync", {}),
("/api/v1/repositories/gitea%3A64/scans", {"json": {"force": True}}),
("/api/v1/webhooks/gitea", {"content": b"not validated while paused"}),
("/api/v1/operator/database-recovery/acknowledge", {}),
]
for path, kwargs in mutation_requests:
response = client.post(path, **kwargs)
assert response.status_code == 423, (path, response.status_code, response.text)
assert client.get("/api/v1/repositories").status_code == 200
assert client.get("/api/v1/scan-jobs/" + queued.id).status_code == 200
assert main_module.store.queue_stats()["queued"] == 1
assert main_module.store.get_job(queued.id).status == ScanJobStatus.QUEUED
def test_paused_worker_makes_no_scheduler_writes_or_job_claims(tmp_path: Path) -> None:
store = ScanStore(str(tmp_path / "history.db"))
repository = _gitea_repository()
store.upsert_repository(repository)
queued = store.enqueue_job(repository.id, repository.latest_revision, ScanTrigger.SCHEDULED)
store.set_state("worker_heartbeat", "before-maintenance")
store.set_state("last_gitea_sync", "prior-sync")
store.set_maintenance_mode(True)
class CountingGitea:
configured = True
def __init__(self) -> None:
self.sync_calls = 0
def sync(self, _store):
self.sync_calls += 1
raise AssertionError("scheduler must not sync in maintenance mode")
gitea = CountingGitea()
runner = ScanJobRunner(store, gitea=gitea) # type: ignore[arg-type]
assert runner._run_iteration() is None
assert gitea.sync_calls == 0
assert store.get_state("worker_heartbeat") == "before-maintenance"
assert store.get_state("last_gitea_sync") == "prior-sync"
assert store.claim_next_job() is None
assert store.queue_stats()["queued"] == 1
assert store.get_job(queued.id).status == ScanJobStatus.QUEUED
def test_already_claimed_job_can_finish_during_maintenance(tmp_path: Path) -> None:
source = tmp_path / "source"
source.mkdir()
(source / "README.md").write_text("# Existing work\n", encoding="utf-8")
(source / "app.py").write_text("print('ok')\n", encoding="utf-8")
store = ScanStore(str(tmp_path / "history.db"))
repository = Repository(
id="local:existing-job",
provider="local",
provider_repository_id="existing-job",
owner="local",
name="Existing job",
full_name="local/Existing job",
default_branch="main",
)
store.upsert_repository(repository, source_path=str(source))
store.enqueue_job(repository.id, None, ScanTrigger.MANUAL)
claimed = store.claim_next_job()
assert claimed is not None
runner = ScanJobRunner(store, gitea=GiteaClient())
store.set_maintenance_mode(True)
assert store.maintenance_status()["enabled"] is True
assert store.queue_stats()["active"] == 1
runner._process(claimed)
assert store.get_job(claimed.id).status == ScanJobStatus.COMPLETED
assert store.queue_stats()["active"] == 0
assert store.maintenance_status()["enabled"] is True
def test_invalid_persisted_maintenance_state_fails_closed(tmp_path: Path) -> None:
store = ScanStore(str(tmp_path / "history.db"))
store.set_state("maintenance_mode", "not-json")
assert store.maintenance_status()["enabled"] is True
with pytest.raises(MaintenanceModeError):
store.enqueue_job("missing", "revision", ScanTrigger.MANUAL)
+86
View File
@@ -0,0 +1,86 @@
from __future__ import annotations
import sqlite3
from pathlib import Path
from app.models import (
DependencySignals,
RepositoryMetrics,
ScanReport,
)
from app.models import (
TestSignals as SignalsModel,
)
from app.scanner import scan_repository
from app.store import SCHEMA_VERSION, ScanStore
def test_v1_scan_schema_is_migrated_and_legacy_history_is_cataloged(tmp_path: Path) -> None:
database = tmp_path / "history.db"
report = ScanReport(
id="legacy-scan",
repository_path=str((tmp_path / "repo").resolve()),
repository_name="repo",
scanned_at="2026-09-12T00:00:00+00:00",
duration_ms=1,
metrics=RepositoryMetrics(),
languages={},
category_scores={},
health_score=100,
release_readiness="READY",
findings=[],
tests=SignalsModel(),
dependencies=DependencySignals(),
)
connection = sqlite3.connect(database)
connection.execute(
"""
CREATE TABLE scans (
id TEXT PRIMARY KEY,
repository_path TEXT NOT NULL,
repository_name TEXT NOT NULL,
scanned_at TEXT NOT NULL,
health_score INTEGER NOT NULL,
report_json TEXT NOT NULL
)
"""
)
connection.execute(
"INSERT INTO scans VALUES (?, ?, ?, ?, ?, ?)",
(report.id, report.repository_path, report.repository_name, report.scanned_at.isoformat(), 100, report.model_dump_json()),
)
connection.commit()
connection.close()
store = ScanStore(str(database))
assert store.get_state("missing") is None
assert store.repository_count() == 1
migrated = store.get("legacy-scan")
assert migrated is not None
assert migrated.source.provider == "local"
assert migrated.source.repository_id is not None
assert store.list_repositories()[0].last_scan_id == "legacy-scan"
connection = sqlite3.connect(database)
assert connection.execute("PRAGMA user_version").fetchone()[0] == SCHEMA_VERSION
connection.close()
def test_schema_upgrade_creates_a_timestamped_pre_migration_backup(tmp_path: Path) -> None:
database = tmp_path / "history.db"
repository = tmp_path / "repo"
repository.mkdir()
(repository / "README.md").write_text("# Demo\n", encoding="utf-8")
(repository / ".gitignore").write_text("*.db\n", encoding="utf-8")
original = ScanStore(str(database))
saved = original.save(scan_repository(str(repository)))
connection = sqlite3.connect(database)
connection.execute("PRAGMA user_version = 3")
connection.commit()
connection.close()
migrated = ScanStore(str(database))
assert list(tmp_path.glob("history.db.pre-v3-*.bak"))
assert migrated.get(saved.id) is not None
+12
View File
@@ -0,0 +1,12 @@
from app.reporting import content_disposition, safe_report_filename
def test_report_filenames_are_safe_and_support_unicode() -> None:
filename = safe_report_filename('repo/evil\r\n"<é>', "html")
disposition = content_disposition(filename)
assert filename.startswith("repopulse-repo_evil___")
assert "\r" not in disposition
assert "\n" not in disposition
assert 'filename="' in disposition
assert "filename*=UTF-8''" in disposition
+223
View File
@@ -0,0 +1,223 @@
import json
import subprocess
from pathlib import Path
import pytest
from app.models import Severity
from app.scanner import scan_repository
def _write_good_repo(root: Path) -> None:
(root / "README.md").write_text("# Demo\n", encoding="utf-8")
(root / "LICENSE").write_text("MIT\n", encoding="utf-8")
(root / ".gitignore").write_text(".env\n", encoding="utf-8")
(root / "src").mkdir()
(root / "tests").mkdir()
(root / ".gitea" / "workflows").mkdir(parents=True)
(root / ".gitea" / "workflows" / "check.yml").write_text("name: check\n", encoding="utf-8")
(root / "src" / "core.py").write_text("def answer():\n return 42\n", encoding="utf-8")
(root / "src" / "app.py").write_text("import src.core\n\ndef run():\n return src.core.answer()\n", encoding="utf-8")
(root / "tests" / "test_app.py").write_text("def test_app():\n assert True\n", encoding="utf-8")
def _realistic_fixture_secret() -> str:
parts = ["production", "secret", "value", "123456"]
return "-".join(parts)
def test_scan_collects_v1_signals_and_architecture(tmp_path: Path) -> None:
_write_good_repo(tmp_path)
report = scan_repository(str(tmp_path))
assert report.repository_name == tmp_path.name
assert report.metrics.source_files == 3
assert report.metrics.test_files == 1
assert report.languages["Python"] > 0
assert report.tests.ci_workflows == [".gitea/workflows/check.yml"]
assert report.release_gate is not None
assert report.release_gate.status in {"READY", "READY WITH WARNINGS"}
assert any(edge.source == "src/app.py" and edge.target == "src/core.py" for edge in report.architecture.edges)
assert any(item.path == "src/app.py" for item in report.hotspots)
assert 0 <= report.health_score <= 100
def test_scan_collects_debt_and_dependency_signals(tmp_path: Path) -> None:
_write_good_repo(tmp_path)
(tmp_path / "package.json").write_text(
'{"dependencies":{"react":"latest"},"devDependencies":{"vitest":"^3.0.0"}}',
encoding="utf-8",
)
(tmp_path / "src" / "app.py").write_text(
"# TODO improve this\n# FIXME remove workaround\nprint('hello')\n",
encoding="utf-8",
)
report = scan_repository(str(tmp_path))
assert report.metrics.todo == 1
assert report.metrics.fixme == 1
assert report.dependencies.declared_dependencies == 2
assert report.dependencies.unbounded_dependencies == 1
assert any(item.code == "dependency.unbounded" for item in report.findings)
def test_potential_secret_is_redacted_and_blocks_release(tmp_path: Path) -> None:
(tmp_path / "README.md").write_text("# Demo\n", encoding="utf-8")
(tmp_path / ".gitignore").write_text("*.db\n", encoding="utf-8")
secret_name = "API_" + "KEY"
(tmp_path / "app.py").write_text(
f"{secret_name} = 'super-secret-value-123456'\n",
encoding="utf-8",
)
report = scan_repository(str(tmp_path))
secret_findings = [item for item in report.findings if item.code == "security.secret-assignment"]
assert report.release_readiness == "BLOCKED"
assert report.release_gate is not None
assert report.release_gate.status == "BLOCKED"
assert secret_findings
assert secret_findings[0].severity == Severity.BLOCKER
assert "super-secret-example-value" not in secret_findings[0].message
def test_secret_detection_scans_real_literals_in_tests_but_ignores_obvious_templates(tmp_path: Path) -> None:
_write_good_repo(tmp_path)
realistic_secret = _realistic_fixture_secret()
token_key = "to" + "ken"
(tmp_path / "src" / "settings.py").write_text(
"secret = os.getenv('APP_SECRET')\n" + f"api_key = '{realistic_secret}'\n",
encoding="utf-8",
)
(tmp_path / "tests" / "test_transport.py").write_text(
token_key + ' = "' + realistic_secret + '"\nresponse = b"token=test-token upstream body"\n',
encoding="utf-8",
)
(tmp_path / "README.md").write_text("# Demo\n\n`token=example-documentation-value`\n", encoding="utf-8")
(tmp_path / ".env.development.example").write_text("APP_SECRET=example-development-secret\n", encoding="utf-8")
report = scan_repository(str(tmp_path))
secret_findings = [item for item in report.findings if item.code == "security.secret-assignment"]
assert len(secret_findings) == 2
assert {item.path for item in secret_findings} == {"src/settings.py", "tests/test_transport.py"}
assert all(item.guidance and item.guidance.remediation_class == "security remediation" for item in secret_findings)
assert secret_findings[0].line == 2
assert not any(item.code == "security.env-committed" for item in report.findings)
def test_scan_budget_reports_coverage_without_turning_truncation_into_a_blocker(tmp_path: Path) -> None:
_write_good_repo(tmp_path)
for index in range(4):
(tmp_path / f"module_{index}.py").write_text("print('ok')\n", encoding="utf-8")
from app.models import ScanLimits
report = scan_repository(str(tmp_path), limits=ScanLimits(max_files=2, max_text_bytes_per_file=2_000_000, max_findings=20, max_secret_findings=5))
assert report.coverage is not None and report.coverage.truncated is True
assert report.coverage.skipped_files_by_budget > 0
assert any(check.key == "analysis_coverage" and check.state.value == "warn" for check in report.release_gate.checks)
assert any(item.code == "analysis.findings-truncated" for item in report.findings) is False
def test_configuration_literal_secret_remains_a_blocker(tmp_path: Path) -> None:
_write_good_repo(tmp_path)
realistic_secret = _realistic_fixture_secret()
token_key = "to" + "ken"
(tmp_path / "settings.json").write_text(
json.dumps({token_key: realistic_secret}) + "\n",
encoding="utf-8",
)
report = scan_repository(str(tmp_path))
secret_findings = [item for item in report.findings if item.code == "security.secret-assignment"]
assert report.release_readiness == "BLOCKED"
assert [(item.path, item.line) for item in secret_findings] == [("settings.json", 1)]
def test_member_literal_secret_remains_a_blocker(tmp_path: Path) -> None:
_write_good_repo(tmp_path)
(tmp_path / "src" / "client.py").write_text(
"class Client:\n def __init__(self):\n self.api_key = 'production-secret-value-123456'\n",
encoding="utf-8",
)
report = scan_repository(str(tmp_path))
secret_findings = [item for item in report.findings if item.code == "security.secret-assignment"]
assert report.release_readiness == "BLOCKED"
assert [(item.path, item.line) for item in secret_findings] == [("src/client.py", 3)]
def test_documentation_only_repository_does_not_require_test_files(tmp_path: Path) -> None:
(tmp_path / "README.md").write_text("# Catalog entry\n", encoding="utf-8")
(tmp_path / ".gitignore").write_text(".DS_Store\n", encoding="utf-8")
report = scan_repository(str(tmp_path))
assert report.metrics.source_files == 0
assert not any(item.code == "testing.none-detected" for item in report.findings)
def test_scan_skips_symlinked_files_outside_repository(tmp_path: Path) -> None:
_write_good_repo(tmp_path)
external = tmp_path.parent / f"{tmp_path.name}-outside.py"
external.write_text("API_KEY = 'outside-secret-example-value'\n", encoding="utf-8")
linked = tmp_path / "linked.py"
try:
linked.symlink_to(external)
except (NotImplementedError, OSError):
pytest.skip("symlinks are unavailable in this environment")
report = scan_repository(str(tmp_path))
assert report.metrics.source_files == 3
assert all(finding.path != "linked.py" for finding in report.findings)
def test_scan_skips_symlinked_directories_outside_repository(tmp_path: Path) -> None:
_write_good_repo(tmp_path)
external = tmp_path.parent / f"{tmp_path.name}-outside-dir"
external.mkdir()
(external / "outside.py").write_text("API_KEY = 'outside-secret-example-value'\n", encoding="utf-8")
linked = tmp_path / "linked"
try:
linked.symlink_to(external, target_is_directory=True)
except (NotImplementedError, OSError):
pytest.skip("directory symlinks are unavailable in this environment")
report = scan_repository(str(tmp_path))
assert report.metrics.source_files == 3
assert all(finding.path != "linked/outside.py" for finding in report.findings)
def test_malformed_dependency_manifests_are_reported_without_crashing(tmp_path: Path) -> None:
(tmp_path / "README.md").write_text("# Demo\n", encoding="utf-8")
(tmp_path / ".gitignore").write_text("\n", encoding="utf-8")
(tmp_path / "package.json").write_text("[]", encoding="utf-8")
(tmp_path / "pyproject.toml").write_text("[project]\ndependencies = \"invalid\"\n", encoding="utf-8")
(tmp_path / "pom.xml").write_text("<project>", encoding="utf-8")
report = scan_repository(str(tmp_path))
codes = {finding.code for finding in report.findings}
assert "dependency.invalid-package-json" in codes
assert "dependency.invalid-pyproject" in codes
assert "dependency.invalid-pom" in codes
def test_empty_git_repository_does_not_fallback_to_untracked_files(tmp_path: Path) -> None:
(tmp_path / "stray.py").write_text("print('not tracked')\n", encoding="utf-8")
subprocess.run(["git", "init", "--quiet", str(tmp_path)], check=True)
report = scan_repository(str(tmp_path))
assert report.git.available is True
assert report.git.tracked_files == 0
assert report.metrics.files == 0
assert report.metrics.source_files == 0
+113
View File
@@ -0,0 +1,113 @@
from pathlib import Path
from app.signals import analyze_dependencies, detect_test_signals
def test_dependency_analysis_handles_multiple_ecosystems(tmp_path: Path) -> None:
package = tmp_path / "package.json"
package.write_text('{"dependencies":{"react":"latest"}}', encoding="utf-8")
requirements = tmp_path / "requirements.txt"
requirements.write_text("fastapi>=0.100\n", encoding="utf-8")
lock = tmp_path / "package-lock.json"
lock.write_text("{}", encoding="utf-8")
signals, findings = analyze_dependencies(
tmp_path,
{
"package.json": [package],
"requirements.txt": [requirements],
"package-lock.json": [lock],
},
)
assert signals.declared_dependencies == 2
assert signals.unbounded_dependencies == 2
assert signals.unpinned_dependencies == 0
assert "package-lock.json" in signals.lockfiles
assert any(item.code == "dependency.unbounded" for item in findings)
def test_python_dependency_ranges_only_flag_truly_unbounded_declarations(tmp_path: Path) -> None:
pyproject = tmp_path / "pyproject.toml"
pyproject.write_text(
"""
[project]
dependencies = [
"exact==1.2.3",
"compatible~=2.0",
"bounded>=3.0,<4.0",
"open>=5.0",
]
optional-dependencies.dev = ["pytest~=8.0"]
""".strip(),
encoding="utf-8",
)
signals, findings = analyze_dependencies(tmp_path, {"pyproject.toml": [pyproject]})
assert signals.declared_dependencies == 5
assert signals.unbounded_dependencies == 1
assert [finding.code for finding in findings] == ["dependency.python-unbounded"]
def test_test_signals_detect_ci_coverage_and_frameworks() -> None:
text_by_path = {
"package.json": '{"devDependencies":{"vitest":"^3.0.0","@playwright/test":"^1.0.0"}}',
}
signals = detect_test_signals(
["src/app.ts", "src/app.test.ts", ".gitea/workflows/ci.yml", "coverage/lcov.info"],
source_files=2,
test_files=1,
text_by_path=text_by_path,
)
assert "Vitest" in signals.frameworks
assert "Playwright" in signals.frameworks
assert signals.ci_workflows == [".gitea/workflows/ci.yml"]
assert signals.coverage_artifacts == ["coverage/lcov.info"]
assert signals.test_source_ratio == 50.0
def test_test_signals_distinguish_validation_from_deploy_only_workflows() -> None:
signals = detect_test_signals(
[".gitea/workflows/validate.yml", ".gitea/workflows/deploy.yml"],
source_files=1,
test_files=1,
text_by_path={
".gitea/workflows/validate.yml": "steps:\n - run: pytest --cov\n - run: npm run build\n",
".gitea/workflows/deploy.yml": "steps:\n - run: deploy production\n",
},
)
assert signals.ci_validation_workflows == [".gitea/workflows/validate.yml"]
assert signals.ci_build_workflows == [".gitea/workflows/validate.yml"]
assert signals.ci_deploy_only_workflows == [".gitea/workflows/deploy.yml"]
assert signals.coverage_evidence == [".gitea/workflows/validate.yml"]
def test_dependency_analysis_follows_local_requirements_includes_and_workspace_locks(tmp_path: Path) -> None:
packages = tmp_path / "packages" / "web"
packages.mkdir(parents=True)
package = packages / "package.json"
package.write_text('{"dependencies":{"shared":"workspace:*"}}', encoding="utf-8")
lock = tmp_path / "pnpm-lock.yaml"
lock.write_text("lockfileVersion: '9.0'\n", encoding="utf-8")
requirements = tmp_path / "requirements.txt"
included = tmp_path / "requirements-prod.txt"
requirements.write_text("-r requirements-prod.txt\n", encoding="utf-8")
included.write_text("fastapi==0.116.0\n", encoding="utf-8")
signals, findings = analyze_dependencies(
tmp_path,
{
"package.json": [package],
"pnpm-lock.yaml": [lock],
"requirements.txt": [requirements],
"requirements-prod.txt": [included],
},
)
assert signals.unbounded_dependencies == 0
assert signals.included_requirements == ["requirements-prod.txt"]
assert signals.declared_dependencies == 2
assert not any(item.code == "dependency.no-lockfile" for item in findings)
+59
View File
@@ -0,0 +1,59 @@
from __future__ import annotations
import subprocess
from pathlib import Path
from app.git_analysis import analyze_git
from app.scanner import _walk_files, scan_repository
def test_non_git_fallback_applies_root_gitignore_rules(tmp_path: Path) -> None:
(tmp_path / ".gitignore").write_text("ignored.py\nignored-dir/\n*.log\n!keep.log\n", encoding="utf-8")
(tmp_path / "kept.py").write_text("print('kept')\n", encoding="utf-8")
(tmp_path / "ignored.py").write_text("print('ignored')\n", encoding="utf-8")
(tmp_path / "keep.log").write_text("kept\n", encoding="utf-8")
(tmp_path / "drop.log").write_text("dropped\n", encoding="utf-8")
(tmp_path / "ignored-dir").mkdir()
(tmp_path / "ignored-dir" / "nested.py").write_text("print('nested')\n", encoding="utf-8")
files = {path.relative_to(tmp_path).as_posix() for path in _walk_files(tmp_path)}
assert "kept.py" in files
assert "keep.log" in files
assert "ignored.py" not in files
assert "drop.log" not in files
assert "ignored-dir/nested.py" not in files
def test_git_churn_attributes_renames_to_the_new_path(tmp_path: Path) -> None:
subprocess.run(["git", "init", "--quiet", str(tmp_path)], check=True)
subprocess.run(["git", "-C", str(tmp_path), "config", "user.email", "test@example.com"], check=True)
subprocess.run(["git", "-C", str(tmp_path), "config", "user.name", "RepoPulse Test"], check=True)
(tmp_path / "old.py").write_text("print('hello')\n", encoding="utf-8")
subprocess.run(["git", "-C", str(tmp_path), "add", "."], check=True)
subprocess.run(["git", "-C", str(tmp_path), "commit", "--quiet", "-m", "initial"], check=True)
subprocess.run(["git", "-C", str(tmp_path), "mv", "old.py", "new.py"], check=True)
subprocess.run(["git", "-C", str(tmp_path), "commit", "--quiet", "-m", "rename"], check=True)
(tmp_path / "new.py").write_text("print('hello world')\n", encoding="utf-8")
subprocess.run(["git", "-C", str(tmp_path), "commit", "--quiet", "-am", "edit after rename"], check=True)
signals, churn = analyze_git(tmp_path)
assert signals.renamed_files == ["new.py"]
assert churn.get("new.py", 0) > 0
assert "old.py" not in churn
def test_policy_suppresses_accepted_risk_without_erasing_the_observation(tmp_path: Path) -> None:
(tmp_path / "README.md").write_text("# Demo\n", encoding="utf-8")
(tmp_path / ".gitignore").write_text("\n", encoding="utf-8")
(tmp_path / ".repopulse.yml").write_text("suppressions:\n - security.secret-assignment\n", encoding="utf-8")
(tmp_path / "app.py").write_text("api_key = 'abcdefghijk1234'\n", encoding="utf-8")
report = scan_repository(str(tmp_path))
finding = next(item for item in report.findings if item.code == "security.secret-assignment")
assert finding.suppressed is True
assert report.category_scores["Security basics"] == 100
assert report.release_gate is not None
assert report.release_gate.checks[0].state.value == "pass"
+161
View File
@@ -0,0 +1,161 @@
import json
from datetime import UTC, datetime, timedelta
from pathlib import Path
import app.store as store_module
from app.scanner import scan_repository
from app.store import ScanStore
def test_scan_history_comparison_and_trend(tmp_path: Path) -> None:
repository = tmp_path / "repo"
repository.mkdir()
(repository / "README.md").write_text("# Demo\n", encoding="utf-8")
(repository / ".gitignore").write_text("*.db\n", encoding="utf-8")
store = ScanStore(str(tmp_path / "repopulse.db"))
first = store.save(scan_repository(str(repository)))
(repository / "app.py").write_text("print('hello')\n", encoding="utf-8")
(repository / "tests").mkdir()
(repository / "tests" / "test_app.py").write_text("def test_ok():\n assert True\n", encoding="utf-8")
second = store.save(scan_repository(str(repository)))
assert store.get(second.id) == second
assert store.recent(1) == [second]
assert store.previous_for_repository(second.repository_path, second.id) == first
comparison = store.compare(second)
assert comparison.previous_scan_id == first.id
assert comparison.health_score_delta == second.health_score - first.health_score
assert comparison.metric_deltas["source_files"] == 2
trend = store.trend(second.repository_path)
assert [item.scan_id for item in trend] == [first.id, second.id]
assert trend[-1].health_score == second.health_score
def test_comparison_uses_the_scan_before_the_selected_scan(tmp_path: Path) -> None:
repository = tmp_path / "repo"
repository.mkdir()
(repository / "README.md").write_text("# Demo\n", encoding="utf-8")
(repository / ".gitignore").write_text("*.db\n", encoding="utf-8")
store = ScanStore(str(tmp_path / "repopulse.db"))
first = store.save(scan_repository(str(repository)))
second = store.save(scan_repository(str(repository)))
third = store.save(scan_repository(str(repository)))
assert store.previous_for_repository(second.repository_path, second.id) == first
assert store.previous_for_repository(third.repository_path, third.id) == second
def test_repository_summary_exposes_compact_remediation_context(tmp_path: Path) -> None:
repository = tmp_path / "repo"
repository.mkdir()
(repository / "README.md").write_text("# Demo\n", encoding="utf-8")
(repository / ".gitignore").write_text(".env\n", encoding="utf-8")
(repository / ".env").write_text("TOKEN=actual-secret-value-123456\n", encoding="utf-8")
store = ScanStore(str(tmp_path / "repopulse.db"))
store.save(scan_repository(str(repository)))
(repository / ".env").unlink()
(repository / "app.py").write_text("api_key = 'actual-secret-value-123456'\n", encoding="utf-8")
store.save(scan_repository(str(repository)))
summary = store.list_repositories()[0]
assert summary.last_scan_blockers == 1
assert summary.last_scan_primary_signal is not None
assert summary.last_scan_primary_signal.code == "security.secret-assignment"
assert summary.last_scan_primary_signal.count == 1
assert summary.last_scan_added_findings >= 1
assert summary.last_scan_resolved_findings == 1
def test_corrupt_database_is_quarantined_and_recreated(tmp_path: Path) -> None:
database = tmp_path / "repopulse.db"
database.write_bytes(b"not a sqlite database")
store = ScanStore(str(database))
assert store.recent() == []
assert list(tmp_path.glob("repopulse.db.corrupt-*"))
assert store.database_recovery_status()["status"] == "degraded_recovered"
def test_schema_migration_records_the_durable_backup_artifact(tmp_path: Path) -> None:
database = tmp_path / "repopulse.db"
initial_store = ScanStore(str(database))
with initial_store._connection() as db:
db.execute("PRAGMA user_version = 3")
migrated_store = ScanStore(str(database))
recorded = migrated_store.get_state("last_schema_migration_backup")
assert recorded is not None
metadata = json.loads(recorded)
assert metadata["from_version"] == 3
assert metadata["to_version"] == store_module.SCHEMA_VERSION
assert (tmp_path / metadata["backup_file"]).is_file()
assert migrated_store.storage_stats()["last_schema_migration_backup"] == recorded
def test_history_is_bounded_per_repository(tmp_path: Path, monkeypatch) -> None:
repository = tmp_path / "repo"
repository.mkdir()
(repository / "README.md").write_text("# Demo\n", encoding="utf-8")
monkeypatch.setattr(store_module, "MAX_HISTORY_PER_REPOSITORY", 2)
store = ScanStore(str(tmp_path / "repopulse.db"))
first = store.save(scan_repository(str(repository)))
second = store.save(scan_repository(str(repository)))
third = store.save(scan_repository(str(repository)))
assert [item.id for item in store.recent_for_repository(third.repository_path)] == [third.id, second.id]
assert store.get(first.id) is None
def test_retention_compacts_old_detailed_payloads_and_expires_old_snapshots(tmp_path: Path, monkeypatch) -> None:
repository = tmp_path / "repo"
repository.mkdir()
(repository / "README.md").write_text("# Demo\n", encoding="utf-8")
(repository / ".gitignore").write_text("*.db\n", encoding="utf-8")
monkeypatch.setenv("REPOPULSE_FULL_REPORTS_PER_REPOSITORY", "1")
monkeypatch.setenv("REPOPULSE_REPORT_RETENTION_DAYS", "30")
store = ScanStore(str(tmp_path / "repopulse.db"))
first = store.save(scan_repository(str(repository)))
second = store.save(scan_repository(str(repository)))
compacted = store.get(first.id)
assert compacted is not None and compacted.compacted is True
assert compacted.findings == []
assert store.get(second.id) is not None
expired = second.model_copy(
update={"id": "expired-scan", "scanned_at": datetime.now(UTC) - timedelta(days=31)}
)
store.save(expired)
assert store.get("expired-scan") is None
def test_catalog_summary_uses_denormalized_fields_without_report_json_parsing(tmp_path: Path, monkeypatch) -> None:
repository = tmp_path / "repo"
repository.mkdir()
(repository / "README.md").write_text("# Demo\n", encoding="utf-8")
(repository / ".gitignore").write_text("*.db\n", encoding="utf-8")
store = ScanStore(str(tmp_path / "repopulse.db"))
store.save(scan_repository(str(repository)))
def fail_if_called(_row):
raise AssertionError("catalog list must not parse report_json")
monkeypatch.setattr(store, "_report_from_row", fail_if_called)
summary = store.list_repositories()[0]
assert summary.last_scan_id
storage = store.storage_stats()
assert storage["scan_count"] == 1
assert storage["mirror_count"] >= 0
assert storage["worktree_count"] >= 0