Initial curated RepoPulse source from 2524ccc3c1c27739996dccb2b7992989f88cd26d

backend/app/signals.py

@@ -0,0 +1,494 @@
+from __future__ import annotations
+
+import json
+import re
+import tomllib
+import xml.etree.ElementTree as ET
+from collections import defaultdict
+from pathlib import Path
+
+from .models import DependencySignals, Finding, Severity, TestSignals
+
+LOCKFILES = {
+    "package-lock.json",
+    "pnpm-lock.yaml",
+    "yarn.lock",
+    "bun.lock",
+    "bun.lockb",
+    "uv.lock",
+    "poetry.lock",
+    "Pipfile.lock",
+    "Cargo.lock",
+    "go.sum",
+}
+MANIFEST_MAX_BYTES = 2_000_000
+COVERAGE_NAMES = {
+    "coverage.xml",
+    "lcov.info",
+    "cobertura.xml",
+    ".coverage",
+    "jacoco.xml",
+}
+COVERAGE_NAMES_CASEFOLD = {item.casefold() for item in COVERAGE_NAMES}
+
+
+def _python_dependency_name(declaration: str) -> str:
+    return re.split(r"[<>=!~ @;\[]", declaration, maxsplit=1)[0].strip()
+
+
+def _python_constraint_kind(declaration: str) -> str:
+    """Classify the useful subset of Python version constraints without penalising bounded ranges."""
+
+    value = declaration.split(";", maxsplit=1)[0].strip().lower()
+    if " @ " in value or "==" in value:
+        return "pinned"
+    if "~=" in value or "^" in value:
+        return "bounded"
+    has_lower = ">" in value
+    has_upper = "<" in value
+    if has_lower and has_upper:
+        return "bounded"
+    return "unbounded"
+
+
+def _python_dependency_finding(declaration: str, relative: str) -> Finding | None:
+    if _python_constraint_kind(declaration) != "unbounded":
+        return None
+    return Finding(
+        code="dependency.python-unbounded",
+        category="Dependencies",
+        severity=Severity.WARNING,
+        message=f"Python dependency has no bounded version range: {declaration.split(';')[0]}",
+        path=relative,
+        deduction=3,
+    )
+
+
+def detect_test_signals(
+    relative_paths: list[str],
+    source_files: int,
+    test_files: int,
+    text_by_path: dict[str, str],
+) -> TestSignals:
+    frameworks: set[str] = set()
+    coverage: list[str] = []
+    ci: list[str] = []
+    ci_validation: list[str] = []
+    ci_build: list[str] = []
+    ci_deploy_only: list[str] = []
+    coverage_evidence: list[str] = []
+
+    for path in relative_paths:
+        normalized = path.replace("\\", "/")
+        name = Path(normalized).name
+        name_lower = name.casefold()
+        lower = normalized.lower()
+        if name_lower in COVERAGE_NAMES_CASEFOLD or "/coverage/" in f"/{lower}":
+            coverage.append(normalized)
+        if lower.startswith((".github/workflows/", ".gitea/workflows/")) or name_lower in {
+            ".gitlab-ci.yml",
+            "jenkinsfile",
+            "azure-pipelines.yml",
+        }:
+            ci.append(normalized)
+
+        if (name_lower.startswith("test_") or "/tests/" in f"/{lower}") and name_lower.endswith(".py"):
+            frameworks.add("pytest/unittest")
+        if (".test." in name_lower or ".spec." in name_lower) and name_lower.endswith(
+            (".ts", ".tsx", ".js", ".jsx")
+        ):
+            frameworks.add("JavaScript test runner")
+        if name_lower.endswith("_test.go"):
+            frameworks.add("go test")
+        if name_lower.endswith(("tests.cs", "test.cs")):
+            frameworks.add(".NET test")
+
+    for path, text in text_by_path.items():
+        name = Path(path).name
+        normalized = path.replace("\\", "/")
+        lowered_text = text.casefold()
+        if normalized in ci:
+            validates = any(
+                token in lowered_text
+                for token in ("pytest", "vitest", "jest", "npm test", "pnpm test", "go test", "dotnet test", "ruff", "eslint", "typecheck", "py_compile")
+            )
+            builds = any(
+                token in lowered_text
+                for token in ("npm run build", "pnpm build", "docker build", "vite build", "cargo build", "go build", "dotnet build")
+            )
+            deploys = any(token in lowered_text for token in ("deploy", "release", "publish", "kubectl apply"))
+            if validates:
+                ci_validation.append(normalized)
+            if builds:
+                ci_build.append(normalized)
+            if deploys and not validates:
+                ci_deploy_only.append(normalized)
+            if any(token in lowered_text for token in ("--cov", "coverage", "lcov", "codecov")):
+                coverage_evidence.append(normalized)
+        if name.casefold() in COVERAGE_NAMES_CASEFOLD:
+            coverage_evidence.append(normalized)
+        if name.casefold() == "package.json":
+            try:
+                data = json.loads(text)
+            except (TypeError, json.JSONDecodeError):
+                continue
+            if not isinstance(data, dict):
+                continue
+            deps: dict[object, object] = {}
+            for section_name in (
+                "dependencies",
+                "devDependencies",
+                "peerDependencies",
+                "optionalDependencies",
+            ):
+                section = data.get(section_name, {})
+                if isinstance(section, dict):
+                    deps.update(section)
+            if "vitest" in deps:
+                frameworks.add("Vitest")
+            if "jest" in deps:
+                frameworks.add("Jest")
+            if "@playwright/test" in deps:
+                frameworks.add("Playwright")
+            if "cypress" in deps:
+                frameworks.add("Cypress")
+        elif name.casefold() == "pyproject.toml":
+            lowered = text.lower()
+            if "pytest" in lowered:
+                frameworks.add("pytest")
+
+    ratio = round((test_files / source_files) * 100, 2) if source_files else None
+    return TestSignals(
+        frameworks=sorted(frameworks),
+        coverage_artifacts=sorted(set(coverage))[:30],
+        ci_workflows=sorted(set(ci))[:30],
+        ci_validation_workflows=sorted(set(ci_validation))[:30],
+        ci_build_workflows=sorted(set(ci_build))[:30],
+        ci_deploy_only_workflows=sorted(set(ci_deploy_only))[:30],
+        coverage_evidence=sorted(set(coverage_evidence))[:30],
+        test_source_ratio=ratio,
+    )
+
+
+def analyze_dependencies(
+    root: Path,
+    files_by_name: dict[str, list[Path]],
+) -> tuple[DependencySignals, list[Finding]]:
+    findings: list[Finding] = []
+    manifests: list[str] = []
+    lockfiles: list[str] = []
+    dependency_origins: dict[str, set[str]] = defaultdict(set)
+    declared = 0
+    unbounded = 0
+    unpinned = 0
+
+    for lock_name in sorted(LOCKFILES):
+        for path in files_by_name.get(lock_name, []):
+            lockfiles.append(str(path.relative_to(root)).replace("\\", "/"))
+
+    for package_json in files_by_name.get("package.json", []):
+        relative = str(package_json.relative_to(root)).replace("\\", "/")
+        manifests.append(relative)
+        try:
+            if package_json.stat().st_size > MANIFEST_MAX_BYTES:
+                raise ValueError("package.json is too large to parse")
+            data = json.loads(package_json.read_text(encoding="utf-8"))
+            if not isinstance(data, dict):
+                raise TypeError("package.json root must be an object")
+        except (OSError, UnicodeDecodeError, json.JSONDecodeError, TypeError):
+            findings.append(
+                Finding(
+                    code="dependency.invalid-package-json",
+                    category="Dependencies",
+                    severity=Severity.WARNING,
+                    message="package.json could not be parsed safely",
+                    path=relative,
+                    deduction=8,
+                )
+            )
+            continue
+        deps: dict[str, object] = {}
+        invalid_sections = False
+        for section_name in ("dependencies", "devDependencies", "peerDependencies", "optionalDependencies"):
+            section = data.get(section_name, {})
+            if section is None:
+                continue
+            if not isinstance(section, dict):
+                invalid_sections = True
+                continue
+            deps.update(section)
+        if invalid_sections:
+            findings.append(
+                Finding(
+                    code="dependency.invalid-package-json",
+                    category="Dependencies",
+                    severity=Severity.WARNING,
+                    message="One or more package dependency sections are not objects",
+                    path=relative,
+                    deduction=8,
+                )
+            )
+        declared += len(deps)
+        for name, version in deps.items():
+            dependency_name = str(name)
+            dependency_origins[dependency_name.casefold()].add(relative)
+            raw = str(version).strip().lower()
+            if raw in {"*", "latest", "x"} and not raw.startswith("workspace:"):
+                unbounded += 1
+                findings.append(
+                    Finding(
+                        code="dependency.unbounded",
+                        category="Dependencies",
+                        severity=Severity.WARNING,
+                        message=f"Dependency {dependency_name} uses an unbounded version",
+                        path=relative,
+                        deduction=5,
+                    )
+                )
+        parent = package_json.parent
+        package_roots = [parent, *[ancestor for ancestor in parent.parents if ancestor == root or root in ancestor.parents]]
+        if not any(
+            (package_root / lock).exists()
+            for package_root in package_roots
+            for lock in ("package-lock.json", "pnpm-lock.yaml", "yarn.lock", "bun.lock", "bun.lockb")
+        ):
+            findings.append(Finding(code="dependency.no-lockfile", category="Dependencies", severity=Severity.WARNING, message="JavaScript package has no recognized lockfile", path=relative, deduction=8))
+
+    requirement_paths = [
+        path
+        for name, values in files_by_name.items()
+        if name.casefold().startswith("requirements") and name.casefold().endswith(".txt")
+        for path in values
+    ]
+    included_requirements: list[str] = []
+    processed_requirements: set[Path] = set()
+    pending_requirements = list(requirement_paths)
+    while pending_requirements:
+        requirements = pending_requirements.pop(0)
+        try:
+            requirements = requirements.resolve(strict=True)
+        except OSError:
+            continue
+        if requirements in processed_requirements or requirements != root and root not in requirements.parents:
+            continue
+        processed_requirements.add(requirements)
+        relative = str(requirements.relative_to(root)).replace("\\", "/")
+        manifests.append(relative)
+        try:
+            lines = requirements.read_text(encoding="utf-8").splitlines()
+        except (OSError, UnicodeDecodeError):
+            findings.append(
+                Finding(
+                    code="dependency.unreadable-requirements",
+                    category="Dependencies",
+                    severity=Severity.WARNING,
+                    message="Python requirements file could not be read",
+                    path=relative,
+                    deduction=8,
+                )
+            )
+            continue
+        for line in lines:
+            stripped = line.strip()
+            if not stripped or stripped.startswith("#"):
+                continue
+            if stripped.startswith(("-r ", "--requirement ")):
+                include = stripped.split(maxsplit=1)[1].strip()
+                candidate = (requirements.parent / include).resolve()
+                if candidate == root or root in candidate.parents:
+                    pending_requirements.append(candidate)
+                    included_requirements.append(str(candidate.relative_to(root)).replace("\\", "/"))
+                continue
+            if stripped.startswith("-"):
+                continue
+            declared += 1
+            name = _python_dependency_name(stripped)
+            if name:
+                dependency_origins[name.casefold()].add(relative)
+            finding = _python_dependency_finding(stripped, relative)
+            if finding:
+                unbounded += 1
+                findings.append(finding)
+
+    for pyproject in files_by_name.get("pyproject.toml", []):
+        relative = str(pyproject.relative_to(root)).replace("\\", "/")
+        manifests.append(relative)
+        try:
+            if pyproject.stat().st_size > MANIFEST_MAX_BYTES:
+                raise ValueError("pyproject.toml is too large to parse")
+            data = tomllib.loads(pyproject.read_text(encoding="utf-8"))
+            if not isinstance(data, dict):
+                raise TypeError("pyproject.toml root must be a table")
+        except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError, TypeError):
+            findings.append(
+                Finding(
+                    code="dependency.invalid-pyproject",
+                    category="Dependencies",
+                    severity=Severity.WARNING,
+                    message="pyproject.toml could not be parsed safely",
+                    path=relative,
+                    deduction=8,
+                )
+            )
+            continue
+        project = data.get("project", {})
+        if not isinstance(project, dict):
+            findings.append(
+                Finding(
+                    code="dependency.invalid-pyproject",
+                    category="Dependencies",
+                    severity=Severity.WARNING,
+                    message="pyproject.toml project section is not a table",
+                    path=relative,
+                    deduction=8,
+                )
+            )
+            project = {}
+        project_deps = project.get("dependencies", [])
+        if isinstance(project_deps, list):
+            declared += len(project_deps)
+            for dependency in project_deps:
+                declaration = str(dependency)
+                name = _python_dependency_name(declaration)
+                if name:
+                    dependency_origins[name.casefold()].add(relative)
+                finding = _python_dependency_finding(declaration, relative)
+                if finding:
+                    unbounded += 1
+                    findings.append(finding)
+        elif project_deps is not None:
+            findings.append(
+                Finding(
+                    code="dependency.invalid-pyproject",
+                    category="Dependencies",
+                    severity=Severity.WARNING,
+                    message="pyproject.toml project.dependencies is not a list",
+                    path=relative,
+                    deduction=8,
+                )
+            )
+
+        optional_deps = project.get("optional-dependencies", {})
+        if isinstance(optional_deps, dict):
+            for dependencies in optional_deps.values():
+                if not isinstance(dependencies, list):
+                    continue
+                declared += len(dependencies)
+                for dependency in dependencies:
+                    declaration = str(dependency)
+                    name = _python_dependency_name(declaration)
+                    if name:
+                        dependency_origins[name.casefold()].add(relative)
+                    finding = _python_dependency_finding(declaration, relative)
+                    if finding:
+                        unbounded += 1
+                        findings.append(finding)
+        elif optional_deps is not None:
+            findings.append(
+                Finding(
+                    code="dependency.invalid-pyproject",
+                    category="Dependencies",
+                    severity=Severity.WARNING,
+                    message="pyproject.toml project.optional-dependencies is not a table",
+                    path=relative,
+                    deduction=8,
+                )
+            )
+
+    for go_mod in files_by_name.get("go.mod", []):
+        relative = str(go_mod.relative_to(root)).replace("\\", "/")
+        manifests.append(relative)
+        try:
+            text = go_mod.read_text(encoding="utf-8")
+        except (OSError, UnicodeDecodeError):
+            findings.append(
+                Finding(
+                    code="dependency.unreadable-go-mod",
+                    category="Dependencies",
+                    severity=Severity.WARNING,
+                    message="go.mod could not be read",
+                    path=relative,
+                    deduction=8,
+                )
+            )
+            continue
+        matches = list(re.finditer(r"(?m)^\s*([\w./-]+)\s+v\d[^\s]*", text))
+        if "require" in text and not matches:
+            findings.append(
+                Finding(
+                    code="dependency.invalid-go-mod",
+                    category="Dependencies",
+                    severity=Severity.WARNING,
+                    message="go.mod contains no parseable versioned dependencies",
+                    path=relative,
+                    deduction=8,
+                )
+            )
+        for match in matches:
+            declared += 1
+            dependency_origins[match.group(1).casefold()].add(relative)
+
+    for cargo in files_by_name.get("Cargo.toml", []):
+        relative = str(cargo.relative_to(root)).replace("\\", "/")
+        manifests.append(relative)
+        try:
+            data = tomllib.loads(cargo.read_text(encoding="utf-8"))
+        except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError):
+            findings.append(
+                Finding(
+                    code="dependency.invalid-cargo-toml",
+                    category="Dependencies",
+                    severity=Severity.WARNING,
+                    message="Cargo.toml could not be parsed safely",
+                    path=relative,
+                    deduction=8,
+                )
+            )
+            continue
+        cargo_deps = data.get("dependencies", {})
+        if isinstance(cargo_deps, dict):
+            declared += len(cargo_deps)
+            for name in cargo_deps:
+                dependency_origins[str(name).casefold()].add(relative)
+
+    for pom in files_by_name.get("pom.xml", []):
+        relative = str(pom.relative_to(root)).replace("\\", "/")
+        manifests.append(relative)
+        try:
+            xml_root = ET.parse(pom).getroot()
+        except (OSError, UnicodeError, ET.ParseError):
+            findings.append(
+                Finding(
+                    code="dependency.invalid-pom",
+                    category="Dependencies",
+                    severity=Severity.WARNING,
+                    message="pom.xml could not be parsed safely",
+                    path=relative,
+                    deduction=8,
+                )
+            )
+            continue
+        dependencies = [element for element in xml_root.iter() if element.tag.split("}")[-1] == "dependency"]
+        declared += len(dependencies)
+        for dependency in dependencies:
+            artifact = next((child.text for child in dependency if child.tag.split("}")[-1] == "artifactId"), None)
+            if artifact:
+                dependency_origins[artifact.casefold()].add(relative)
+
+    duplicates = sorted(name for name, origins in dependency_origins.items() if len(origins) > 1)
+    for name in duplicates[:25]:
+        origins = sorted(dependency_origins[name])
+        findings.append(Finding(code="dependency.duplicate-manifest", category="Dependencies", severity=Severity.INFO, message=f"Dependency {name} is declared across multiple manifests", path=origins[0], deduction=1))
+
+    return (
+        DependencySignals(
+            manifests=sorted(set(manifests)),
+            lockfiles=sorted(set(lockfiles)),
+            declared_dependencies=declared,
+            unbounded_dependencies=unbounded,
+            unpinned_dependencies=unpinned,
+            duplicate_dependencies=duplicates[:50],
+            included_requirements=sorted(set(included_requirements))[:50],
+        ),
+        findings,
+    )