from __future__ import annotations import json import re import tomllib import xml.etree.ElementTree as ET from collections import defaultdict from pathlib import Path from .models import DependencySignals, Finding, Severity, TestSignals LOCKFILES = { "package-lock.json", "pnpm-lock.yaml", "yarn.lock", "bun.lock", "bun.lockb", "uv.lock", "poetry.lock", "Pipfile.lock", "Cargo.lock", "go.sum", } MANIFEST_MAX_BYTES = 2_000_000 COVERAGE_NAMES = { "coverage.xml", "lcov.info", "cobertura.xml", ".coverage", "jacoco.xml", } COVERAGE_NAMES_CASEFOLD = {item.casefold() for item in COVERAGE_NAMES} def _python_dependency_name(declaration: str) -> str: return re.split(r"[<>=!~ @;\[]", declaration, maxsplit=1)[0].strip() def _python_constraint_kind(declaration: str) -> str: """Classify the useful subset of Python version constraints without penalising bounded ranges.""" value = declaration.split(";", maxsplit=1)[0].strip().lower() if " @ " in value or "==" in value: return "pinned" if "~=" in value or "^" in value: return "bounded" has_lower = ">" in value has_upper = "<" in value if has_lower and has_upper: return "bounded" return "unbounded" def _python_dependency_finding(declaration: str, relative: str) -> Finding | None: if _python_constraint_kind(declaration) != "unbounded": return None return Finding( code="dependency.python-unbounded", category="Dependencies", severity=Severity.WARNING, message=f"Python dependency has no bounded version range: {declaration.split(';')[0]}", path=relative, deduction=3, ) def detect_test_signals( relative_paths: list[str], source_files: int, test_files: int, text_by_path: dict[str, str], ) -> TestSignals: frameworks: set[str] = set() coverage: list[str] = [] ci: list[str] = [] ci_validation: list[str] = [] ci_build: list[str] = [] ci_deploy_only: list[str] = [] coverage_evidence: list[str] = [] for path in relative_paths: normalized = path.replace("\\", "/") name = Path(normalized).name name_lower = name.casefold() lower = normalized.lower() if name_lower in COVERAGE_NAMES_CASEFOLD or "/coverage/" in f"/{lower}": coverage.append(normalized) if lower.startswith((".github/workflows/", ".gitea/workflows/")) or name_lower in { ".gitlab-ci.yml", "jenkinsfile", "azure-pipelines.yml", }: ci.append(normalized) if (name_lower.startswith("test_") or "/tests/" in f"/{lower}") and name_lower.endswith(".py"): frameworks.add("pytest/unittest") if (".test." in name_lower or ".spec." in name_lower) and name_lower.endswith( (".ts", ".tsx", ".js", ".jsx") ): frameworks.add("JavaScript test runner") if name_lower.endswith("_test.go"): frameworks.add("go test") if name_lower.endswith(("tests.cs", "test.cs")): frameworks.add(".NET test") for path, text in text_by_path.items(): name = Path(path).name normalized = path.replace("\\", "/") lowered_text = text.casefold() if normalized in ci: validates = any( token in lowered_text for token in ("pytest", "vitest", "jest", "npm test", "pnpm test", "go test", "dotnet test", "ruff", "eslint", "typecheck", "py_compile") ) builds = any( token in lowered_text for token in ("npm run build", "pnpm build", "docker build", "vite build", "cargo build", "go build", "dotnet build") ) deploys = any(token in lowered_text for token in ("deploy", "release", "publish", "kubectl apply")) if validates: ci_validation.append(normalized) if builds: ci_build.append(normalized) if deploys and not validates: ci_deploy_only.append(normalized) if any(token in lowered_text for token in ("--cov", "coverage", "lcov", "codecov")): coverage_evidence.append(normalized) if name.casefold() in COVERAGE_NAMES_CASEFOLD: coverage_evidence.append(normalized) if name.casefold() == "package.json": try: data = json.loads(text) except (TypeError, json.JSONDecodeError): continue if not isinstance(data, dict): continue deps: dict[object, object] = {} for section_name in ( "dependencies", "devDependencies", "peerDependencies", "optionalDependencies", ): section = data.get(section_name, {}) if isinstance(section, dict): deps.update(section) if "vitest" in deps: frameworks.add("Vitest") if "jest" in deps: frameworks.add("Jest") if "@playwright/test" in deps: frameworks.add("Playwright") if "cypress" in deps: frameworks.add("Cypress") elif name.casefold() == "pyproject.toml": lowered = text.lower() if "pytest" in lowered: frameworks.add("pytest") ratio = round((test_files / source_files) * 100, 2) if source_files else None return TestSignals( frameworks=sorted(frameworks), coverage_artifacts=sorted(set(coverage))[:30], ci_workflows=sorted(set(ci))[:30], ci_validation_workflows=sorted(set(ci_validation))[:30], ci_build_workflows=sorted(set(ci_build))[:30], ci_deploy_only_workflows=sorted(set(ci_deploy_only))[:30], coverage_evidence=sorted(set(coverage_evidence))[:30], test_source_ratio=ratio, ) def analyze_dependencies( root: Path, files_by_name: dict[str, list[Path]], ) -> tuple[DependencySignals, list[Finding]]: findings: list[Finding] = [] manifests: list[str] = [] lockfiles: list[str] = [] dependency_origins: dict[str, set[str]] = defaultdict(set) declared = 0 unbounded = 0 unpinned = 0 for lock_name in sorted(LOCKFILES): for path in files_by_name.get(lock_name, []): lockfiles.append(str(path.relative_to(root)).replace("\\", "/")) for package_json in files_by_name.get("package.json", []): relative = str(package_json.relative_to(root)).replace("\\", "/") manifests.append(relative) try: if package_json.stat().st_size > MANIFEST_MAX_BYTES: raise ValueError("package.json is too large to parse") data = json.loads(package_json.read_text(encoding="utf-8")) if not isinstance(data, dict): raise TypeError("package.json root must be an object") except (OSError, UnicodeDecodeError, json.JSONDecodeError, TypeError): findings.append( Finding( code="dependency.invalid-package-json", category="Dependencies", severity=Severity.WARNING, message="package.json could not be parsed safely", path=relative, deduction=8, ) ) continue deps: dict[str, object] = {} invalid_sections = False for section_name in ("dependencies", "devDependencies", "peerDependencies", "optionalDependencies"): section = data.get(section_name, {}) if section is None: continue if not isinstance(section, dict): invalid_sections = True continue deps.update(section) if invalid_sections: findings.append( Finding( code="dependency.invalid-package-json", category="Dependencies", severity=Severity.WARNING, message="One or more package dependency sections are not objects", path=relative, deduction=8, ) ) declared += len(deps) for name, version in deps.items(): dependency_name = str(name) dependency_origins[dependency_name.casefold()].add(relative) raw = str(version).strip().lower() if raw in {"*", "latest", "x"} and not raw.startswith("workspace:"): unbounded += 1 findings.append( Finding( code="dependency.unbounded", category="Dependencies", severity=Severity.WARNING, message=f"Dependency {dependency_name} uses an unbounded version", path=relative, deduction=5, ) ) parent = package_json.parent package_roots = [parent, *[ancestor for ancestor in parent.parents if ancestor == root or root in ancestor.parents]] if not any( (package_root / lock).exists() for package_root in package_roots for lock in ("package-lock.json", "pnpm-lock.yaml", "yarn.lock", "bun.lock", "bun.lockb") ): findings.append(Finding(code="dependency.no-lockfile", category="Dependencies", severity=Severity.WARNING, message="JavaScript package has no recognized lockfile", path=relative, deduction=8)) requirement_paths = [ path for name, values in files_by_name.items() if name.casefold().startswith("requirements") and name.casefold().endswith(".txt") for path in values ] included_requirements: list[str] = [] processed_requirements: set[Path] = set() pending_requirements = list(requirement_paths) while pending_requirements: requirements = pending_requirements.pop(0) try: requirements = requirements.resolve(strict=True) except OSError: continue if requirements in processed_requirements or requirements != root and root not in requirements.parents: continue processed_requirements.add(requirements) relative = str(requirements.relative_to(root)).replace("\\", "/") manifests.append(relative) try: lines = requirements.read_text(encoding="utf-8").splitlines() except (OSError, UnicodeDecodeError): findings.append( Finding( code="dependency.unreadable-requirements", category="Dependencies", severity=Severity.WARNING, message="Python requirements file could not be read", path=relative, deduction=8, ) ) continue for line in lines: stripped = line.strip() if not stripped or stripped.startswith("#"): continue if stripped.startswith(("-r ", "--requirement ")): include = stripped.split(maxsplit=1)[1].strip() candidate = (requirements.parent / include).resolve() if candidate == root or root in candidate.parents: pending_requirements.append(candidate) included_requirements.append(str(candidate.relative_to(root)).replace("\\", "/")) continue if stripped.startswith("-"): continue declared += 1 name = _python_dependency_name(stripped) if name: dependency_origins[name.casefold()].add(relative) finding = _python_dependency_finding(stripped, relative) if finding: unbounded += 1 findings.append(finding) for pyproject in files_by_name.get("pyproject.toml", []): relative = str(pyproject.relative_to(root)).replace("\\", "/") manifests.append(relative) try: if pyproject.stat().st_size > MANIFEST_MAX_BYTES: raise ValueError("pyproject.toml is too large to parse") data = tomllib.loads(pyproject.read_text(encoding="utf-8")) if not isinstance(data, dict): raise TypeError("pyproject.toml root must be a table") except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError, TypeError): findings.append( Finding( code="dependency.invalid-pyproject", category="Dependencies", severity=Severity.WARNING, message="pyproject.toml could not be parsed safely", path=relative, deduction=8, ) ) continue project = data.get("project", {}) if not isinstance(project, dict): findings.append( Finding( code="dependency.invalid-pyproject", category="Dependencies", severity=Severity.WARNING, message="pyproject.toml project section is not a table", path=relative, deduction=8, ) ) project = {} project_deps = project.get("dependencies", []) if isinstance(project_deps, list): declared += len(project_deps) for dependency in project_deps: declaration = str(dependency) name = _python_dependency_name(declaration) if name: dependency_origins[name.casefold()].add(relative) finding = _python_dependency_finding(declaration, relative) if finding: unbounded += 1 findings.append(finding) elif project_deps is not None: findings.append( Finding( code="dependency.invalid-pyproject", category="Dependencies", severity=Severity.WARNING, message="pyproject.toml project.dependencies is not a list", path=relative, deduction=8, ) ) optional_deps = project.get("optional-dependencies", {}) if isinstance(optional_deps, dict): for dependencies in optional_deps.values(): if not isinstance(dependencies, list): continue declared += len(dependencies) for dependency in dependencies: declaration = str(dependency) name = _python_dependency_name(declaration) if name: dependency_origins[name.casefold()].add(relative) finding = _python_dependency_finding(declaration, relative) if finding: unbounded += 1 findings.append(finding) elif optional_deps is not None: findings.append( Finding( code="dependency.invalid-pyproject", category="Dependencies", severity=Severity.WARNING, message="pyproject.toml project.optional-dependencies is not a table", path=relative, deduction=8, ) ) for go_mod in files_by_name.get("go.mod", []): relative = str(go_mod.relative_to(root)).replace("\\", "/") manifests.append(relative) try: text = go_mod.read_text(encoding="utf-8") except (OSError, UnicodeDecodeError): findings.append( Finding( code="dependency.unreadable-go-mod", category="Dependencies", severity=Severity.WARNING, message="go.mod could not be read", path=relative, deduction=8, ) ) continue matches = list(re.finditer(r"(?m)^\s*([\w./-]+)\s+v\d[^\s]*", text)) if "require" in text and not matches: findings.append( Finding( code="dependency.invalid-go-mod", category="Dependencies", severity=Severity.WARNING, message="go.mod contains no parseable versioned dependencies", path=relative, deduction=8, ) ) for match in matches: declared += 1 dependency_origins[match.group(1).casefold()].add(relative) for cargo in files_by_name.get("Cargo.toml", []): relative = str(cargo.relative_to(root)).replace("\\", "/") manifests.append(relative) try: data = tomllib.loads(cargo.read_text(encoding="utf-8")) except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError): findings.append( Finding( code="dependency.invalid-cargo-toml", category="Dependencies", severity=Severity.WARNING, message="Cargo.toml could not be parsed safely", path=relative, deduction=8, ) ) continue cargo_deps = data.get("dependencies", {}) if isinstance(cargo_deps, dict): declared += len(cargo_deps) for name in cargo_deps: dependency_origins[str(name).casefold()].add(relative) for pom in files_by_name.get("pom.xml", []): relative = str(pom.relative_to(root)).replace("\\", "/") manifests.append(relative) try: xml_root = ET.parse(pom).getroot() except (OSError, UnicodeError, ET.ParseError): findings.append( Finding( code="dependency.invalid-pom", category="Dependencies", severity=Severity.WARNING, message="pom.xml could not be parsed safely", path=relative, deduction=8, ) ) continue dependencies = [element for element in xml_root.iter() if element.tag.split("}")[-1] == "dependency"] declared += len(dependencies) for dependency in dependencies: artifact = next((child.text for child in dependency if child.tag.split("}")[-1] == "artifactId"), None) if artifact: dependency_origins[artifact.casefold()].add(relative) duplicates = sorted(name for name, origins in dependency_origins.items() if len(origins) > 1) for name in duplicates[:25]: origins = sorted(dependency_origins[name]) findings.append(Finding(code="dependency.duplicate-manifest", category="Dependencies", severity=Severity.INFO, message=f"Dependency {name} is declared across multiple manifests", path=origins[0], deduction=1)) return ( DependencySignals( manifests=sorted(set(manifests)), lockfiles=sorted(set(lockfiles)), declared_dependencies=declared, unbounded_dependencies=unbounded, unpinned_dependencies=unpinned, duplicate_dependencies=duplicates[:50], included_requirements=sorted(set(included_requirements))[:50], ), findings, )