applepy/applepy/checks/catalogues.py

"""LOLBins / GTFO catalogues (live APIs with cache + bundled fallback), lolapps, cloud paths."""

from __future__ import annotations

import json
import os
from importlib import resources
from pathlib import Path
from typing import Any

from applepy.catalog_cache import (
    GTFOBINS_API_URL,
    LOOBINS_JSON_URL,
    fetch_json_url,
    gtfo_technique_summary,
    iter_gtfo_binaries,
    load_loobins_entries,
)
from applepy.catalog_policy import loobins_entry_is_report_noise
from applepy.context import RunContext
from applepy.findings import Finding, Severity
from applepy.registry import CheckRegistry


def _load_json(name: str) -> list | dict:
    pkg = resources.files("applepy.data")
    with (pkg / name).open(encoding="utf-8") as f:
        return json.load(f)


def _load_json_list_dicts(name: str) -> list[dict[str, Any]]:
    raw = _load_json(name)
    if isinstance(raw, list):
        return [x for x in raw if isinstance(x, dict)]
    return []


def _which(name: str, path_env: str) -> str | None:
    for d in path_env.split(os.pathsep):
        if not d:
            continue
        p = Path(d) / name
        if p.is_file() and os.access(p, os.X_OK):
            return str(p.resolve())
    return None


def _resolve_binary_path(name: str, path_env: str) -> str | None:
    for d in (
        "/usr/bin",
        "/bin",
        "/sbin",
        "/usr/sbin",
        "/usr/local/bin",
        "/opt/homebrew/bin",
    ):
        p = Path(d) / name
        if p.is_file():
            return str(p.resolve())
    return _which(name, path_env)


def _skip_catalog_fetch() -> bool:
    return os.environ.get("APPLEPY_SKIP_CATALOG_FETCH", "").strip() in ("1", "true", "yes")


def _gtfo_max() -> int | None:
    raw = os.environ.get("APPLEPY_GTFO_MAX", "").strip()
    if not raw:
        return None
    try:
        n = int(raw)
        return n if n > 0 else None
    except ValueError:
        return None


def _gtfo_detailed() -> bool:
    return os.environ.get("APPLEPY_GTFO_DETAILED", "").strip() in ("1", "true", "yes")


def _gtfo_list_absent() -> bool:
    return os.environ.get("APPLEPY_GTFO_LIST_ABSENT", "").strip() in ("1", "true", "yes")


def _gtfo_rollup_max_lines() -> int:
    raw = os.environ.get("APPLEPY_GTFO_ROLLUP_MAX_LINES", "").strip()
    if not raw:
        return 120
    try:
        n = int(raw)
        return max(20, min(n, 5000))
    except ValueError:
        return 120


def register(registry: CheckRegistry) -> None:
    registry.register("cat_lolbins", check_lolbins, phases=("unprivileged",))
    registry.register("cat_gtfo", check_gtfo, phases=("unprivileged",))
    registry.register("cat_lolapps", check_lolapps, phases=("unprivileged",))
    registry.register("cat_lottunnels", check_lottunnels, phases=("unprivileged",))
    registry.register("cat_cloud_paths", check_cloud_credential_paths, phases=("unprivileged",))


def check_lolbins(ctx: RunContext) -> list[Finding]:
    findings: list[Finding] = []
    source_note = ""
    rows: list[dict] = []

    if _skip_catalog_fetch():
        rows = load_loobins_entries(_load_json_list_dicts("lolbins_macos.json"))
        source_note = "bundled lolbins_macos.json (APPLEPY_SKIP_CATALOG_FETCH)"
    else:
        parsed, status = fetch_json_url(LOOBINS_JSON_URL)
        if isinstance(parsed, list):
            rows = load_loobins_entries(parsed)
            source_note = f"https://www.loobins.io/loobins.json ({status})"
        else:
            rows = load_loobins_entries(_load_json_list_dicts("lolbins_macos.json"))
            source_note = f"fetch_unavailable_using_bundled ({status})"

    path_env = os.environ.get("PATH", "/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/opt/homebrew/bin")
    present_lines: list[str] = []
    missing_lines: list[str] = []
    per_entry: list[Finding] = []
    noise_omitted: list[str] = []

    for row in rows:
        name = row["name"]
        skip_detail = loobins_entry_is_report_noise(name)
        if skip_detail:
            noise_omitted.append(name)
        doc_paths = row.get("paths") or []
        note = str(row.get("short_description", "") or "")
        hit = None
        for p in doc_paths:
            if Path(p).is_file():
                hit = p
                break
        if not hit:
            hit = _resolve_binary_path(name, path_env)
        if hit:
            present_lines.append(f"{name} -> {hit}")
            if not skip_detail:
                per_entry.append(
                    Finding(
                        id=f"loob-api-{name.replace(' ', '_').replace('/', '_')}",
                        title=f"LOOBins: {name} present on host",
                        category="LOLBins",
                        severity=Severity.INFORMATIONAL,
                        description=(
                            "LOOBins documents this binary as a macOS living-off-the-land candidate. "
                            "Presence alone is not a vulnerability; prioritise entries that sit outside "
                            "your standard build or that support execution or download in sensitive contexts."
                        ),
                        evidence=f"path={hit}\nnotes={note}\nsource={source_note}",
                        worksheet="LOLBins",
                        mitre_techniques=("T1105", "T1059.002", "T1059.007"),
                        risk="Living-off-the-land primitives may support download or execution chains.",
                        impact="Attackers can avoid dropping custom tools for early-stage actions.",
                        remediation="Restrict script execution and outbound paths per policy; monitor ancestry.",
                        references=("https://www.loobins.io/", "https://www.loobins.io/loobins.json"),
                    )
                )
        else:
            miss = f"{name}: documented on LOOBins; not at listed paths or common macOS prefixes — {note}"
            missing_lines.append(miss)
            if not skip_detail:
                per_entry.append(
                    Finding(
                        id=f"loob-miss-{name.replace(' ', '_').replace('/', '_')}",
                        title=f"LOOBins: {name} not at canonical paths",
                        category="LOLBins",
                        severity=Severity.INFORMATIONAL,
                        description=(
                            "Listed on LOOBins but not resolved under documented paths, standard prefixes, or PATH. "
                            "This often reflects a relocated binary, a bundle-only install, or a catalogue path skew — "
                            "confirm with `which` or your software inventory before treating as absent."
                        ),
                        evidence=f"{miss}\nsource={source_note}",
                        worksheet="LOLBins",
                        mitre_techniques=("T1059",),
                        remediation="Validate with `which`, mdfind, or inventory tooling.",
                        references=("https://www.loobins.io/", "https://www.loobins.io/loobins.json"),
                    )
                )

    noise_note = ""
    if noise_omitted:
        uniq = ", ".join(sorted({n for n in noise_omitted}))
        noise_note = (
            f"\nOmitted per-binary rows for ubiquitous UI entries ({uniq}); "
            "they remain in counts above. Set APPLEPY_CATALOG_INCLUDE_NOISE=1 to emit those rows."
        )

    findings.append(
        Finding(
            id="lol-001",
            title="LOLBins — LOOBins.io catalogue coverage",
            category="LOLBins",
            severity=Severity.INFORMATIONAL,
            description=(
                "Cross-check of the LOOBins macOS catalogue against this host (live JSON with cache under "
                "~/.cache/applepy, or bundled fallback). The summary row establishes scope; substantive binaries "
                "appear as individual findings below."
            ),
            evidence=(
                f"Source: {source_note}\n"
                f"entries={len(rows)} present_count={len(present_lines)} missing_count={len(missing_lines)}\n"
                "Set APPLEPY_CATALOG_OFFLINE=1 to use cache only; APPLEPY_SKIP_CATALOG_FETCH=1 for bundled only."
                f"{noise_note}"
            ),
            worksheet="LOLBins",
            mitre_techniques=("T1105", "T1059.002", "T1059.007"),
            risk="Living-off-the-land execution and download primitives aid stealthy operations.",
            impact="Attackers may avoid dropping custom malware for initial actions.",
            remediation="Restrict script execution and outbound connectivity per policy; monitor process ancestry.",
            references=("https://www.loobins.io/", "https://www.loobins.io/docs/api/pyloobins/"),
        )
    )
    findings.extend(per_entry)
    return findings


def check_gtfo(ctx: RunContext) -> list[Finding]:
    findings: list[Finding] = []
    path_env = os.environ.get("PATH", "/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/opt/homebrew/bin")

    if _skip_catalog_fetch():
        return _check_gtfo_bundled(path_env)

    parsed, status = fetch_json_url(GTFOBINS_API_URL)
    if not isinstance(parsed, dict):
        findings.append(
            Finding(
                id="lol-007",
                title="GTFOBins live catalogue unavailable — bundled macOS subset used",
                category="LOLBins",
                severity=Severity.INFORMATIONAL,
                description=(
                    "The scanner could not obtain a fresh GTFOBins `api.json` (network, timeout, or first-run "
                    "without cache). ApplePY fell back to the packaged `gtfo_macos.json` shortlist. "
                    "That is an expected resilience path, not a host failure."
                ),
                evidence=(
                    f"fetch_status={status}\n"
                    "Re-try when outbound HTTPS is available, or rely on the bundled list for this workstation. "
                    "Catalogue traffic uses ~/.cache/applepy; see APPLEPY_CATALOG_OFFLINE and "
                    "APPLEPY_SKIP_CATALOG_FETCH in README."
                ),
                worksheet="LOLBins",
                references=("https://gtfobins.org/api.json",),
            )
        )
        findings.extend(_check_gtfo_bundled(path_env))
        return findings

    binaries = iter_gtfo_binaries(parsed)
    gmax = _gtfo_max()
    if gmax is not None:
        binaries = binaries[:gmax]

    lines_hit: list[str] = []
    lines_miss: list[str] = []
    per_hit: list[Finding] = []
    detailed = _gtfo_detailed()

    for binary, entry in binaries:
        tech = gtfo_technique_summary(entry)
        hit = _resolve_binary_path(binary, path_env)
        if hit:
            lines_hit.append(f"{binary}: {hit} — {tech}")
            if detailed:
                per_hit.append(
                    Finding(
                        id=f"gtfo-api-{binary.replace('/', '_').replace(' ', '_')}",
                        title=f"GTFOBins catalogue: {binary} present on host",
                        category="LOLBins",
                        severity=Severity.INFORMATIONAL,
                        description=(
                            "This executable name appears in the GTFOBins reference set and resolved on disk. "
                            "GTFOBins is Linux- and Unix-centric; treat hits as **context** for dual-use tooling, "
                            "not as proof of misuse."
                        ),
                        evidence=f"path={hit}\nfunctions={tech}\napi_load={status}",
                        worksheet="LOLBins",
                        mitre_techniques=("T1059",),
                        references=(
                            "https://gtfobins.org/",
                            "https://gtfobins.org/api.json",
                            "https://mitre-attack.github.io/attack-navigator/"
                            "#layerURL=https://gtfobins.org/mitre.json",
                        ),
                    )
                )
        else:
            lines_miss.append(binary)

    if lines_miss and _gtfo_list_absent():
        findings.append(
            Finding(
                id="gtfo-absent-all",
                title="GTFOBins API — catalogue names not resolved on this host (full list)",
                category="LOLBins",
                severity=Severity.INFORMATIONAL,
                description=(
                    "Optional exhaustive list: every `executables` entry from the GTFOBins API that did not "
                    "resolve under standard macOS prefixes or PATH for this run. Most misses are normal on "
                    "macOS because the catalogue is Linux-heavy. Enable only when you explicitly need the "
                    "complete negative set (APPLEPY_GTFO_LIST_ABSENT=1)."
                ),
                evidence="\n".join(lines_miss) + f"\n\napi_load={status} total_missing={len(lines_miss)}",
                worksheet="LOLBins",
                references=("https://gtfobins.org/api.json",),
            )
        )

    summary_bits: list[str] = [
        f"api_status={status}",
        f"executables_checked={len(binaries)}",
        f"on_host={len(lines_hit)} not_found={len(lines_miss)}",
    ]
    if detailed:
        summary_bits.append("APPLEPY_GTFO_DETAILED=1: per-binary worksheet rows follow this summary.")
        summary_bits.append("Optional: APPLEPY_GTFO_MAX=N to cap catalogue size for testing.")
    else:
        cap = _gtfo_rollup_max_lines()
        summary_bits.append("On-host matches (capped for workbook readability):")
        summary_bits.extend(lines_hit[:cap])
        if len(lines_hit) > cap:
            summary_bits.append(
                f"... and {len(lines_hit) - cap} further on-host matches omitted "
                "(raise APPLEPY_GTFO_ROLLUP_MAX_LINES or set APPLEPY_GTFO_DETAILED=1)."
            )
        summary_bits.append(
            "Absent catalogue names are omitted by default (most are Linux-only). "
            "Set APPLEPY_GTFO_LIST_ABSENT=1 for a dedicated finding listing every absent name."
        )
        summary_bits.append("Optional: APPLEPY_GTFO_MAX=N to cap checks for testing.")

    findings.insert(
        0,
        Finding(
            id="lol-002",
            title="GTFOBins — API catalogue versus this host",
            category="LOLBins",
            severity=Severity.INFORMATIONAL,
            description=(
                "Comparison of the GTFOBins `api.json` executable set (cached under ~/.cache/applepy when "
                "fetched successfully) against standard macOS binary locations and PATH. Default output rolls "
                "matches into this row so spreadsheets stay usable; use APPLEPY_GTFO_DETAILED=1 for one row per "
                "on-host binary."
            ),
            evidence="\n".join(summary_bits),
            worksheet="LOLBins",
            mitre_techniques=("T1059",),
            references=(
                "https://gtfobins.org/",
                "https://gtfobins.org/api.json",
                "https://mitre-attack.github.io/attack-navigator/#layerURL=https://gtfobins.org/mitre.json",
            ),
        ),
    )
    findings.extend(per_hit)
    return findings


def _check_gtfo_bundled(path_env: str) -> list[Finding]:
    data = _load_json_list_dicts("gtfo_macos.json")
    lines_hit: list[str] = []
    lines_miss: list[str] = []
    for row in data:
        binary = row["binary"]
        paths = row.get("paths") or []
        tech = row.get("technique", "")
        hit = next((p for p in paths if Path(p).is_file()), None) or _resolve_binary_path(
            binary, path_env
        )
        if hit:
            lines_hit.append(f"{binary}: {hit} — {tech}")
        else:
            lines_miss.append(f"{binary}: not at bundled path — {tech}")

    return [
        Finding(
            id="lol-002b",
            title="GTFOBins — bundled macOS-oriented executable shortlist",
            category="LOLBins",
            severity=Severity.INFORMATIONAL,
            description=(
                "Curated dual-use Unix utilities from packaged `gtfo_macos.json`, used when live catalogue "
                "fetch is disabled or unreachable. Use this row to anchor discussion; it is narrower than the "
                "full GTFOBins API."
            ),
            evidence="\n".join(lines_hit) if lines_hit else "(none at resolved paths)",
            worksheet="LOLBins",
            mitre_techniques=("T1059",),
            references=("https://gtfobins.org/", "https://gtfobins.org/api.json"),
        ),
        *(
            [
                Finding(
                    id="lol-006",
                    title="GTFO-style entries — bundled path miss",
                    category="LOLBins",
                    severity=Severity.INFORMATIONAL,
                    description="Technique documented in bundled list; path not found on this host.",
                    evidence="\n".join(lines_miss),
                    worksheet="LOLBins",
                    remediation="Confirm binary location with `which` and vendor documentation.",
                    references=("https://gtfobins.org/",),
                )
            ]
            if lines_miss
            else []
        ),
    ]


def _apps_dir_list() -> list[str]:
    apps = Path("/Applications")
    if not apps.is_dir():
        return []
    return sorted(p.name for p in apps.iterdir() if p.is_dir() and p.suffix == ".app")


def check_lolapps(ctx: RunContext) -> list[Finding]:
    raw = _load_json("lolapps.json")
    targets = [str(t) for t in raw] if isinstance(raw, list) else []
    installed = _apps_dir_list()
    hits = []
    for t in targets:
        for app in installed:
            if t.lower() in app.lower():
                hits.append(app)
                break
    ev = "Matches in /Applications:\n" + ("\n".join(hits) if hits else "(none from curated list)")
    return [
        Finding(
            id="lol-003",
            title="Potentially unnecessary software (remote / virtualisation) — informational",
            category="LOLBins",
            severity=Severity.INFORMATIONAL,
            description="Presence-only match against a lolapps-style list; validate business need.",
            evidence=ev,
            worksheet="LOLBins",
            mitre_techniques=("T1219", "T1021.001"),
            remediation="Remove unused remote-access tools or enforce conditional access and logging.",
            references=("https://lolapps-project.github.io/",),
        )
    ]


def check_lottunnels(ctx: RunContext) -> list[Finding]:
    names = _load_json("lottunnels.json")
    home = ctx.home
    path_env = os.environ.get("PATH", "/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/opt/homebrew/bin")
    hits: list[str] = []
    for n in names:
        w = _which(n, path_env)
        if w:
            hits.append(f"{n} -> {w}")
            continue
        for cand in (Path("/usr/local/bin") / n, Path("/opt/homebrew/bin") / n, home / "bin" / n):
            if cand.is_file():
                hits.append(str(cand))
                break
    ev = "Tunnelling-related binaries:\n" + ("\n".join(hits) if hits else "(none found via PATH and common prefixes)")
    return [
        Finding(
            id="lol-004",
            title="Tunnelling applications identified",
            category="LOLBins",
            severity=Severity.INFORMATIONAL,
            description=(
                "Informational: searches PATH and common prefixes for names from a lottunnels-style list."
            ),
            evidence=ev,
            worksheet="LOLBins",
            mitre_techniques=("T1090", "T1572"),
            remediation="Inventory authorised tunnelling tools; block or alert on unknown egress paths.",
            references=("https://lottunnels.github.io/",),
        )
    ]


def check_cloud_credential_paths(ctx: RunContext) -> list[Finding]:
    """Presence-only checks for common cloud-credential path hints (no secret reads)."""
    home = ctx.home
    checks: list[tuple[str, Path]] = [
        ("AWS credentials", home / ".aws" / "credentials"),
        ("AWS config", home / ".aws" / "config"),
        ("Azure CLI", home / ".azure"),
        ("gcloud config", home / ".config" / "gcloud"),
        ("GCP application default credentials", home / ".config" / "gcloud" / "application_default_credentials.json"),
        ("GitHub CLI hosts", home / ".config" / "gh" / "hosts.yml"),
        ("GitHub CLI hosts (yaml)", home / ".config" / "gh" / "hosts.yaml"),
        ("Docker config", home / ".docker" / "config.json"),
        ("kubectl config", home / ".kube" / "config"),
        (".netrc", home / ".netrc"),
        (".git-credentials", home / ".git-credentials"),
        ("npmrc", home / ".npmrc"),
        ("SSH directory", home / ".ssh"),
        (".env in home", home / ".env"),
        ("Shell history (zsh)", home / ".zsh_history"),
        ("Shell history (bash)", home / ".bash_history"),
        ("Safari binary cookies", home / "Library" / "Cookies" / "Cookies.binarycookies"),
        ("Firefox profiles dir", home / "Library" / "Application Support" / "Firefox" / "Profiles"),
        ("Chrome user data dir", home / "Library" / "Application Support" / "Google" / "Chrome"),
    ]
    lines: list[str] = []
    for label, path in checks:
        if path.is_dir():
            lines.append(f"{label}: directory exists {path}")
        elif path.is_file():
            try:
                lines.append(f"{label}: file exists {path} ({path.stat().st_size} bytes)")
            except OSError as e:
                lines.append(f"{label}: {path} ({e})")
        else:
            lines.append(f"{label}: not present at {path}")
    gac = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", "").strip()
    if gac:
        gp = Path(gac).expanduser()
        try:
            if gp.is_file():
                lines.append(
                    f"GOOGLE_APPLICATION_CREDENTIALS: file exists {gp} ({gp.stat().st_size} bytes)"
                )
            else:
                lines.append(f"GOOGLE_APPLICATION_CREDENTIALS: set but path not a file ({gp})")
        except OSError as e:
            lines.append(f"GOOGLE_APPLICATION_CREDENTIALS: {gp} ({e})")
    return [
        Finding(
            id="cloud-001",
            title="Cloud and developer credential locations (presence only)",
            category="Credentials",
            severity=Severity.INFORMATIONAL,
            description=(
                "Files and directories commonly targeted for cloud, browser, VCS credentials, and shell "
                "history (presence and size only); ApplePY does not read file contents."
            ),
            evidence="\n".join(lines),
            worksheet="Credentials",
            mitre_techniques=("T1552.001", "T1552.003", "T1528", "T1539"),
            risk="Stored credentials enable lateral movement and cloud resource abuse.",
            impact="Account takeover, data exfiltration, or supply-chain access via tokens.",
            remediation="Use short-lived credentials, hardware-backed storage, and MDM-managed keychain policies.",
            references=(
                "https://attack.mitre.org/matrices/enterprise/macos/",
            ),
        )
    ]