Gabriel Vidal
0ff9e40242
main.py (5146 lines, 106 routes) becomes an assembly only: one package per domain — core, files, settings, dashboards, conversations, diff, notifications, forms, runs, accounts, workers, models, cron, agents, projects, services, goals, memories, plans, templates — each exposing an APIRouter; the flat domain modules move into their package behind a barrel that keeps the old `import conversations` / `import projects` spellings. The shared singletons (store, meta_store, hub, indexer, …) are built once by core.state.build_state() and attached to app.state.ai; routes take them as the `deps: State` dependency and helpers as an explicit `deps: AppState`. conversations/pricing.py carries the per-model rates out of the parser. Verified: route table and OpenAPI byte-identical; 90 read endpoints golden-diffed against the monolith on a copy of the live data (identical); write routes smoke-tested; 66 backend tests pass. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
339 lines
13 KiB
Python
339 lines
13 KiB
Python
"""Real ``git diff`` of a conversation's live git worktree(s).
|
||
|
||
Conversations do their work in an isolated worktree (``scripts/new-worktree.sh``
|
||
→ ``~/worktrees/<repo>-<name>``), so until that branch merges its changes are
|
||
invisible to the canonical checkouts the container mounts — which is why
|
||
``editsdiff`` reconstructs the "not committed yet" view from the transcript.
|
||
Mounting the worktrees root (``WORKTREES_DIR``, read-only) lets us diff the real
|
||
thing instead, and a real diff beats the replay on every axis: it sees edits made
|
||
by shell tools (``sed``, ``npm``, a generator script), tracks renames, counts
|
||
untracked files, and is exact rather than approximate.
|
||
|
||
The translation trick: a worktree's ``.git`` is a *file* pointing at a **host**
|
||
path (``/home/<u>/projects/<slug>/.git/worktrees/<dir>``), which does not exist
|
||
inside the container. We parse it, map the host repo root onto its mounted
|
||
equivalent (``PROJECTS_DIR/<slug>`` or ``REPO_DIR``), and drive git with an
|
||
explicit ``--git-dir`` / ``--work-tree`` pair. The superproject's ``.git`` is
|
||
mounted read-only, hence ``--no-optional-locks`` everywhere.
|
||
|
||
The transcript replay stays as the fallback for conversations whose worktree has
|
||
already been torn down (see ``editsdiff``).
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import re
|
||
import subprocess
|
||
import threading
|
||
import time
|
||
from pathlib import Path, PurePosixPath
|
||
|
||
from diff.editsdiff import _unified_chunk
|
||
from diff.gitdiff import _parse_unified_diff
|
||
from projects import PROJECTS_DIR, REPO_DIR
|
||
|
||
WORKTREES_DIR = Path(os.environ.get("WORKTREES_DIR", "/worktrees"))
|
||
|
||
# `.git` in a linked worktree is a file: `gitdir: <abs path>`.
|
||
_GITDIR_RE = re.compile(r"^gitdir:\s*(\S.*?)\s*$", re.M)
|
||
_WT_MARKER = "/.git/worktrees/"
|
||
# An untracked file bigger than this is listed without content (a whole new
|
||
# 40 MB asset has no business being inlined into a diff response).
|
||
_MAX_NEW_FILE_BYTES = 512 * 1024
|
||
# Cap a blob served out of a worktree, mirroring gitdiff._MAX_BLOB_BYTES.
|
||
_MAX_BLOB_BYTES = 12 * 1024 * 1024
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# resolve a worktree directory → git handles
|
||
# --------------------------------------------------------------------------- #
|
||
def _container_gitdir(host_gitdir: str) -> tuple[Path, str] | None:
|
||
"""Map a worktree's host ``gitdir`` path → (container gitdir, repo token).
|
||
|
||
``/home/u/projects/ai-agent/.git/worktrees/ai-agent-x`` → the mounted
|
||
``PROJECTS_DIR/ai-agent/.git/worktrees/ai-agent-x`` + ``project:ai-agent``;
|
||
anything else is assumed to be the superproject (``REPO_DIR``).
|
||
"""
|
||
idx = host_gitdir.find(_WT_MARKER)
|
||
if idx < 0:
|
||
return None
|
||
host_root = PurePosixPath(host_gitdir[:idx])
|
||
wt_name = host_gitdir[idx + len(_WT_MARKER):].strip("/")
|
||
if not wt_name or "/" in wt_name:
|
||
return None
|
||
if host_root.parent.name == PROJECTS_DIR.name and host_root.name:
|
||
root, token = PROJECTS_DIR / host_root.name, f"project:{host_root.name}"
|
||
else:
|
||
root, token = REPO_DIR, "super:_"
|
||
gitdir = root / ".git" / "worktrees" / wt_name
|
||
return (gitdir, token) if gitdir.is_dir() else None
|
||
|
||
|
||
def resolve(dir_name: str) -> dict | None:
|
||
"""A live worktree directory basename → its git handles, or None.
|
||
|
||
None means "not usable": the directory is gone (torn down), isn't a linked
|
||
worktree, or its owning repo isn't mounted — all cases where the caller
|
||
should fall back to the transcript replay.
|
||
"""
|
||
safe = Path(dir_name or "").name
|
||
if not safe or safe.startswith("."):
|
||
return None
|
||
work_tree = WORKTREES_DIR / safe
|
||
dotgit = work_tree / ".git"
|
||
if not dotgit.is_file():
|
||
return None
|
||
try:
|
||
m = _GITDIR_RE.search(dotgit.read_text(encoding="utf-8", errors="replace")[:4096])
|
||
except OSError:
|
||
return None
|
||
if not m:
|
||
return None
|
||
resolved = _container_gitdir(m.group(1))
|
||
if not resolved:
|
||
return None
|
||
gitdir, token = resolved
|
||
wt = {"dir": safe, "workTree": work_tree, "gitDir": gitdir, "repo": token,
|
||
"branch": None}
|
||
wt["branch"] = (_git(wt, ["rev-parse", "--abbrev-ref", "HEAD"]) or "").strip() or None
|
||
return wt
|
||
|
||
|
||
def _git(wt: dict, args: list[str], timeout: int = 20) -> str | None:
|
||
cmd = ["git", "-c", "safe.directory=*", "--no-optional-locks",
|
||
f"--git-dir={wt['gitDir']}", f"--work-tree={wt['workTree']}", *args]
|
||
try:
|
||
out = subprocess.run(cmd, capture_output=True, text=True,
|
||
cwd=str(wt["workTree"]), timeout=timeout)
|
||
except (OSError, subprocess.SubprocessError):
|
||
return None
|
||
return out.stdout if out.returncode == 0 else None
|
||
|
||
|
||
def live_worktrees(summary: dict, meta: dict) -> list[dict]:
|
||
"""Every still-usable worktree a conversation created (newest first).
|
||
|
||
Worktrees the conversation explicitly removed are skipped, as are ones whose
|
||
directory no longer resolves — both mean "diff it from the transcript".
|
||
"""
|
||
seen: set[str] = set()
|
||
out: list[dict] = []
|
||
entries = (summary.get("worktreesAuto") or []) + (meta.get("worktrees") or [])
|
||
for e in entries:
|
||
name = (e or {}).get("dir") or (e or {}).get("name")
|
||
if not name or name in seen:
|
||
continue
|
||
seen.add(name)
|
||
if (e or {}).get("removedAt"):
|
||
continue
|
||
wt = resolve(name)
|
||
if wt:
|
||
out.append(wt)
|
||
return out
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# worktree → DiffResult
|
||
# --------------------------------------------------------------------------- #
|
||
def _untracked(wt: dict) -> list[str]:
|
||
out = _git(wt, ["ls-files", "--others", "--exclude-standard", "-z"])
|
||
return [p for p in (out or "").split("\0") if p]
|
||
|
||
|
||
def _new_file_entry(wt: dict, rel: str, *, stats: bool) -> tuple[str | None, dict | None]:
|
||
"""An untracked file as either a unified-diff chunk or a bare file model.
|
||
|
||
Returns ``(chunk, file)`` — exactly one is set (or neither, when the path is
|
||
unreadable). Binary / oversized / stats-only files skip the content read and
|
||
come back as a file model with no hunks.
|
||
"""
|
||
path = wt["workTree"] / rel
|
||
try:
|
||
if path.is_symlink() or not path.is_file():
|
||
return None, None
|
||
size = path.stat().st_size
|
||
data = b"" if size > _MAX_NEW_FILE_BYTES else path.read_bytes()
|
||
except OSError:
|
||
return None, None
|
||
binary = b"\0" in data[:8192]
|
||
bare = {"path": rel, "oldPath": None, "status": "added",
|
||
"additions": 0, "deletions": 0, "binary": binary,
|
||
"truncated": size > _MAX_NEW_FILE_BYTES, "hunks": []}
|
||
if binary or size > _MAX_NEW_FILE_BYTES:
|
||
return None, bare
|
||
text = data.decode("utf-8", errors="replace")
|
||
lines = len(text.splitlines())
|
||
if stats:
|
||
return None, {**bare, "additions": lines}
|
||
chunk = _unified_chunk(rel, "added", None, text)
|
||
if not chunk:
|
||
return None, {**bare, "additions": lines}
|
||
return chunk, None
|
||
|
||
|
||
def diff_one(wt: dict, *, stats: bool = False) -> list[dict]:
|
||
"""One worktree's tracked-changes-vs-HEAD + untracked files, as file models.
|
||
|
||
``stats`` skips the hunk bodies (the conversation panel only needs paths and
|
||
+/− counts, and it refetches on every SSE ping).
|
||
"""
|
||
key = (wt["dir"], stats)
|
||
now = time.monotonic()
|
||
with _cache_lock:
|
||
hit = _cache.get(key)
|
||
if hit and now - hit[0] < _CACHE_TTL:
|
||
return hit[1]
|
||
files = _diff_one(wt, stats=stats)
|
||
with _cache_lock:
|
||
_cache[key] = (now, files)
|
||
while len(_cache) > _CACHE_CAP:
|
||
_cache.pop(next(iter(_cache)))
|
||
return files
|
||
|
||
|
||
# One SSE ping invalidates both the stats and the full query, so the panel and
|
||
# the diff page ask within milliseconds of each other — a short TTL collapses
|
||
# that into one round of git calls per worktree.
|
||
_CACHE_TTL = 1.5
|
||
_CACHE_CAP = 16
|
||
_cache: dict[tuple[str, bool], tuple[float, list[dict]]] = {}
|
||
_cache_lock = threading.Lock()
|
||
|
||
|
||
def _diff_one(wt: dict, *, stats: bool) -> list[dict]:
|
||
files: list[dict] = []
|
||
if stats:
|
||
raw = _git(wt, ["diff", "HEAD", "--numstat", "-M", "-z"]) or ""
|
||
status = _status_map(wt)
|
||
for path, add, dele in _parse_numstat_z(raw):
|
||
files.append({"path": path, "oldPath": None,
|
||
"status": status.get(path, "modified"),
|
||
"additions": add, "deletions": dele, "binary": False,
|
||
"truncated": False, "hunks": []})
|
||
else:
|
||
patch = _git(wt, ["diff", "HEAD", "--no-color", "--no-ext-diff",
|
||
"-U3", "-M"]) or ""
|
||
files += _parse_unified_diff(patch)
|
||
|
||
chunks: list[str] = []
|
||
for rel in _untracked(wt):
|
||
chunk, bare = _new_file_entry(wt, rel, stats=stats)
|
||
if chunk:
|
||
chunks.append(chunk)
|
||
elif bare:
|
||
files.append(bare)
|
||
files += _parse_unified_diff("\n".join(chunks))
|
||
|
||
for f in files:
|
||
f["repo"] = wt["repo"]
|
||
f["wt"] = wt["dir"]
|
||
return files
|
||
|
||
|
||
_STATUS_LETTERS = {"A": "added", "D": "deleted", "R": "renamed", "C": "added"}
|
||
|
||
|
||
def _status_map(wt: dict) -> dict[str, str]:
|
||
"""path → diff status, so the hunk-less ``stats`` payload isn't all
|
||
"modified". ``--name-status -z`` emits ``<letter>\\0<path>`` (renames carry
|
||
a second path field, which is the one we key on)."""
|
||
fields = [f for f in (_git(wt, ["diff", "HEAD", "--name-status", "-M", "-z"]) or "").split("\0") if f != ""]
|
||
out: dict[str, str] = {}
|
||
i = 0
|
||
while i + 1 < len(fields):
|
||
letter = fields[i][:1]
|
||
take = 3 if letter in ("R", "C") else 2
|
||
path = fields[i + take - 1]
|
||
out[path] = _STATUS_LETTERS.get(letter, "modified")
|
||
i += take
|
||
return out
|
||
|
||
|
||
def _parse_numstat_z(raw: str) -> list[tuple[str, int, int]]:
|
||
"""``git diff --numstat -z`` → (path, additions, deletions) triples.
|
||
|
||
With ``-z`` a rename emits its two paths as separate NUL-terminated fields
|
||
after the counts, so the record length varies — hence the manual walk.
|
||
"""
|
||
fields = [f for f in raw.split("\0")]
|
||
out: list[tuple[str, int, int]] = []
|
||
i = 0
|
||
while i < len(fields):
|
||
rec = fields[i]
|
||
i += 1
|
||
if not rec:
|
||
continue
|
||
parts = rec.split("\t")
|
||
if len(parts) < 3:
|
||
continue
|
||
add, dele, path = parts[0], parts[1], parts[2]
|
||
if not path: # rename: the old + new paths are the next two fields
|
||
if i + 1 >= len(fields):
|
||
break
|
||
path = fields[i + 1]
|
||
i += 2
|
||
out.append((path, int(add) if add.isdigit() else 0,
|
||
int(dele) if dele.isdigit() else 0))
|
||
return out
|
||
|
||
|
||
def subtitle(wts: list[dict], n: int, extra: int = 0) -> str:
|
||
"""The DiffResult subtitle: which worktree(s), how many files, how they were
|
||
read. ``extra`` counts files merged in from the transcript replay because
|
||
they live outside the worktree(s)."""
|
||
where = ", ".join(w["branch"] or w["dir"] for w in wts)
|
||
tail = f" (+{extra} outside it, replayed)" if extra else ""
|
||
return (f"worktree {where} · {n} file{'s' if n != 1 else ''} · "
|
||
f"uncommitted, read from the working tree{tail}")
|
||
|
||
|
||
def worktree_diff(summary: dict, meta: dict, title: str, *,
|
||
stats: bool = False) -> dict | None:
|
||
"""The conversation's uncommitted work, read from its live worktree(s).
|
||
|
||
None when no worktree resolves — the caller then falls back to the
|
||
transcript replay. An empty (but non-None) result is meaningful: it means the
|
||
worktree really is clean.
|
||
"""
|
||
wts = live_worktrees(summary, meta)
|
||
if not wts:
|
||
return None
|
||
files: list[dict] = []
|
||
for wt in wts:
|
||
files += diff_one(wt, stats=stats)
|
||
files.sort(key=lambda f: (f.get("wt") or "", f["path"]))
|
||
add = sum(f["additions"] for f in files)
|
||
dele = sum(f["deletions"] for f in files)
|
||
n = len(files)
|
||
return {
|
||
"title": title or "Worktree changes",
|
||
"subtitle": subtitle(wts, n),
|
||
"files": files, "additions": add, "deletions": dele,
|
||
"commits": None, "approx": False, "source": "worktree",
|
||
"worktrees": [{"dir": w["dir"], "branch": w["branch"], "repo": w["repo"]}
|
||
for w in wts],
|
||
}
|
||
|
||
|
||
def blob_bytes(dir_name: str, path: str) -> bytes | None:
|
||
"""Raw bytes of a working-tree file inside a worktree (the diff's image side)."""
|
||
wt = resolve(dir_name)
|
||
if not wt:
|
||
return None
|
||
rel = Path(path)
|
||
if not path or rel.is_absolute() or ".." in rel.parts:
|
||
return None
|
||
root = wt["workTree"].resolve()
|
||
target = (root / rel).resolve()
|
||
try:
|
||
target.relative_to(root)
|
||
except ValueError:
|
||
return None
|
||
if not target.is_file():
|
||
return None
|
||
try:
|
||
data = target.read_bytes()
|
||
except OSError:
|
||
return None
|
||
return data if len(data) <= _MAX_BLOB_BYTES else None
|